Add changes to ensure session policy is enforced in LDAP STS (#12716)

- Bonus: Fix bug in webidentity sts that doesnt parse session policy correctly.

- update ldap.go to support session policy argument

Dockerfile.release

@@ -17,7 +17,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_USER_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
+    MINIO_CONFIG_ENV_FILE=config.env
 
 COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

Dockerfile.release.fips

@@ -17,7 +17,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_USER_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
-    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav"
+    MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
+    MINIO_CONFIG_ENV_FILE=config.env
 
 COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh

README.md

@@ -21,7 +21,10 @@ for more complete documentation.
 Run the following command to run the latest stable image of MinIO as a container using an ephemeral data volume:
 
 ```sh
-podman run -p 9000:9000 minio/minio server /data
+podman run \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  minio/minio server /data --console-address ":9001"
 ```
 
 The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. You can test the deployment using the MinIO Console, an embedded
@@ -206,6 +209,13 @@ MinIO Server comes with an embedded web based object browser. Point your web bro
 
 > NOTE: MinIO runs console on random port by default if you wish choose a specific port use `--console-address` to pick a specific interface and port.
 
+### Things to consider
+MinIO redirects browser access requests to the configured server port (i.e. `127.0.0.1:9000`) to the configured Console port. MinIO uses the hostname or IP address specified in the request when building the redirect URL. The URL and port *must* be accessible by the client for the redirection to work.
+
+For deployments behind a load balancer, proxy, or ingress rule where the MinIO host IP address or port is not public, use the `MINIO_BROWSER_REDIRECT_URL` environment variable to specify the external hostname for the redirect. The LB/Proxy must have rules for directing traffic to the Console port specifically.
+
+For example, consider a MinIO deployment behind a proxy `https://minio.example.net`, `https://console.minio.example.net` with rules for forwarding traffic on port :9000 and :9001 to MinIO and the MinIO Console respectively on the internal network. Set `MINIO_BROWSER_REDIRECT_URL` to `https://console.minio.example.net` to ensure the browser receives a valid reachable URL.
+
 | Dashboard                                                                                   | Creating a bucket                                                                           |
 | -------------                                                                               | -------------                                                                               |
 | ![Dashboard](https://github.com/minio/minio/blob/master/docs/screenshots/pic1.png?raw=true) | ![Dashboard](https://github.com/minio/minio/blob/master/docs/screenshots/pic2.png?raw=true) |

cmd/admin-handlers-users.go

@@ -956,6 +956,13 @@ func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Re
 		return
 	}
 
+	for _, nerr := range globalNotificationSys.DeleteServiceAccount(serviceAccount) {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
 	writeSuccessNoContent(w)
 }
 
@@ -984,9 +991,10 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	// Set delimiter value for "s3:delimiter" policy conditionals.
 	r.Header.Set("delimiter", SlashSeparator)
 
+	// Check if we are asked to return prefix usage
+	enablePrefixUsage := r.URL.Query().Get("prefix-usage") == "true"
+
 	isAllowedAccess := func(bucketName string) (rd, wr bool) {
-		// Use the following trick to filter in place
-		// https://github.com/golang/go/wiki/SliceTricks#filter-in-place
 		if globalIAMSys.IsAllowed(iampolicy.Args{
 			AccountName:     cred.AccessKey,
 			Groups:          cred.Groups,
@@ -1016,11 +1024,15 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 		return rd, wr
 	}
 
-	// Load the latest calculated data usage
-	dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI)
-	if err != nil {
-		// log the error, continue with the accounting response
-		logger.LogIf(ctx, err)
+	var dataUsageInfo madmin.DataUsageInfo
+	var err error
+	if !globalIsGateway {
+		// Load the latest calculated data usage
+		dataUsageInfo, err = loadDataUsageFromBackend(ctx, objectAPI)
+		if err != nil {
+			// log the error, continue with the accounting response
+			logger.LogIf(ctx, err)
+		}
 	}
 
 	// If etcd, dns federation configured list buckets from etcd.
@@ -1084,15 +1096,25 @@ func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Requ
 	for _, bucket := range buckets {
 		rd, wr := isAllowedAccess(bucket.Name)
 		if rd || wr {
-			var size uint64
 			// Fetch the data usage of the current bucket
+			var size uint64
 			if !dataUsageInfo.LastUpdate.IsZero() {
 				size = dataUsageInfo.BucketsUsage[bucket.Name].Size
 			}
+			// Fetch the prefix usage of the current bucket
+			var prefixUsage map[string]uint64
+			if enablePrefixUsage {
+				if pu, err := loadPrefixUsageFromBackend(ctx, objectAPI, bucket.Name); err == nil {
+					prefixUsage = pu
+				} else {
+					logger.LogIf(ctx, err)
+				}
+			}
 			acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{
-				Name:    bucket.Name,
-				Created: bucket.Created,
-				Size:    size,
+				Name:        bucket.Name,
+				Created:     bucket.Created,
+				Size:        size,
+				PrefixUsage: prefixUsage,
 				Access: madmin.AccountAccess{
 					Read:  rd,
 					Write: wr,

cmd/admin-handlers.go

@@ -19,17 +19,20 @@ package cmd
 
 import (
 	"context"
+	crand "crypto/rand"
 	"crypto/subtle"
 	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
+	"hash/crc32"
 	"io"
 	"math/rand"
 	"net/http"
 	"net/url"
 	"os"
 	"path"
+	"regexp"
 	"runtime"
 	"sort"
 	"strconv"
@@ -37,6 +40,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/klauspost/compress/zip"
 	"github.com/minio/kes"
 	"github.com/minio/madmin-go"
 	"github.com/minio/minio/internal/auth"
@@ -49,6 +53,7 @@ import (
 	"github.com/minio/minio/internal/logger/message/log"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
+	"github.com/secure-io/sio-go"
 )
 
 const (
@@ -903,8 +908,6 @@ func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}
-
-	w.(http.Flusher).Flush()
 }
 
 func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, action iampolicy.AdminAction) (ObjectLayer, auth.Credentials) {
@@ -1181,8 +1184,14 @@ func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) {
 			if err := enc.Encode(entry); err != nil {
 				return
 			}
-			w.(http.Flusher).Flush()
+			if len(traceCh) == 0 {
+				// Flush if nothing is queued
+				w.(http.Flusher).Flush()
+			}
 		case <-keepAliveTicker.C:
+			if len(traceCh) > 0 {
+				continue
+			}
 			if _, err := w.Write([]byte(" ")); err != nil {
 				return
 			}
@@ -1252,9 +1261,15 @@ func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Reque
 				if err := enc.Encode(log); err != nil {
 					return
 				}
+			}
+			if len(logCh) == 0 {
+				// Flush if nothing is queued
 				w.(http.Flusher).Flush()
 			}
 		case <-keepAliveTicker.C:
+			if len(logCh) > 0 {
+				continue
+			}
 			if _, err := w.Write([]byte(" ")); err != nil {
 				return
 			}
@@ -1558,41 +1573,163 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 	}
 	defer nsLock.Unlock(lkctx.Cancel)
 
-	go func() {
-		defer close(healthInfoCh)
+	hostAnonymizer := createHostAnonymizer()
+	// anonAddr - Anonymizes hosts in given input string.
+	anonAddr := func(addr string) string {
+		newAddr, found := hostAnonymizer[addr]
+		if found {
+			return newAddr
+		}
 
-		partialWrite(healthInfo) // Write first message with only version populated
+		// If we reach here, it means that the given addr doesn't contain any of the hosts.
+		// Return it as is. Can happen for drive paths in non-distributed mode
+		return addr
+	}
 
+	// anonymizedAddr - Updated the addr of the node info with anonymized one
+	anonymizeAddr := func(info madmin.NodeInfo) {
+		info.SetAddr(anonAddr(info.GetAddr()))
+	}
+
+	getAndWriteCPUs := func() {
 		if query.Get("syscpu") == "true" {
-			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, madmin.GetCPUs(deadlinedCtx, r.Host))
-			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, globalNotificationSys.GetCPUs(deadlinedCtx)...)
+			localCPUInfo := madmin.GetCPUs(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localCPUInfo)
+			healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, localCPUInfo)
+
+			peerCPUInfo := globalNotificationSys.GetCPUs(deadlinedCtx)
+			for _, cpuInfo := range peerCPUInfo {
+				anonymizeAddr(&cpuInfo)
+				healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, cpuInfo)
+			}
+
 			partialWrite(healthInfo)
 		}
+	}
 
+	getAndWritePartitions := func() {
 		if query.Get("sysdrivehw") == "true" {
-			healthInfo.Sys.Partitions = append(healthInfo.Sys.Partitions, madmin.GetPartitions(deadlinedCtx, r.Host))
-			healthInfo.Sys.Partitions = append(healthInfo.Sys.Partitions, globalNotificationSys.GetPartitions(deadlinedCtx)...)
+			localPartitions := madmin.GetPartitions(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localPartitions)
+			healthInfo.Sys.Partitions = append(healthInfo.Sys.Partitions, localPartitions)
+
+			peerPartitions := globalNotificationSys.GetPartitions(deadlinedCtx)
+			for _, p := range peerPartitions {
+				anonymizeAddr(&p)
+				healthInfo.Sys.Partitions = append(healthInfo.Sys.Partitions, p)
+			}
 			partialWrite(healthInfo)
 		}
+	}
 
+	getAndWriteOSInfo := func() {
 		if query.Get("sysosinfo") == "true" {
-			healthInfo.Sys.OSInfo = append(healthInfo.Sys.OSInfo, madmin.GetOSInfo(deadlinedCtx, r.Host))
-			healthInfo.Sys.OSInfo = append(healthInfo.Sys.OSInfo, globalNotificationSys.GetOSInfo(deadlinedCtx)...)
+			localOSInfo := madmin.GetOSInfo(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localOSInfo)
+			healthInfo.Sys.OSInfo = append(healthInfo.Sys.OSInfo, localOSInfo)
+
+			peerOSInfos := globalNotificationSys.GetOSInfo(deadlinedCtx)
+			for _, o := range peerOSInfos {
+				anonymizeAddr(&o)
+				healthInfo.Sys.OSInfo = append(healthInfo.Sys.OSInfo, o)
+			}
 			partialWrite(healthInfo)
 		}
+	}
 
+	getAndWriteMemInfo := func() {
 		if query.Get("sysmem") == "true" {
-			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, madmin.GetMemInfo(deadlinedCtx, r.Host))
-			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, globalNotificationSys.GetMemInfo(deadlinedCtx)...)
+			localMemInfo := madmin.GetMemInfo(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localMemInfo)
+			healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, localMemInfo)
+
+			peerMemInfos := globalNotificationSys.GetMemInfo(deadlinedCtx)
+			for _, m := range peerMemInfos {
+				anonymizeAddr(&m)
+				healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, m)
+			}
 			partialWrite(healthInfo)
 		}
+	}
 
+	anonymizeCmdLine := func(cmdLine string) string {
+		if !globalIsDistErasure {
+			// FS mode - single server - hard code to `server1`
+			return strings.Replace(cmdLine, globalLocalNodeName, "server1", -1)
+		}
+
+		// Server start command regex groups:
+		// 1 - minio server
+		// 2 - flags e.g. `--address :9000 --certs-dir /etc/minio/certs`
+		// 3 - pool args e.g. `https://node{01...16}.domain/data/disk{001...204} https://node{17...32}.domain/data/disk{001...204}`
+		re := regexp.MustCompile(`^(.*minio\s+server\s+)(--[^\s]+\s+[^\s]+\s+)*(.*)`)
+
+		// stays unchanged in the anonymized version
+		cmdLineWithoutPools := re.ReplaceAllString(cmdLine, `$1$2`)
+
+		// to be anonymized
+		poolsArgs := re.ReplaceAllString(cmdLine, `$3`)
+		var anonPools []string
+
+		if !(strings.Contains(poolsArgs, "{") && strings.Contains(poolsArgs, "}")) {
+			// No ellipses pattern. Anonymize host name from every pool arg
+			pools := strings.Fields(poolsArgs)
+			anonPools = make([]string, len(pools))
+			for _, arg := range pools {
+				anonPools = append(anonPools, anonAddr(arg))
+			}
+			return cmdLineWithoutPools + strings.Join(anonPools, " ")
+		}
+
+		// Ellipses pattern in pool args. Regex groups:
+		// 1 - server prefix
+		// 2 - number sequence for servers
+		// 3 - server suffix
+		// 4 - drive prefix (starting with /)
+		// 5 - number sequence for drives
+		// 6 - drive suffix
+		re = regexp.MustCompile(`([^\s^{]*)({\d+...\d+})?([^\s^{^/]*)(/[^\s^{]*)({\d+...\d+})?([^\s]*)`)
+		poolsMatches := re.FindAllStringSubmatch(poolsArgs, -1)
+
+		anonPools = make([]string, len(poolsMatches))
+		idxMap := map[int]string{
+			1: "spfx",
+			3: "ssfx",
+		}
+		for pi, poolsMatch := range poolsMatches {
+			// Replace the server prefix/suffix with anonymized ones
+			for idx, lbl := range idxMap {
+				if len(poolsMatch[idx]) > 0 {
+					poolsMatch[idx] = fmt.Sprintf("%s%d", lbl, crc32.ChecksumIEEE([]byte(poolsMatch[idx])))
+				}
+			}
+
+			// Remove the original pools args present at index 0
+			anonPools[pi] = strings.Join(poolsMatch[1:], "")
+		}
+		return cmdLineWithoutPools + strings.Join(anonPools, " ")
+	}
+
+	anonymizeProcInfo := func(p *madmin.ProcInfo) {
+		p.CmdLine = anonymizeCmdLine(p.CmdLine)
+		anonymizeAddr(p)
+	}
+
+	getAndWriteProcInfo := func() {
 		if query.Get("sysprocess") == "true" {
-			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, madmin.GetProcInfo(deadlinedCtx, r.Host))
-			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, globalNotificationSys.GetProcInfo(deadlinedCtx)...)
+			localProcInfo := madmin.GetProcInfo(deadlinedCtx, globalLocalNodeName)
+			anonymizeProcInfo(&localProcInfo)
+			healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, localProcInfo)
+			peerProcInfos := globalNotificationSys.GetProcInfo(deadlinedCtx)
+			for _, p := range peerProcInfos {
+				anonymizeProcInfo(&p)
+				healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, p)
+			}
 			partialWrite(healthInfo)
 		}
+	}
 
+	getAndWriteMinioConfig := func() {
 		if query.Get("minioconfig") == "true" {
 			config, err := readServerConfig(ctx, objectAPI)
 			if err != nil {
@@ -1606,47 +1743,101 @@ func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Reque
 			}
 			partialWrite(healthInfo)
 		}
+	}
 
+	getAndWriteDrivePerfInfo := func() {
 		if query.Get("perfdrive") == "true" {
-			healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, getDrivePerfInfos(deadlinedCtx, r.Host))
+			localDPI := getDrivePerfInfos(deadlinedCtx, globalLocalNodeName)
+			anonymizeAddr(&localDPI)
+			healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, localDPI)
 			partialWrite(healthInfo)
 
 			perfCh := globalNotificationSys.GetDrivePerfInfos(deadlinedCtx)
 			for perfInfo := range perfCh {
+				anonymizeAddr(&perfInfo)
 				healthInfo.Perf.Drives = append(healthInfo.Perf.Drives, perfInfo)
 				partialWrite(healthInfo)
 			}
-
-			partialWrite(healthInfo)
 		}
+	}
 
+	anonymizeNetPerfInfo := func(npi *madmin.NetPerfInfo) {
+		anonymizeAddr(npi)
+		rps := npi.RemotePeers
+		for idx, peer := range rps {
+			anonymizeAddr(&peer)
+			rps[idx] = peer
+		}
+		npi.RemotePeers = rps
+	}
+
+	getAndWriteNetPerfInfo := func() {
 		if globalIsDistErasure && query.Get("perfnet") == "true" {
-			healthInfo.Perf.Net = append(healthInfo.Perf.Net, globalNotificationSys.GetNetPerfInfo(deadlinedCtx))
+			localNPI := globalNotificationSys.GetNetPerfInfo(deadlinedCtx)
+			anonymizeNetPerfInfo(&localNPI)
+			healthInfo.Perf.Net = append(healthInfo.Perf.Net, localNPI)
+
 			partialWrite(healthInfo)
 
 			netInfos := globalNotificationSys.DispatchNetPerfChan(deadlinedCtx)
 			for netInfo := range netInfos {
+				anonymizeNetPerfInfo(&netInfo)
 				healthInfo.Perf.Net = append(healthInfo.Perf.Net, netInfo)
 				partialWrite(healthInfo)
 			}
-			partialWrite(healthInfo)
 
-			healthInfo.Perf.NetParallel = globalNotificationSys.GetParallelNetPerfInfo(deadlinedCtx)
+			ppi := globalNotificationSys.GetParallelNetPerfInfo(deadlinedCtx)
+			anonymizeNetPerfInfo(&ppi)
+			healthInfo.Perf.NetParallel = ppi
 			partialWrite(healthInfo)
 		}
+	}
+
+	anonymizeNetwork := func(network map[string]string) map[string]string {
+		anonNetwork := map[string]string{}
+		for endpoint, status := range network {
+			anonEndpoint := anonAddr(endpoint)
+			anonNetwork[anonEndpoint] = status
+		}
+		return anonNetwork
+
+	}
+
+	anonymizeDrives := func(drives []madmin.Disk) []madmin.Disk {
+		anonDrives := []madmin.Disk{}
+		for _, drive := range drives {
+			drive.Endpoint = anonAddr(drive.Endpoint)
+			anonDrives = append(anonDrives, drive)
+		}
+		return anonDrives
+	}
+
+	go func() {
+		defer close(healthInfoCh)
+
+		partialWrite(healthInfo) // Write first message with only version populated
+		getAndWriteCPUs()
+		getAndWritePartitions()
+		getAndWriteOSInfo()
+		getAndWriteMemInfo()
+		getAndWriteProcInfo()
+		getAndWriteMinioConfig()
+		getAndWriteDrivePerfInfo()
+		getAndWriteNetPerfInfo()
 
 		if query.Get("minioinfo") == "true" {
 			infoMessage := getServerInfo(ctx, r)
 			servers := []madmin.ServerInfo{}
 			for _, server := range infoMessage.Servers {
+				anonEndpoint := anonAddr(server.Endpoint)
 				servers = append(servers, madmin.ServerInfo{
 					State:      server.State,
-					Endpoint:   server.Endpoint,
+					Endpoint:   anonEndpoint,
 					Uptime:     server.Uptime,
 					Version:    server.Version,
 					CommitID:   server.CommitID,
-					Network:    server.Network,
-					Drives:     server.Disks,
+					Network:    anonymizeNetwork(server.Network),
+					Drives:     anonymizeDrives(server.Disks),
 					PoolNumber: server.PoolNumber,
 					MemStats: madmin.MemStats{
 						Alloc:      server.MemStats.Alloc,
@@ -1951,3 +2142,186 @@ func checkConnection(endpointStr string, timeout time.Duration) error {
 	defer xhttp.DrainBody(resp.Body)
 	return nil
 }
+
+// getRawDataer provides an interface for getting raw FS files.
+type getRawDataer interface {
+	GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error
+}
+
+// InspectDataHandler - GET /minio/admin/v3/inspect-data
+// ----------
+// Download file from all nodes in a zip format
+func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "InspectData")
+
+	// Validate request signature.
+	_, adminAPIErr := checkAdminRequestAuth(ctx, r, iampolicy.InspectDataAction, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL)
+		return
+	}
+	defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r))
+
+	o, ok := newObjectLayerFn().(getRawDataer)
+	if !ok {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+		return
+	}
+
+	volume := r.URL.Query().Get("volume")
+	file := r.URL.Query().Get("file")
+	if len(volume) == 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL)
+		return
+	}
+	if len(file) == 0 {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL)
+		return
+	}
+
+	var key [32]byte
+	// MUST use crypto/rand
+	n, err := crand.Read(key[:])
+	if err != nil || n != len(key) {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+	stream, err := sio.AES_256_GCM.Stream(key[:])
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+	// Zero nonce, we only use each key once, and 32 bytes is plenty.
+	nonce := make([]byte, stream.NonceSize())
+	encw := stream.EncryptWriter(w, nonce, nil)
+
+	defer encw.Close()
+
+	// Write a version for making *incompatible* changes.
+	// The AdminClient will reject any version it does not know.
+	w.Write([]byte{1})
+
+	// Write key first (without encryption)
+	_, err = w.Write(key[:])
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), r.URL)
+		return
+	}
+
+	// Initialize a zip writer which will provide a zipped content
+	// of profiling data of all nodes
+	zipWriter := zip.NewWriter(encw)
+	defer zipWriter.Close()
+
+	err = o.GetRawData(ctx, volume, file, func(r io.Reader, host, disk, filename string, size int64, modtime time.Time) error {
+		// Prefix host+disk
+		filename = path.Join(host, disk, filename)
+		header, zerr := zip.FileInfoHeader(dummyFileInfo{
+			name:    filename,
+			size:    size,
+			mode:    0600,
+			modTime: modtime,
+			isDir:   false,
+			sys:     nil,
+		})
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		header.Method = zip.Deflate
+		zwriter, zerr := zipWriter.CreateHeader(header)
+		if zerr != nil {
+			logger.LogIf(ctx, zerr)
+			return nil
+		}
+		if _, err = io.Copy(zwriter, r); err != nil {
+			logger.LogIf(ctx, err)
+		}
+		return nil
+	})
+	logger.LogIf(ctx, err)
+}
+
+func createHostAnonymizerForFSMode() map[string]string {
+	hostAnonymizer := map[string]string{
+		globalLocalNodeName: "server1",
+	}
+
+	apiEndpoints := getAPIEndpoints()
+	for _, ep := range apiEndpoints {
+		if len(ep) > 0 {
+			if url, err := xnet.ParseHTTPURL(ep); err == nil {
+				// In FS mode the drive names don't include the host.
+				// So mapping just the host should be sufficient.
+				hostAnonymizer[url.Host] = "server1"
+			}
+		}
+	}
+	return hostAnonymizer
+}
+
+// anonymizeHost - Add entries related to given endpoint in the host anonymizer map
+// The health report data can contain the hostname in various forms e.g. host, host:port,
+// host:port/drivepath, full url (http://host:port/drivepath)
+// The anonymizer map will have mappings for all these varients for efficiently replacing
+// any of these strings to the anonymized versions at the time of health report generation.
+func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum int, srvrNum int) {
+	if len(endpoint.Host) == 0 {
+		return
+	}
+
+	currentURL := endpoint.String()
+
+	// mapIfNotPresent - Maps the given key to the value only if the key is not present in the map
+	mapIfNotPresent := func(m map[string]string, key string, val string) {
+		_, found := m[key]
+		if !found {
+			m[key] = val
+		}
+	}
+
+	_, found := hostAnonymizer[currentURL]
+	if !found {
+		// In distributed setup, anonymized addr = 'poolNum.serverNum'
+		newHost := fmt.Sprintf("pool%d.server%d", poolNum, srvrNum)
+
+		// Hostname
+		mapIfNotPresent(hostAnonymizer, endpoint.Hostname(), newHost)
+
+		newHostPort := newHost
+		if len(endpoint.Port()) > 0 {
+			// Host + port
+			newHostPort = newHost + ":" + endpoint.Port()
+			mapIfNotPresent(hostAnonymizer, endpoint.Host, newHostPort)
+		}
+
+		newHostPortPath := newHostPort
+		if len(endpoint.Path) > 0 {
+			// Host + port + path
+			currentHostPortPath := endpoint.Host + endpoint.Path
+			newHostPortPath = newHostPort + endpoint.Path
+			mapIfNotPresent(hostAnonymizer, currentHostPortPath, newHostPortPath)
+		}
+
+		// Full url
+		hostAnonymizer[currentURL] = endpoint.Scheme + "://" + newHostPortPath
+	}
+}
+
+// createHostAnonymizer - Creats a map of various strings to corresponding anonymized names
+func createHostAnonymizer() map[string]string {
+	if !globalIsDistErasure {
+		return createHostAnonymizerForFSMode()
+	}
+
+	hostAnonymizer := map[string]string{}
+
+	for poolIdx, pool := range globalEndpoints {
+		for srvrIdx, endpoint := range pool.Endpoints {
+			anonymizeHost(hostAnonymizer, endpoint, poolIdx+1, srvrIdx+1)
+		}
+	}
+	return hostAnonymizer
+}

cmd/admin-handlers_test.go

@@ -77,7 +77,7 @@ func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, erro
 
 	// Setup admin mgmt REST API handlers.
 	adminRouter := mux.NewRouter()
-	registerAdminRouter(adminRouter, true, true)
+	registerAdminRouter(adminRouter, true)
 
 	return &adminErasureTestBed{
 		erasureDirs: erasureDirs,

cmd/admin-router.go

@@ -21,6 +21,8 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/klauspost/compress/gzhttp"
+	"github.com/klauspost/compress/gzip"
 	"github.com/minio/madmin-go"
 )
 
@@ -34,7 +36,7 @@ const (
 type adminAPIHandlers struct{}
 
 // registerAdminRouter - Add handler functions for each service REST API routes.
-func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool) {
+func registerAdminRouter(router *mux.Router, enableConfigOps bool) {
 
 	adminAPI := adminAPIHandlers{}
 	// Admin router
@@ -46,174 +48,183 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminAPIVersionPrefix,
 	}
 
+	gz := func(h http.HandlerFunc) http.HandlerFunc {
+		return h
+	}
+
+	wrapper, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
+	if err == nil {
+		gz = func(h http.HandlerFunc) http.HandlerFunc {
+			return wrapper(h).(http.HandlerFunc)
+		}
+	}
 	for _, adminVersion := range adminVersions {
 		// Restart and stop MinIO service.
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(httpTraceAll(adminAPI.ServiceHandler)).Queries("action", "{action:.*}")
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(gz(httpTraceAll(adminAPI.ServiceHandler))).Queries("action", "{action:.*}")
 		// Update MinIO servers.
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(httpTraceAll(adminAPI.ServerUpdateHandler)).Queries("updateURL", "{updateURL:.*}")
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(gz(httpTraceAll(adminAPI.ServerUpdateHandler))).Queries("updateURL", "{updateURL:.*}")
 
 		// Info operations
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(httpTraceAll(adminAPI.ServerInfoHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(gz(httpTraceAll(adminAPI.ServerInfoHandler)))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/inspect-data").HandlerFunc(httpTraceHdrs(adminAPI.InspectDataHandler)).Queries("volume", "{volume:.*}", "file", "{file:.*}")
 
 		// StorageInfo operations
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(httpTraceAll(adminAPI.StorageInfoHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.StorageInfoHandler)))
 		// DataUsageInfo operations
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(httpTraceAll(adminAPI.DataUsageInfoHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(gz(httpTraceAll(adminAPI.DataUsageInfoHandler)))
 
 		if globalIsDistErasure || globalIsErasure {
 			/// Heal operations
 
 			// Heal processing endpoint.
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(httpTraceAll(adminAPI.HealHandler))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(gz(httpTraceAll(adminAPI.HealHandler)))
 
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(httpTraceAll(adminAPI.BackgroundHealStatusHandler))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(gz(httpTraceAll(adminAPI.BackgroundHealStatusHandler)))
 
 			/// Health operations
 
 		}
 
 		// Profiling operations
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(httpTraceAll(adminAPI.StartProfilingHandler)).
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(gz(httpTraceAll(adminAPI.StartProfilingHandler))).
 			Queries("profilerType", "{profilerType:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(httpTraceAll(adminAPI.DownloadProfilingHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(gz(httpTraceAll(adminAPI.DownloadProfilingHandler)))
 
 		// Config KV operations.
 		if enableConfigOps {
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigKVHandler))
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(httpTraceHdrs(adminAPI.DelConfigKVHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetConfigKVHandler))).Queries("key", "{key:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetConfigKVHandler)))
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.DelConfigKVHandler)))
 		}
 
 		// Enable config help in all modes.
-		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(httpTraceAll(adminAPI.HelpConfigKVHandler)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(gz(httpTraceAll(adminAPI.HelpConfigKVHandler))).Queries("subSys", "{subSys:.*}", "key", "{key:.*}")
 
 		// Config KV history operations.
 		if enableConfigOps {
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(httpTraceAll(adminAPI.ListConfigHistoryKVHandler)).Queries("count", "{count:[0-9]+}")
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}")
+			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(gz(httpTraceAll(adminAPI.ListConfigHistoryKVHandler))).Queries("count", "{count:[0-9]+}")
+			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.ClearConfigHistoryKVHandler))).Queries("restoreId", "{restoreId:.*}")
+			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(gz(httpTraceHdrs(adminAPI.RestoreConfigHistoryKVHandler))).Queries("restoreId", "{restoreId:.*}")
 		}
 
 		/// Config import/export bulk operations
 		if enableConfigOps {
 			// Get config
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(httpTraceHdrs(adminAPI.GetConfigHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetConfigHandler)))
 			// Set config
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(httpTraceHdrs(adminAPI.SetConfigHandler))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetConfigHandler)))
 		}
 
-		if enableIAMOps {
-			// -- IAM APIs --
+		// -- IAM APIs --
 
-			// Add policy IAM
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(httpTraceAll(adminAPI.AddCannedPolicy)).Queries("name", "{name:.*}")
+		// Add policy IAM
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(gz(httpTraceAll(adminAPI.AddCannedPolicy))).Queries("name", "{name:.*}")
 
-			// Add user IAM
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(httpTraceAll(adminAPI.AccountInfoHandler))
+		// Add user IAM
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(gz(httpTraceAll(adminAPI.AccountInfoHandler)))
 
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(httpTraceHdrs(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddUser))).Queries("accessKey", "{accessKey:.*}")
 
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(httpTraceHdrs(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetUserStatus))).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}")
 
-			// Service accounts ops
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(httpTraceHdrs(adminAPI.AddServiceAccount))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(httpTraceHdrs(adminAPI.UpdateServiceAccount)).Queries("accessKey", "{accessKey:.*}")
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(httpTraceHdrs(adminAPI.InfoServiceAccount)).Queries("accessKey", "{accessKey:.*}")
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(httpTraceHdrs(adminAPI.ListServiceAccounts))
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(httpTraceHdrs(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}")
+		// Service accounts ops
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddServiceAccount)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateServiceAccount))).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.InfoServiceAccount))).Queries("accessKey", "{accessKey:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListServiceAccounts)))
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(gz(httpTraceHdrs(adminAPI.DeleteServiceAccount))).Queries("accessKey", "{accessKey:.*}")
 
-			// Info policy IAM latest
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}")
-			// List policies latest
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListBucketPolicies)).Queries("bucket", "{bucket:.*}")
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
+		// Info policy IAM latest
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.InfoCannedPolicy))).Queries("name", "{name:.*}")
+		// List policies latest
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListBucketPolicies))).Queries("bucket", "{bucket:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListCannedPolicies)))
 
-			// Remove policy IAM
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(httpTraceHdrs(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}")
+		// Remove policy IAM
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveCannedPolicy))).Queries("name", "{name:.*}")
 
-			// Set user or group policy
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
-				HandlerFunc(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup)).
-				Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
+		// Set user or group policy
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy").
+			HandlerFunc(gz(httpTraceHdrs(adminAPI.SetPolicyForUserOrGroup))).
+			Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}")
 
-			// Remove user IAM
-			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(httpTraceHdrs(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}")
+		// Remove user IAM
+		adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(gz(httpTraceHdrs(adminAPI.RemoveUser))).Queries("accessKey", "{accessKey:.*}")
 
-			// List users
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListBucketUsers)).Queries("bucket", "{bucket:.*}")
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(httpTraceHdrs(adminAPI.ListUsers))
+		// List users
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListBucketUsers))).Queries("bucket", "{bucket:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListUsers)))
 
-			// User info
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(httpTraceHdrs(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}")
-			// Add/Remove members from group
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(httpTraceHdrs(adminAPI.UpdateGroupMembers))
+		// User info
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetUserInfo))).Queries("accessKey", "{accessKey:.*}")
+		// Add/Remove members from group
+		adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(gz(httpTraceHdrs(adminAPI.UpdateGroupMembers)))
 
-			// Get Group
-			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(httpTraceHdrs(adminAPI.GetGroup)).Queries("group", "{group:.*}")
+		// Get Group
+		adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(gz(httpTraceHdrs(adminAPI.GetGroup))).Queries("group", "{group:.*}")
 
-			// List Groups
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(httpTraceHdrs(adminAPI.ListGroups))
+		// List Groups
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListGroups)))
 
-			// Set Group Status
-			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(httpTraceHdrs(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
-		}
+		// Set Group Status
+		adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(gz(httpTraceHdrs(adminAPI.SetGroupStatus))).Queries("group", "{group:.*}").Queries("status", "{status:.*}")
 
 		if globalIsDistErasure || globalIsErasure {
 			// GetBucketQuotaConfig
 			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc(
-				httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+				gz(httpTraceHdrs(adminAPI.GetBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
 			// PutBucketQuotaConfig
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc(
-				httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}")
+				gz(httpTraceHdrs(adminAPI.PutBucketQuotaConfigHandler))).Queries("bucket", "{bucket:.*}")
 
 			// Bucket replication operations
 			// GetBucketTargetHandler
 			adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc(
-				httpTraceHdrs(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
+				gz(httpTraceHdrs(adminAPI.ListRemoteTargetsHandler))).Queries("bucket", "{bucket:.*}", "type", "{type:.*}")
 			// SetRemoteTargetHandler
 			adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}")
+				gz(httpTraceHdrs(adminAPI.SetRemoteTargetHandler))).Queries("bucket", "{bucket:.*}")
 			// RemoveRemoteTargetHandler
 			adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc(
-				httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
+				gz(httpTraceHdrs(adminAPI.RemoveRemoteTargetHandler))).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}")
 
 			// Remote Tier management operations
-			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(httpTraceHdrs(adminAPI.AddTierHandler))
-			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(httpTraceHdrs(adminAPI.EditTierHandler))
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(httpTraceHdrs(adminAPI.ListTierHandler))
+			adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.AddTierHandler)))
+			adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(gz(httpTraceHdrs(adminAPI.EditTierHandler)))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(gz(httpTraceHdrs(adminAPI.ListTierHandler)))
 		}
 
 		if globalIsDistErasure {
 			// Top locks
-			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(gz(httpTraceHdrs(adminAPI.TopLocksHandler)))
 			// Force unlocks paths
 			adminRouter.Methods(http.MethodPost).Path(adminVersion+"/force-unlock").
-				Queries("paths", "{paths:.*}").HandlerFunc(httpTraceHdrs(adminAPI.ForceUnlockHandler))
+				Queries("paths", "{paths:.*}").HandlerFunc(gz(httpTraceHdrs(adminAPI.ForceUnlockHandler)))
 		}
 
 		// HTTP Trace
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminAPI.TraceHandler)
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(gz(adminAPI.TraceHandler))
 
 		// Console Logs
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(httpTraceAll(adminAPI.ConsoleLogHandler))
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(gz(httpTraceAll(adminAPI.ConsoleLogHandler)))
 
 		// -- KMS APIs --
 		//
-		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/kms/status").HandlerFunc(httpTraceAll(adminAPI.KMSStatusHandler))
-		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(httpTraceAll(adminAPI.KMSCreateKeyHandler)).Queries("key-id", "{key-id:.*}")
-		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(httpTraceAll(adminAPI.KMSKeyStatusHandler))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion + "/kms/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSStatusHandler)))
+		adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(gz(httpTraceAll(adminAPI.KMSCreateKeyHandler))).Queries("key-id", "{key-id:.*}")
+		adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(gz(httpTraceAll(adminAPI.KMSKeyStatusHandler)))
 
 		if !globalIsGateway {
 			// Keep obdinfo for backward compatibility with mc
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo").
-				HandlerFunc(httpTraceHdrs(adminAPI.HealthInfoHandler))
+				HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
 			// -- Health API --
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo").
-				HandlerFunc(httpTraceHdrs(adminAPI.HealthInfoHandler))
+				HandlerFunc(gz(httpTraceHdrs(adminAPI.HealthInfoHandler)))
 			adminRouter.Methods(http.MethodGet).Path(adminVersion + "/bandwidth").
-				HandlerFunc(httpTraceHdrs(adminAPI.BandwidthMonitorHandler))
+				HandlerFunc(gz(httpTraceHdrs(adminAPI.BandwidthMonitorHandler)))
 		}
 	}
 

cmd/api-router.go

@@ -18,10 +18,12 @@
 package cmd
 
 import (
+	"compress/gzip"
 	"net"
 	"net/http"
 
 	"github.com/gorilla/mux"
+	"github.com/klauspost/compress/gzhttp"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/pkg/wildcard"
 	"github.com/rs/cors"
@@ -206,6 +208,17 @@ func registerAPIRouter(router *mux.Router) {
 	}
 	routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter())
 
+	gz := func(h http.HandlerFunc) http.HandlerFunc {
+		return h
+	}
+
+	wrapper, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed))
+	if err == nil {
+		gz = func(h http.HandlerFunc) http.HandlerFunc {
+			return wrapper(h).(http.HandlerFunc)
+		}
+	}
+
 	for _, router := range routers {
 		// Register all rejected object APIs
 		for _, r := range rejectedObjAPIs {
@@ -218,218 +231,218 @@ func registerAPIRouter(router *mux.Router) {
 		// Object operations
 		// HeadObject
 		router.Methods(http.MethodHead).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("headobject", maxClients(httpTraceAll(api.HeadObjectHandler))))
+			collectAPIStats("headobject", maxClients(gz(httpTraceAll(api.HeadObjectHandler)))))
 		// CopyObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").
 			HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").
-			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(httpTraceAll(api.CopyObjectPartHandler)))).
+			HandlerFunc(collectAPIStats("copyobjectpart", maxClients(gz(httpTraceAll(api.CopyObjectPartHandler))))).
 			Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// PutObjectPart
 		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectpart", maxClients(httpTraceHdrs(api.PutObjectPartHandler)))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
+			collectAPIStats("putobjectpart", maxClients(gz(httpTraceHdrs(api.PutObjectPartHandler))))).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}")
 		// ListObjectParts
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("listobjectparts", maxClients(httpTraceAll(api.ListObjectPartsHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("listobjectparts", maxClients(gz(httpTraceAll(api.ListObjectPartsHandler))))).Queries("uploadId", "{uploadId:.*}")
 		// CompleteMultipartUpload
 		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("completemutipartupload", maxClients(httpTraceAll(api.CompleteMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("completemutipartupload", maxClients(gz(httpTraceAll(api.CompleteMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
 		// NewMultipartUpload
 		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("newmultipartupload", maxClients(httpTraceAll(api.NewMultipartUploadHandler)))).Queries("uploads", "")
+			collectAPIStats("newmultipartupload", maxClients(gz(httpTraceAll(api.NewMultipartUploadHandler))))).Queries("uploads", "")
 		// AbortMultipartUpload
 		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("abortmultipartupload", maxClients(httpTraceAll(api.AbortMultipartUploadHandler)))).Queries("uploadId", "{uploadId:.*}")
+			collectAPIStats("abortmultipartupload", maxClients(gz(httpTraceAll(api.AbortMultipartUploadHandler))))).Queries("uploadId", "{uploadId:.*}")
 		// GetObjectACL - this is a dummy call.
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectacl", maxClients(httpTraceHdrs(api.GetObjectACLHandler)))).Queries("acl", "")
+			collectAPIStats("getobjectacl", maxClients(gz(httpTraceHdrs(api.GetObjectACLHandler))))).Queries("acl", "")
 		// PutObjectACL - this is a dummy call.
 		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectacl", maxClients(httpTraceHdrs(api.PutObjectACLHandler)))).Queries("acl", "")
+			collectAPIStats("putobjectacl", maxClients(gz(httpTraceHdrs(api.PutObjectACLHandler))))).Queries("acl", "")
 		// GetObjectTagging
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjecttagging", maxClients(httpTraceHdrs(api.GetObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("getobjecttagging", maxClients(gz(httpTraceHdrs(api.GetObjectTaggingHandler))))).Queries("tagging", "")
 		// PutObjectTagging
 		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjecttagging", maxClients(httpTraceHdrs(api.PutObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("putobjecttagging", maxClients(gz(httpTraceHdrs(api.PutObjectTaggingHandler))))).Queries("tagging", "")
 		// DeleteObjectTagging
 		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("deleteobjecttagging", maxClients(httpTraceHdrs(api.DeleteObjectTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("deleteobjecttagging", maxClients(gz(httpTraceHdrs(api.DeleteObjectTaggingHandler))))).Queries("tagging", "")
 		// SelectObjectContent
 		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("selectobjectcontent", maxClients(httpTraceHdrs(api.SelectObjectContentHandler)))).Queries("select", "").Queries("select-type", "2")
+			collectAPIStats("selectobjectcontent", maxClients(gz(httpTraceHdrs(api.SelectObjectContentHandler))))).Queries("select", "").Queries("select-type", "2")
 		// GetObjectRetention
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectretention", maxClients(httpTraceAll(api.GetObjectRetentionHandler)))).Queries("retention", "")
+			collectAPIStats("getobjectretention", maxClients(gz(httpTraceAll(api.GetObjectRetentionHandler))))).Queries("retention", "")
 		// GetObjectLegalHold
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("getobjectlegalhold", maxClients(httpTraceAll(api.GetObjectLegalHoldHandler)))).Queries("legal-hold", "")
-		// GetObject
+			collectAPIStats("getobjectlegalhold", maxClients(gz(httpTraceAll(api.GetObjectLegalHoldHandler))))).Queries("legal-hold", "")
+		// GetObject - note gzip compression is *not* added due to Range requests.
 		router.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(
 			collectAPIStats("getobject", maxClients(httpTraceHdrs(api.GetObjectHandler))))
 		// CopyObject
 		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?").HandlerFunc(
-			collectAPIStats("copyobject", maxClients(httpTraceAll(api.CopyObjectHandler))))
+			collectAPIStats("copyobject", maxClients(gz(httpTraceAll(api.CopyObjectHandler)))))
 		// PutObjectRetention
 		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectretention", maxClients(httpTraceAll(api.PutObjectRetentionHandler)))).Queries("retention", "")
+			collectAPIStats("putobjectretention", maxClients(gz(httpTraceAll(api.PutObjectRetentionHandler))))).Queries("retention", "")
 		// PutObjectLegalHold
 		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobjectlegalhold", maxClients(httpTraceAll(api.PutObjectLegalHoldHandler)))).Queries("legal-hold", "")
+			collectAPIStats("putobjectlegalhold", maxClients(gz(httpTraceAll(api.PutObjectLegalHoldHandler))))).Queries("legal-hold", "")
 
 		// PutObject with auto-extract support for zip
 		router.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp(xhttp.AmzSnowballExtract, "true").HandlerFunc(
-			collectAPIStats("putobject", maxClients(httpTraceHdrs(api.PutObjectExtractHandler))))
+			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectExtractHandler)))))
 
 		// PutObject
 		router.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("putobject", maxClients(httpTraceHdrs(api.PutObjectHandler))))
+			collectAPIStats("putobject", maxClients(gz(httpTraceHdrs(api.PutObjectHandler)))))
 
 		// DeleteObject
 		router.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("deleteobject", maxClients(httpTraceAll(api.DeleteObjectHandler))))
+			collectAPIStats("deleteobject", maxClients(gz(httpTraceAll(api.DeleteObjectHandler)))))
 
 		// PostRestoreObject
 		router.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(
-			collectAPIStats("restoreobject", maxClients(httpTraceAll(api.PostRestoreObjectHandler)))).Queries("restore", "")
+			collectAPIStats("restoreobject", maxClients(gz(httpTraceAll(api.PostRestoreObjectHandler))))).Queries("restore", "")
 
 		/// Bucket operations
 		// GetBucketLocation
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlocation", maxClients(httpTraceAll(api.GetBucketLocationHandler)))).Queries("location", "")
+			collectAPIStats("getbucketlocation", maxClients(gz(httpTraceAll(api.GetBucketLocationHandler))))).Queries("location", "")
 		// GetBucketPolicy
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketpolicy", maxClients(httpTraceAll(api.GetBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("getbucketpolicy", maxClients(gz(httpTraceAll(api.GetBucketPolicyHandler))))).Queries("policy", "")
 		// GetBucketLifecycle
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlifecycle", maxClients(httpTraceAll(api.GetBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("getbucketlifecycle", maxClients(gz(httpTraceAll(api.GetBucketLifecycleHandler))))).Queries("lifecycle", "")
 		// GetBucketEncryption
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketencryption", maxClients(httpTraceAll(api.GetBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("getbucketencryption", maxClients(gz(httpTraceAll(api.GetBucketEncryptionHandler))))).Queries("encryption", "")
 		// GetBucketObjectLockConfig
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketobjectlockconfiguration", maxClients(httpTraceAll(api.GetBucketObjectLockConfigHandler)))).Queries("object-lock", "")
+			collectAPIStats("getbucketobjectlockconfiguration", maxClients(gz(httpTraceAll(api.GetBucketObjectLockConfigHandler))))).Queries("object-lock", "")
 		// GetBucketReplicationConfig
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketreplicationconfiguration", maxClients(httpTraceAll(api.GetBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("getbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.GetBucketReplicationConfigHandler))))).Queries("replication", "")
 		// GetBucketVersioning
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketversioning", maxClients(httpTraceAll(api.GetBucketVersioningHandler)))).Queries("versioning", "")
+			collectAPIStats("getbucketversioning", maxClients(gz(httpTraceAll(api.GetBucketVersioningHandler))))).Queries("versioning", "")
 		// GetBucketNotification
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketnotification", maxClients(httpTraceAll(api.GetBucketNotificationHandler)))).Queries("notification", "")
+			collectAPIStats("getbucketnotification", maxClients(gz(httpTraceAll(api.GetBucketNotificationHandler))))).Queries("notification", "")
 		// ListenNotification
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
+			collectAPIStats("listennotification", maxClients(gz(httpTraceAll(api.ListenNotificationHandler))))).Queries("events", "{events:.*}")
 
 		// Dummy Bucket Calls
 		// GetBucketACL -- this is a dummy call.
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketacl", maxClients(httpTraceAll(api.GetBucketACLHandler)))).Queries("acl", "")
+			collectAPIStats("getbucketacl", maxClients(gz(httpTraceAll(api.GetBucketACLHandler))))).Queries("acl", "")
 		// PutBucketACL -- this is a dummy call.
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketacl", maxClients(httpTraceAll(api.PutBucketACLHandler)))).Queries("acl", "")
+			collectAPIStats("putbucketacl", maxClients(gz(httpTraceAll(api.PutBucketACLHandler))))).Queries("acl", "")
 		// GetBucketCors - this is a dummy call.
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketcors", maxClients(httpTraceAll(api.GetBucketCorsHandler)))).Queries("cors", "")
+			collectAPIStats("getbucketcors", maxClients(gz(httpTraceAll(api.GetBucketCorsHandler))))).Queries("cors", "")
 		// GetBucketWebsiteHandler - this is a dummy call.
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketwebsite", maxClients(httpTraceAll(api.GetBucketWebsiteHandler)))).Queries("website", "")
+			collectAPIStats("getbucketwebsite", maxClients(gz(httpTraceAll(api.GetBucketWebsiteHandler))))).Queries("website", "")
 		// GetBucketAccelerateHandler - this is a dummy call.
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketaccelerate", maxClients(httpTraceAll(api.GetBucketAccelerateHandler)))).Queries("accelerate", "")
+			collectAPIStats("getbucketaccelerate", maxClients(gz(httpTraceAll(api.GetBucketAccelerateHandler))))).Queries("accelerate", "")
 		// GetBucketRequestPaymentHandler - this is a dummy call.
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketrequestpayment", maxClients(httpTraceAll(api.GetBucketRequestPaymentHandler)))).Queries("requestPayment", "")
+			collectAPIStats("getbucketrequestpayment", maxClients(gz(httpTraceAll(api.GetBucketRequestPaymentHandler))))).Queries("requestPayment", "")
 		// GetBucketLoggingHandler - this is a dummy call.
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketlogging", maxClients(httpTraceAll(api.GetBucketLoggingHandler)))).Queries("logging", "")
+			collectAPIStats("getbucketlogging", maxClients(gz(httpTraceAll(api.GetBucketLoggingHandler))))).Queries("logging", "")
 		// GetBucketTaggingHandler
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbuckettagging", maxClients(httpTraceAll(api.GetBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("getbuckettagging", maxClients(gz(httpTraceAll(api.GetBucketTaggingHandler))))).Queries("tagging", "")
 		//DeleteBucketWebsiteHandler
 		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketwebsite", maxClients(httpTraceAll(api.DeleteBucketWebsiteHandler)))).Queries("website", "")
+			collectAPIStats("deletebucketwebsite", maxClients(gz(httpTraceAll(api.DeleteBucketWebsiteHandler))))).Queries("website", "")
 		// DeleteBucketTaggingHandler
 		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebuckettagging", maxClients(httpTraceAll(api.DeleteBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("deletebuckettagging", maxClients(gz(httpTraceAll(api.DeleteBucketTaggingHandler))))).Queries("tagging", "")
 
 		// ListMultipartUploads
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listmultipartuploads", maxClients(httpTraceAll(api.ListMultipartUploadsHandler)))).Queries("uploads", "")
+			collectAPIStats("listmultipartuploads", maxClients(gz(httpTraceAll(api.ListMultipartUploadsHandler))))).Queries("uploads", "")
 		// ListObjectsV2M
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv2M", maxClients(httpTraceAll(api.ListObjectsV2MHandler)))).Queries("list-type", "2", "metadata", "true")
+			collectAPIStats("listobjectsv2M", maxClients(gz(httpTraceAll(api.ListObjectsV2MHandler))))).Queries("list-type", "2", "metadata", "true")
 		// ListObjectsV2
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv2", maxClients(httpTraceAll(api.ListObjectsV2Handler)))).Queries("list-type", "2")
+			collectAPIStats("listobjectsv2", maxClients(gz(httpTraceAll(api.ListObjectsV2Handler))))).Queries("list-type", "2")
 		// ListObjectVersions
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectversions", maxClients(httpTraceAll(api.ListObjectVersionsHandler)))).Queries("versions", "")
+			collectAPIStats("listobjectversions", maxClients(gz(httpTraceAll(api.ListObjectVersionsHandler))))).Queries("versions", "")
 		// GetBucketPolicyStatus
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getpolicystatus", maxClients(httpTraceAll(api.GetBucketPolicyStatusHandler)))).Queries("policyStatus", "")
+			collectAPIStats("getpolicystatus", maxClients(gz(httpTraceAll(api.GetBucketPolicyStatusHandler))))).Queries("policyStatus", "")
 		// PutBucketLifecycle
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketlifecycle", maxClients(httpTraceAll(api.PutBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("putbucketlifecycle", maxClients(gz(httpTraceAll(api.PutBucketLifecycleHandler))))).Queries("lifecycle", "")
 		// PutBucketReplicationConfig
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketreplicationconfiguration", maxClients(httpTraceAll(api.PutBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("putbucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.PutBucketReplicationConfigHandler))))).Queries("replication", "")
 		// PutBucketEncryption
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketencryption", maxClients(httpTraceAll(api.PutBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("putbucketencryption", maxClients(gz(httpTraceAll(api.PutBucketEncryptionHandler))))).Queries("encryption", "")
 
 		// PutBucketPolicy
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketpolicy", maxClients(httpTraceAll(api.PutBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("putbucketpolicy", maxClients(gz(httpTraceAll(api.PutBucketPolicyHandler))))).Queries("policy", "")
 
 		// PutBucketObjectLockConfig
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketobjectlockconfig", maxClients(httpTraceAll(api.PutBucketObjectLockConfigHandler)))).Queries("object-lock", "")
+			collectAPIStats("putbucketobjectlockconfig", maxClients(gz(httpTraceAll(api.PutBucketObjectLockConfigHandler))))).Queries("object-lock", "")
 		// PutBucketTaggingHandler
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbuckettagging", maxClients(httpTraceAll(api.PutBucketTaggingHandler)))).Queries("tagging", "")
+			collectAPIStats("putbuckettagging", maxClients(gz(httpTraceAll(api.PutBucketTaggingHandler))))).Queries("tagging", "")
 		// PutBucketVersioning
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketversioning", maxClients(httpTraceAll(api.PutBucketVersioningHandler)))).Queries("versioning", "")
+			collectAPIStats("putbucketversioning", maxClients(gz(httpTraceAll(api.PutBucketVersioningHandler))))).Queries("versioning", "")
 		// PutBucketNotification
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucketnotification", maxClients(httpTraceAll(api.PutBucketNotificationHandler)))).Queries("notification", "")
+			collectAPIStats("putbucketnotification", maxClients(gz(httpTraceAll(api.PutBucketNotificationHandler))))).Queries("notification", "")
 		// ResetBucketReplicationState - MinIO extension API
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("resetbucketreplicationstate", maxClients(httpTraceAll(api.ResetBucketReplicationStateHandler)))).Queries("replication-reset", "")
+			collectAPIStats("resetbucketreplicationstate", maxClients(gz(httpTraceAll(api.ResetBucketReplicationStateHandler))))).Queries("replication-reset", "")
 		// PutBucket
 		router.Methods(http.MethodPut).HandlerFunc(
-			collectAPIStats("putbucket", maxClients(httpTraceAll(api.PutBucketHandler))))
+			collectAPIStats("putbucket", maxClients(gz(httpTraceAll(api.PutBucketHandler)))))
 		// HeadBucket
 		router.Methods(http.MethodHead).HandlerFunc(
-			collectAPIStats("headbucket", maxClients(httpTraceAll(api.HeadBucketHandler))))
+			collectAPIStats("headbucket", maxClients(gz(httpTraceAll(api.HeadBucketHandler)))))
 		// PostPolicy
 		router.Methods(http.MethodPost).HeadersRegexp(xhttp.ContentType, "multipart/form-data*").HandlerFunc(
-			collectAPIStats("postpolicybucket", maxClients(httpTraceHdrs(api.PostPolicyBucketHandler))))
+			collectAPIStats("postpolicybucket", maxClients(gz(httpTraceHdrs(api.PostPolicyBucketHandler)))))
 		// DeleteMultipleObjects
 		router.Methods(http.MethodPost).HandlerFunc(
-			collectAPIStats("deletemultipleobjects", maxClients(httpTraceAll(api.DeleteMultipleObjectsHandler)))).Queries("delete", "")
+			collectAPIStats("deletemultipleobjects", maxClients(gz(httpTraceAll(api.DeleteMultipleObjectsHandler))))).Queries("delete", "")
 		// DeleteBucketPolicy
 		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketpolicy", maxClients(httpTraceAll(api.DeleteBucketPolicyHandler)))).Queries("policy", "")
+			collectAPIStats("deletebucketpolicy", maxClients(gz(httpTraceAll(api.DeleteBucketPolicyHandler))))).Queries("policy", "")
 		// DeleteBucketReplication
 		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketreplicationconfiguration", maxClients(httpTraceAll(api.DeleteBucketReplicationConfigHandler)))).Queries("replication", "")
+			collectAPIStats("deletebucketreplicationconfiguration", maxClients(gz(httpTraceAll(api.DeleteBucketReplicationConfigHandler))))).Queries("replication", "")
 		// DeleteBucketLifecycle
 		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketlifecycle", maxClients(httpTraceAll(api.DeleteBucketLifecycleHandler)))).Queries("lifecycle", "")
+			collectAPIStats("deletebucketlifecycle", maxClients(gz(httpTraceAll(api.DeleteBucketLifecycleHandler))))).Queries("lifecycle", "")
 		// DeleteBucketEncryption
 		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucketencryption", maxClients(httpTraceAll(api.DeleteBucketEncryptionHandler)))).Queries("encryption", "")
+			collectAPIStats("deletebucketencryption", maxClients(gz(httpTraceAll(api.DeleteBucketEncryptionHandler))))).Queries("encryption", "")
 		// DeleteBucket
 		router.Methods(http.MethodDelete).HandlerFunc(
-			collectAPIStats("deletebucket", maxClients(httpTraceAll(api.DeleteBucketHandler))))
+			collectAPIStats("deletebucket", maxClients(gz(httpTraceAll(api.DeleteBucketHandler)))))
 
 		// MinIO extension API for replication.
 		//
 		// GetBucketReplicationMetrics
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("getbucketreplicationmetrics", maxClients(httpTraceAll(api.GetBucketReplicationMetricsHandler)))).Queries("replication-metrics", "")
+			collectAPIStats("getbucketreplicationmetrics", maxClients(gz(httpTraceAll(api.GetBucketReplicationMetricsHandler))))).Queries("replication-metrics", "")
 
 		// Register rejected bucket APIs
 		for _, r := range rejectedBucketAPIs {
@@ -440,23 +453,23 @@ func registerAPIRouter(router *mux.Router) {
 
 		// S3 ListObjectsV1 (Legacy)
 		router.Methods(http.MethodGet).HandlerFunc(
-			collectAPIStats("listobjectsv1", maxClients(httpTraceAll(api.ListObjectsV1Handler))))
+			collectAPIStats("listobjectsv1", maxClients(gz(httpTraceAll(api.ListObjectsV1Handler)))))
 	}
 
 	/// Root operation
 
 	// ListenNotification
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listennotification", maxClients(httpTraceAll(api.ListenNotificationHandler)))).Queries("events", "{events:.*}")
+		collectAPIStats("listennotification", maxClients(gz(httpTraceAll(api.ListenNotificationHandler))))).Queries("events", "{events:.*}")
 
 	// ListBuckets
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator).HandlerFunc(
-		collectAPIStats("listbuckets", maxClients(httpTraceAll(api.ListBucketsHandler))))
+		collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler)))))
 
 	// S3 browser with signature v4 adds '//' for ListBuckets request, so rather
 	// than failing with UnknownAPIRequest we simply handle it for now.
 	apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator).HandlerFunc(
-		collectAPIStats("listbuckets", maxClients(httpTraceAll(api.ListBucketsHandler))))
+		collectAPIStats("listbuckets", maxClients(gz(httpTraceAll(api.ListBucketsHandler)))))
 
 	// If none of the routes match add default error handler routes
 	apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler))

cmd/bucket-policy.go

@@ -174,7 +174,7 @@ func getConditionValues(r *http.Request, lc string, username string, claims map[
 			// Special case for AD/LDAP STS users
 			if k == ldapUser {
 				args["user"] = []string{vStr}
-			} else if k == ldapUsername {
+			} else if k == ldapUserN {
 				args["username"] = []string{vStr}
 			} else {
 				args[k] = []string{vStr}

cmd/bucket-replication.go

@@ -839,7 +839,7 @@ func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI Obje
 		for k, v := range objInfo.UserDefined {
 			popts.UserDefined[k] = v
 		}
-		popts.UserDefined[xhttp.AmzBucketReplicationStatus] = replication.Completed.String()
+		popts.UserDefined[xhttp.AmzBucketReplicationStatus] = replicationStatus.String()
 		if objInfo.UserTags != "" {
 			popts.UserDefined[xhttp.AmzObjectTagging] = objInfo.UserTags
 		}

cmd/common-main.go

@@ -415,6 +415,23 @@ func handleCommonEnvVars() {
 	if err != nil {
 		logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable")
 	}
+	if globalBrowserEnabled {
+		if redirectURL := env.Get(config.EnvMinIOBrowserRedirectURL, ""); redirectURL != "" {
+			u, err := xnet.ParseHTTPURL(redirectURL)
+			if err != nil {
+				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value in environment variable")
+			}
+			// Look for if URL has invalid values and return error.
+			if !((u.Scheme == "http" || u.Scheme == "https") &&
+				(u.Path == "/" || u.Path == "") && u.Opaque == "" &&
+				!u.ForceQuery && u.RawQuery == "" && u.Fragment == "") {
+				err := fmt.Errorf("URL contains unexpected resources, expected URL to be of http(s)://minio.example.com format: %v", u)
+				logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value is environment variable")
+			}
+			u.Path = "" // remove any path component such as `/`
+			globalBrowserRedirectURL = u
+		}
+	}
 
 	globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff))
 	if err != nil {

cmd/config-current.go

@@ -43,6 +43,7 @@ import (
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/logger/target/http"
+	"github.com/minio/minio/internal/logger/target/kafka"
 	"github.com/minio/pkg/env"
 )
 
@@ -58,7 +59,8 @@ func initHelp() {
 		config.APISubSys:            api.DefaultKVS,
 		config.CredentialsSubSys:    config.DefaultCredentialKVS,
 		config.LoggerWebhookSubSys:  logger.DefaultKVS,
-		config.AuditWebhookSubSys:   logger.DefaultAuditKVS,
+		config.AuditWebhookSubSys:   logger.DefaultAuditWebhookKVS,
+		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
 		config.HealSubSys:           heal.DefaultKVS,
 		config.ScannerSubSys:        scanner.DefaultKVS,
 	}
@@ -122,6 +124,11 @@ func initHelp() {
 			Description:     "send audit logs to webhook endpoints",
 			MultipleTargets: true,
 		},
+		config.HelpKV{
+			Key:             config.AuditKafkaSubSys,
+			Description:     "send audit logs to kafka endpoints",
+			MultipleTargets: true,
+		},
 		config.HelpKV{
 			Key:             config.NotifyWebhookSubSys,
 			Description:     "publish bucket notifications to webhook endpoints",
@@ -197,7 +204,8 @@ func initHelp() {
 		config.IdentityLDAPSubSys:   xldap.Help,
 		config.PolicyOPASubSys:      opa.Help,
 		config.LoggerWebhookSubSys:  logger.Help,
-		config.AuditWebhookSubSys:   logger.HelpAudit,
+		config.AuditWebhookSubSys:   logger.HelpWebhook,
+		config.AuditKafkaSubSys:     logger.HelpKafka,
 		config.NotifyAMQPSubSys:     notify.HelpAMQP,
 		config.NotifyKafkaSubSys:    notify.HelpKafka,
 		config.NotifyMQTTSubSys:     notify.HelpMQTT,
@@ -478,42 +486,40 @@ func lookupConfigs(s config.Config, setDriveCounts []int) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize logger: %w", err))
 	}
 
-	for k, l := range loggerCfg.HTTP {
+	for _, l := range loggerCfg.HTTP {
 		if l.Enabled {
+			l.LogOnce = logger.LogOnceIf
+			l.UserAgent = loggerUserAgent
+			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http logging
-			if err = logger.AddTarget(
-				http.New(
-					http.WithTargetName(k),
-					http.WithEndpoint(l.Endpoint),
-					http.WithAuthToken(l.AuthToken),
-					http.WithUserAgent(loggerUserAgent),
-					http.WithLogKind(string(logger.All)),
-					http.WithTransport(NewGatewayHTTPTransport()),
-				),
-			); err != nil {
+			if err = logger.AddTarget(http.New(l)); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Unable to initialize console HTTP target: %w", err))
 			}
 		}
 	}
 
-	for k, l := range loggerCfg.Audit {
+	for _, l := range loggerCfg.AuditWebhook {
 		if l.Enabled {
+			l.LogOnce = logger.LogOnceIf
+			l.UserAgent = loggerUserAgent
+			l.Transport = NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)
 			// Enable http audit logging
-			if err = logger.AddAuditTarget(
-				http.New(
-					http.WithTargetName(k),
-					http.WithEndpoint(l.Endpoint),
-					http.WithAuthToken(l.AuthToken),
-					http.WithUserAgent(loggerUserAgent),
-					http.WithLogKind(string(logger.All)),
-					http.WithTransport(NewGatewayHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey)),
-				),
-			); err != nil {
+			if err = logger.AddAuditTarget(http.New(l)); err != nil {
 				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit HTTP target: %w", err))
 			}
 		}
 	}
 
+	for _, l := range loggerCfg.AuditKafka {
+		if l.Enabled {
+			l.LogOnce = logger.LogOnceIf
+			// Enable Kafka audit logging
+			if err = logger.AddAuditTarget(kafka.New(l)); err != nil {
+				logger.LogIf(ctx, fmt.Errorf("Unable to initialize audit Kafka target: %w", err))
+			}
+		}
+	}
+
 	globalConfigTargetList, err = notify.GetNotificationTargets(GlobalContext, s, NewGatewayHTTPTransport(), false)
 	if err != nil {
 		logger.LogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err))
@@ -717,8 +723,8 @@ func loadConfig(objAPI ObjectLayer) error {
 func getOpenIDValidators(cfg openid.Config) *openid.Validators {
 	validators := openid.NewValidators()
 
-	if cfg.JWKS.URL != nil {
-		validators.Add(openid.NewJWT(cfg))
+	if cfg.Enabled {
+		validators.Add(&cfg)
 	}
 
 	return validators

cmd/config-migrate.go

@@ -41,6 +41,7 @@ import (
 	"github.com/minio/minio/internal/event/target"
 	"github.com/minio/minio/internal/kms"
 	"github.com/minio/minio/internal/logger"
+	"github.com/minio/minio/internal/logger/target/http"
 	xnet "github.com/minio/pkg/net"
 	"github.com/minio/pkg/quick"
 )
@@ -2383,8 +2384,8 @@ func migrateV26ToV27() error {
 	// Enable console logging by default to avoid breaking users
 	// current deployments
 	srvConfig.Logger.Console.Enabled = true
-	srvConfig.Logger.HTTP = make(map[string]logger.HTTP)
-	srvConfig.Logger.HTTP["1"] = logger.HTTP{}
+	srvConfig.Logger.HTTP = make(map[string]http.Config)
+	srvConfig.Logger.HTTP["1"] = http.Config{}
 
 	if err = quick.SaveConfig(srvConfig, configFile, globalEtcdClient); err != nil {
 		return fmt.Errorf("Failed to migrate config from ‘26’ to ‘27’. %w", err)
@@ -2748,7 +2749,7 @@ func migrateMinioSysConfigToKV(objAPI ObjectLayer) error {
 	for k, loggerArgs := range cfg.Logger.HTTP {
 		logger.SetLoggerHTTP(newCfg, k, loggerArgs)
 	}
-	for k, auditArgs := range cfg.Logger.Audit {
+	for k, auditArgs := range cfg.Logger.AuditWebhook {
 		logger.SetLoggerHTTPAudit(newCfg, k, auditArgs)
 	}
 

cmd/consolelogger.go

@@ -117,8 +117,8 @@ func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan interface{}, doneCh <-chan
 	sys.pubsub.Subscribe(subCh, doneCh, filter)
 }
 
-// Validate if HTTPConsoleLoggerSys is valid, always returns nil right now
-func (sys *HTTPConsoleLoggerSys) Validate() error {
+// Init if HTTPConsoleLoggerSys is valid, always returns nil right now
+func (sys *HTTPConsoleLoggerSys) Init() error {
 	return nil
 }
 

cmd/data-update-tracker_test.go

@@ -51,7 +51,7 @@ func (t *testingLogger) String() string {
 	return ""
 }
 
-func (t *testingLogger) Validate() error {
+func (t *testingLogger) Init() error {
 	return nil
 }
 

cmd/data-usage-cache.go

@@ -530,6 +530,18 @@ func (h dataUsageHash) Key() string {
 	return string(h)
 }
 
+func (d *dataUsageCache) flattenChildrens(root dataUsageEntry) (m map[string]dataUsageEntry) {
+	m = make(map[string]dataUsageEntry)
+	for id := range root.Children {
+		e := d.Cache[id]
+		if len(e.Children) > 0 {
+			e = d.flatten(e)
+		}
+		m[id] = e
+	}
+	return m
+}
+
 // flatten all children of the root into the root element and return it.
 func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry {
 	for id := range root.Children {

cmd/data-usage.go

@@ -20,7 +20,9 @@ package cmd
 import (
 	"bytes"
 	"context"
+	"errors"
 	"net/http"
+	"strings"
 
 	jsoniter "github.com/json-iterator/go"
 	"github.com/minio/madmin-go"
@@ -59,6 +61,39 @@ func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan
 	}
 }
 
+// loadPrefixUsageFromBackend returns prefix usages found in passed buckets
+//   e.g.:  /testbucket/prefix => 355601334
+func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket string) (map[string]uint64, error) {
+	z, ok := objAPI.(*erasureServerPools)
+	if !ok {
+		return nil, errors.New("prefix usage is not supported")
+	}
+
+	cache := dataUsageCache{}
+
+	m := make(map[string]uint64)
+	for _, pool := range z.serverPools {
+		for _, er := range pool.sets {
+			// Load bucket usage prefixes
+			if err := cache.load(ctx, er, bucket+slashSeparator+dataUsageCacheName); err == nil {
+				root := cache.find(bucket)
+				if root == nil {
+					// We dont have usage information for this bucket in this
+					// set, go to the next set
+					continue
+				}
+
+				for id, usageInfo := range cache.flattenChildrens(*root) {
+					prefix := strings.TrimPrefix(id, bucket+slashSeparator)
+					m[prefix] += uint64(usageInfo.Size)
+				}
+			}
+		}
+	}
+
+	return m, nil
+}
+
 func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (madmin.DataUsageInfo, error) {
 	r, err := objAPI.GetObjectNInfo(ctx, dataUsageBucket, dataUsageObjName, nil, http.Header{}, readLock, ObjectOptions{})
 	if err != nil {

cmd/erasure-object.go

@@ -40,6 +40,7 @@ import (
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/minio/internal/sync/errgroup"
 	"github.com/minio/pkg/mimedb"
+	uatomic "go.uber.org/atomic"
 )
 
 // list all errors which can be ignored in object operations.
@@ -608,19 +609,35 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 
 		// If we have offline disks upgrade the number of erasure codes for this object.
 		parityOrig := parityDrives
+
+		atomicParityDrives := uatomic.NewInt64(0)
+		// Start with current parityDrives
+		atomicParityDrives.Store(int64(parityDrives))
+
+		var wg sync.WaitGroup
 		for _, disk := range storageDisks {
-			if parityDrives >= len(storageDisks)/2 {
-				parityDrives = len(storageDisks) / 2
-				break
-			}
 			if disk == nil {
-				parityDrives++
+				atomicParityDrives.Inc()
 				continue
 			}
-			di, err := disk.DiskInfo(ctx)
-			if err != nil || di.ID == "" {
-				parityDrives++
+			if !disk.IsOnline() {
+				atomicParityDrives.Inc()
+				continue
 			}
+			wg.Add(1)
+			go func(disk StorageAPI) {
+				defer wg.Done()
+				di, err := disk.DiskInfo(ctx)
+				if err != nil || di.ID == "" {
+					atomicParityDrives.Inc()
+				}
+			}(disk)
+		}
+		wg.Wait()
+
+		parityDrives = int(atomicParityDrives.Load())
+		if parityDrives >= len(storageDisks)/2 {
+			parityDrives = len(storageDisks) / 2
 		}
 		if parityOrig != parityDrives {
 			opts.UserDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives)

cmd/erasure-server-pool.go

@@ -147,6 +147,39 @@ func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI {
 	return res
 }
 
+// GetRawData will return all files with a given raw path to the callback.
+// Errors are ignored, only errors from the callback are returned.
+// For now only direct file paths are supported.
+func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
+	for _, s := range z.serverPools {
+		for _, disks := range s.erasureDisks {
+			for i, disk := range disks {
+				if disk == OfflineDisk {
+					continue
+				}
+				si, err := disk.StatInfoFile(ctx, volume, file)
+				if err != nil {
+					continue
+				}
+				r, err := disk.ReadFileStream(ctx, volume, file, 0, si.Size)
+				if err != nil {
+					continue
+				}
+				defer r.Close()
+				did, err := disk.GetDiskID()
+				if err != nil {
+					did = fmt.Sprintf("disk-%d", i)
+				}
+				err = fn(r, disk.Hostname(), did, pathJoin(volume, file), si.Size, si.ModTime)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
 func (z *erasureServerPools) SetDriveCounts() []int {
 	setDriveCounts := make([]int, len(z.serverPools))
 	for i := range z.serverPools {

cmd/fs-v1.go

@@ -27,6 +27,7 @@ import (
 	"os"
 	"os/user"
 	"path"
+	"path/filepath"
 	"sort"
 	"strings"
 	"sync"
@@ -533,7 +534,7 @@ func (fs *FSObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 		atomic.AddInt64(&fs.activeIOCount, -1)
 	}()
 
-	entries, err := readDir(fs.fsPath)
+	entries, err := readDirWithOpts(fs.fsPath, readDirOpts{count: -1, followDirSymlink: true})
 	if err != nil {
 		logger.LogIf(ctx, errDiskNotFound)
 		return nil, toObjectErr(errDiskNotFound)
@@ -1509,3 +1510,19 @@ func (fs *FSObjects) TransitionObject(ctx context.Context, bucket, object string
 func (fs *FSObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error {
 	return NotImplemented{}
 }
+
+// GetRawData returns raw file data to the callback.
+// Errors are ignored, only errors from the callback are returned.
+// For now only direct file paths are supported.
+func (fs *FSObjects) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, size int64, modtime time.Time) error) error {
+	f, err := os.Open(filepath.Join(fs.fsPath, volume, file))
+	if err != nil {
+		return nil
+	}
+	defer f.Close()
+	st, err := f.Stat()
+	if err != nil || st.IsDir() {
+		return nil
+	}
+	return fn(f, "fs", fs.fsUUID, file, st.Size(), st.ModTime())
+}

cmd/gateway-main.go

@@ -248,16 +248,12 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// avoid URL path encoding minio/minio#8950
 	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
 
-	if globalEtcdClient != nil {
-		// Enable STS router if etcd is enabled.
-		registerSTSRouter(router)
-	}
-
-	enableIAMOps := globalEtcdClient != nil
+	// Enable STS router if etcd is enabled.
+	registerSTSRouter(router)
 
 	// Enable IAM admin APIs if etcd is enabled, if not just enable basic
 	// operations such as profiling, server info etc.
-	registerAdminRouter(router, enableConfigOps, enableIAMOps)
+	registerAdminRouter(router, enableConfigOps)
 
 	// Add healthcheck router
 	registerHealthCheckRouter(router)
@@ -315,12 +311,10 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		logger.FatalIf(globalNotificationSys.Init(GlobalContext, buckets, newObject), "Unable to initialize notification system")
 	}
 
-	if enableIAMOps {
-		// Initialize users credentials and policies in background.
-		globalIAMSys.InitStore(newObject)
+	// Initialize users credentials and policies in background.
+	globalIAMSys.InitStore(newObject)
 
-		go globalIAMSys.Init(GlobalContext, newObject)
-	}
+	go globalIAMSys.Init(GlobalContext, newObject)
 
 	if globalCacheConfig.Enabled {
 		// initialize the new disk cache objects.

cmd/generic-handlers.go

@@ -21,13 +21,13 @@ import (
 	"context"
 	"net"
 	"net/http"
-	"net/url"
 	"path"
 	"strings"
 	"sync/atomic"
 	"time"
 
 	"github.com/minio/minio-go/v7/pkg/set"
+	xnet "github.com/minio/pkg/net"
 
 	humanize "github.com/dustin/go-humanize"
 	"github.com/minio/minio/internal/config/dns"
@@ -154,15 +154,15 @@ func setRedirectHandler(h http.Handler) http.Handler {
 
 func guessIsBrowserReq(r *http.Request) bool {
 	aType := getRequestAuthType(r)
-	ok := strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && globalBrowserEnabled &&
-		(aType == authTypeJWT || aType == authTypeAnonymous)
-	return ok
+	return strings.Contains(r.Header.Get("User-Agent"), "Mozilla") &&
+		globalBrowserEnabled && aType == authTypeAnonymous
 }
 
 func setBrowserRedirectHandler(h http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		read := r.Method == http.MethodGet || r.Method == http.MethodHead
 		// Re-direction is handled specifically for browser requests.
-		if guessIsBrowserReq(r) {
+		if guessIsBrowserReq(r) && read {
 			// Fetch the redirect location if any.
 			if u := getRedirectLocation(r); u != nil {
 				// Employ a temporary re-direct.
@@ -183,7 +183,7 @@ func shouldProxy() bool {
 // redirectable, this is purely internal function and
 // serves only limited purpose on redirect-handler for
 // browser requests.
-func getRedirectLocation(r *http.Request) *url.URL {
+func getRedirectLocation(r *http.Request) *xnet.URL {
 	resource, err := getResource(r.URL.Path, r.Host, globalDomainNames)
 	if err != nil {
 		return nil
@@ -197,11 +197,14 @@ func getRedirectLocation(r *http.Request) *url.URL {
 	} {
 		bucket, _ := path2BucketObject(resource)
 		if path.Clean(bucket) == prefix || resource == slashSeparator {
+			if globalBrowserRedirectURL != nil {
+				return globalBrowserRedirectURL
+			}
 			hostname, _, _ := net.SplitHostPort(r.Host)
 			if hostname == "" {
 				hostname = r.Host
 			}
-			return &url.URL{
+			return &xnet.URL{
 				Host: net.JoinHostPort(hostname, globalMinioConsolePort),
 				Scheme: func() string {
 					scheme := "http"

cmd/globals.go

@@ -45,6 +45,7 @@ import (
 	"github.com/minio/minio/internal/event"
 	"github.com/minio/minio/internal/pubsub"
 	"github.com/minio/pkg/certs"
+	xnet "github.com/minio/pkg/net"
 )
 
 // minio configuration related constants.
@@ -137,6 +138,10 @@ var (
 	// This flag is set to 'true' by default
 	globalBrowserEnabled = true
 
+	// Custom browser redirect URL, not set by default
+	// and it is automatically deduced.
+	globalBrowserRedirectURL *xnet.URL
+
 	// This flag is set to 'true' when MINIO_UPDATE env is set to 'off'. Default is false.
 	globalInplaceUpdateDisabled = false
 

cmd/healthinfo.go

@@ -106,7 +106,7 @@ func getDrivePerfInfos(ctx context.Context, addr string) madmin.DrivePerfInfos {
 	serialPerf := getDrivePerfInfo(ctx, false)
 	parallelPerf := getDrivePerfInfo(ctx, true)
 	return madmin.DrivePerfInfos{
-		Addr:         addr,
+		NodeCommon:   madmin.NodeCommon{Addr: addr},
 		SerialPerf:   serialPerf,
 		ParallelPerf: parallelPerf,
 	}

cmd/iam-dummy-store.go

@@ -0,0 +1,133 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"context"
+	"sync"
+
+	"github.com/minio/minio/internal/auth"
+	iampolicy "github.com/minio/pkg/iam/policy"
+)
+
+type iamDummyStore struct {
+	sync.RWMutex
+}
+
+func (ids *iamDummyStore) lock() {
+	ids.Lock()
+}
+
+func (ids *iamDummyStore) unlock() {
+	ids.Unlock()
+}
+
+func (ids *iamDummyStore) rlock() {
+	ids.RLock()
+}
+
+func (ids *iamDummyStore) runlock() {
+	ids.RUnlock()
+}
+
+func (ids *iamDummyStore) migrateBackendFormat(context.Context) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]iampolicy.Policy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadPolicyDocs(ctx context.Context, m map[string]iampolicy.Policy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]auth.Credentials) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadGroups(ctx context.Context, m map[string]GroupInfo) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m map[string]MappedPolicy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadAll(ctx context.Context, sys *IAMSys) error {
+	return sys.Load(ctx, ids)
+}
+
+func (ids *iamDummyStore) saveIAMConfig(ctx context.Context, item interface{}, path string, opts ...options) error {
+	return nil
+}
+
+func (ids *iamDummyStore) loadIAMConfig(ctx context.Context, item interface{}, path string) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deleteIAMConfig(ctx context.Context, path string) error {
+	return nil
+}
+
+func (ids *iamDummyStore) savePolicyDoc(ctx context.Context, policyName string, p iampolicy.Policy) error {
+	return nil
+}
+
+func (ids *iamDummyStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error {
+	return nil
+}
+
+func (ids *iamDummyStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error {
+	return nil
+}
+
+func (ids *iamDummyStore) saveGroupInfo(ctx context.Context, group string, gi GroupInfo) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deletePolicyDoc(ctx context.Context, policyName string) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error {
+	return nil
+}
+
+func (ids *iamDummyStore) deleteGroupInfo(ctx context.Context, name string) error {
+	return nil
+}
+
+func (ids *iamDummyStore) watch(context.Context, *IAMSys) {
+}

cmd/iam-etcd-store.go

@@ -162,7 +162,7 @@ func (ies *IAMEtcdStore) migrateUsersConfigToV1(ctx context.Context) error {
 
 			// 2. copy policy to new loc.
 			mp := newMappedPolicy(policyName)
-			userType := regularUser
+			userType := regUser
 			path := getMappedPolicyPath(user, userType, false)
 			if err := ies.saveIAMConfig(ctx, mp, path); err != nil {
 				return err
@@ -335,7 +335,7 @@ func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAM
 func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
 	var basePrefix string
 	switch userType {
-	case srvAccUser:
+	case svcUser:
 		basePrefix = iamConfigServiceAccountsPrefix
 	case stsUser:
 		basePrefix = iamConfigSTSPrefix
@@ -432,7 +432,7 @@ func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUse
 		basePrefix = iamConfigPolicyDBGroupsPrefix
 	} else {
 		switch userType {
-		case srvAccUser:
+		case svcUser:
 			basePrefix = iamConfigPolicyDBServiceAccountsPrefix
 		case stsUser:
 			basePrefix = iamConfigPolicyDBSTSUsersPrefix
@@ -567,7 +567,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 		case usersPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigUsersPrefix))
-			ies.loadUser(ctx, accessKey, regularUser, sys.iamUsersMap)
+			ies.loadUser(ctx, accessKey, regUser, sys.iamUsersMap)
 		case stsPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigSTSPrefix))
@@ -593,7 +593,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 		case svcPrefix:
 			accessKey := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigServiceAccountsPrefix))
-			ies.loadUser(ctx, accessKey, srvAccUser, sys.iamUsersMap)
+			ies.loadUser(ctx, accessKey, svcUser, sys.iamUsersMap)
 		case groupsPrefix:
 			group := path.Dir(strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigGroupsPrefix))
@@ -609,7 +609,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBUsersPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(ctx, user, regularUser, false, sys.iamUserPolicyMap)
+			ies.loadMappedPolicy(ctx, user, regUser, false, sys.iamUserPolicyMap)
 		case policyDBSTSUsersPrefix:
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBSTSUsersPrefix)
@@ -619,7 +619,7 @@ func (ies *IAMEtcdStore) reloadFromEvent(sys *IAMSys, event *etcd.Event) {
 			policyMapFile := strings.TrimPrefix(string(event.Kv.Key),
 				iamConfigPolicyDBGroupsPrefix)
 			user := strings.TrimSuffix(policyMapFile, ".json")
-			ies.loadMappedPolicy(ctx, user, regularUser, true, sys.iamGroupPolicyMap)
+			ies.loadMappedPolicy(ctx, user, regUser, true, sys.iamGroupPolicyMap)
 		}
 	case eventDelete:
 		switch {

cmd/iam-object-store.go

@@ -104,7 +104,7 @@ func (iamOS *IAMObjectStore) migrateUsersConfigToV1(ctx context.Context) error {
 
 			// 2. copy policy file to new location.
 			mp := newMappedPolicy(policyName)
-			userType := regularUser
+			userType := regUser
 			if err := iamOS.saveMappedPolicy(ctx, user, userType, false, mp); err != nil {
 				return err
 			}
@@ -279,7 +279,7 @@ func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType
 func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]auth.Credentials) error {
 	var basePrefix string
 	switch userType {
-	case srvAccUser:
+	case svcUser:
 		basePrefix = iamConfigServiceAccountsPrefix
 	case stsUser:
 		basePrefix = iamConfigSTSPrefix
@@ -348,7 +348,7 @@ func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IA
 		basePath = iamConfigPolicyDBGroupsPrefix
 	} else {
 		switch userType {
-		case srvAccUser:
+		case svcUser:
 			basePath = iamConfigPolicyDBServiceAccountsPrefix
 		case stsUser:
 			basePath = iamConfigPolicyDBSTSUsersPrefix

cmd/iam.go

@@ -112,7 +112,7 @@ func getIAMFormatFilePath() string {
 func getUserIdentityPath(user string, userType IAMUserType) string {
 	var basePath string
 	switch userType {
-	case srvAccUser:
+	case svcUser:
 		basePath = iamConfigServiceAccountsPrefix
 	case stsUser:
 		basePath = iamConfigSTSPrefix
@@ -135,7 +135,7 @@ func getMappedPolicyPath(name string, userType IAMUserType, isGroup bool) string
 		return pathJoin(iamConfigPolicyDBGroupsPrefix, name+".json")
 	}
 	switch userType {
-	case srvAccUser:
+	case svcUser:
 		return pathJoin(iamConfigPolicyDBServiceAccountsPrefix, name+".json")
 	case stsUser:
 		return pathJoin(iamConfigPolicyDBSTSUsersPrefix, name+".json")
@@ -230,9 +230,9 @@ type IAMSys struct {
 type IAMUserType int
 
 const (
-	regularUser IAMUserType = iota
+	regUser IAMUserType = iota
 	stsUser
-	srvAccUser
+	svcUser
 )
 
 // key options
@@ -355,7 +355,7 @@ func (sys *IAMSys) LoadPolicyMapping(objAPI ObjectLayer, userOrGroup string, isG
 
 	if globalEtcdClient == nil {
 		var err error
-		userType := regularUser
+		userType := regUser
 		if sys.usersSysType == LDAPUsersSysType {
 			userType = stsUser
 		}
@@ -432,7 +432,7 @@ func (sys *IAMSys) LoadServiceAccount(accessKey string) error {
 	defer sys.store.unlock()
 
 	if globalEtcdClient == nil {
-		err := sys.store.loadUser(context.Background(), accessKey, srvAccUser, sys.iamUsersMap)
+		err := sys.store.loadUser(context.Background(), accessKey, svcUser, sys.iamUsersMap)
 		if err != nil {
 			return err
 		}
@@ -452,7 +452,11 @@ func (sys *IAMSys) InitStore(objAPI ObjectLayer) {
 	defer sys.Unlock()
 
 	if globalEtcdClient == nil {
-		sys.store = newIAMObjectStore(objAPI)
+		if globalIsGateway {
+			sys.store = &iamDummyStore{}
+		} else {
+			sys.store = newIAMObjectStore(objAPI)
+		}
 	} else {
 		sys.store = newIAMEtcdStore()
 	}
@@ -492,7 +496,7 @@ func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
 	setDefaultCannedPolicies(iamPolicyDocsMap)
 
 	if isMinIOUsersSys {
-		if err := store.loadUsers(ctx, regularUser, iamUsersMap); err != nil {
+		if err := store.loadUsers(ctx, regUser, iamUsersMap); err != nil {
 			return err
 		}
 		if err := store.loadGroups(ctx, iamGroupsMap); err != nil {
@@ -501,16 +505,16 @@ func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
 	}
 
 	// load polices mapped to users
-	if err := store.loadMappedPolicies(ctx, regularUser, false, iamUserPolicyMap); err != nil {
+	if err := store.loadMappedPolicies(ctx, regUser, false, iamUserPolicyMap); err != nil {
 		return err
 	}
 
 	// load policies mapped to groups
-	if err := store.loadMappedPolicies(ctx, regularUser, true, iamGroupPolicyMap); err != nil {
+	if err := store.loadMappedPolicies(ctx, regUser, true, iamGroupPolicyMap); err != nil {
 		return err
 	}
 
-	if err := store.loadUsers(ctx, srvAccUser, iamUsersMap); err != nil {
+	if err := store.loadUsers(ctx, svcUser, iamUsersMap); err != nil {
 		return err
 	}
 
@@ -545,33 +549,11 @@ func (sys *IAMSys) Load(ctx context.Context, store IAMStorageAPI) error {
 	}
 
 	// purge any expired entries which became expired now.
-	var expiredEntries []string
 	for k, v := range sys.iamUsersMap {
 		if v.IsExpired() {
 			delete(sys.iamUsersMap, k)
 			delete(sys.iamUserPolicyMap, k)
-			expiredEntries = append(expiredEntries, k)
-			// Deleting on the disk is taken care of in the next cycle
-		}
-	}
-
-	for _, v := range sys.iamUsersMap {
-		if v.IsServiceAccount() {
-			for _, accessKey := range expiredEntries {
-				if v.ParentUser == accessKey {
-					_ = store.deleteUserIdentity(ctx, v.AccessKey, srvAccUser)
-					delete(sys.iamUsersMap, v.AccessKey)
-				}
-			}
-		}
-	}
-
-	// purge any expired entries which became expired now.
-	for k, v := range sys.iamUsersMap {
-		if v.IsExpired() {
-			delete(sys.iamUsersMap, k)
-			delete(sys.iamUserPolicyMap, k)
-			// Deleting on the etcd is taken care of in the next cycle
+			// deleting will be done in the next cycle.
 		}
 	}
 
@@ -668,6 +650,16 @@ func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer) {
 		break
 	}
 
+	if globalOpenIDConfig.ProviderEnabled() {
+		go func() {
+			// Purge expired credentials
+			for {
+				time.Sleep(globalRefreshIAMInterval)
+				sys.purgeExpiredCredentialsForExternalSSO(ctx)
+			}
+		}()
+	}
+
 	go sys.store.watch(ctx, sys)
 }
 
@@ -708,7 +700,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 			if cr.IsTemp() {
 				sys.policyDBSet(u, strings.Join(pset.ToSlice(), ","), stsUser, false)
 			} else {
-				sys.policyDBSet(u, strings.Join(pset.ToSlice(), ","), regularUser, false)
+				sys.policyDBSet(u, strings.Join(pset.ToSlice(), ","), regUser, false)
 			}
 		}
 	}
@@ -718,7 +710,7 @@ func (sys *IAMSys) DeletePolicy(policyName string) error {
 		pset := mp.policySet()
 		if pset.Contains(policyName) {
 			pset.Remove(policyName)
-			sys.policyDBSet(g, strings.Join(pset.ToSlice(), ","), regularUser, true)
+			sys.policyDBSet(g, strings.Join(pset.ToSlice(), ","), regUser, true)
 		}
 	}
 
@@ -823,7 +815,7 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 		// Delete any service accounts if any first.
 		if u.IsServiceAccount() {
 			if u.ParentUser == accessKey {
-				_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, srvAccUser)
+				_ = sys.store.deleteUserIdentity(context.Background(), u.AccessKey, svcUser)
 				delete(sys.iamUsersMap, u.AccessKey)
 			}
 		}
@@ -837,8 +829,8 @@ func (sys *IAMSys) DeleteUser(accessKey string) error {
 	}
 
 	// It is ok to ignore deletion error on the mapped policy
-	sys.store.deleteMappedPolicy(context.Background(), accessKey, regularUser, false)
-	err := sys.store.deleteUserIdentity(context.Background(), accessKey, regularUser)
+	sys.store.deleteMappedPolicy(context.Background(), accessKey, regUser, false)
+	err := sys.store.deleteUserIdentity(context.Background(), accessKey, regUser)
 	if err == errNoSuchUser {
 		// ignore if user is already deleted.
 		err = nil
@@ -1153,7 +1145,7 @@ func (sys *IAMSys) SetUserStatus(accessKey string, status madmin.AccountStatus)
 		}(),
 	})
 
-	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, uinfo); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regUser, uinfo); err != nil {
 		return err
 	}
 
@@ -1240,7 +1232,7 @@ func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, gro
 
 	u := newUserIdentity(cred)
 
-	if err := sys.store.saveUserIdentity(context.Background(), u.Credentials.AccessKey, srvAccUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), u.Credentials.AccessKey, svcUser, u); err != nil {
 		return auth.Credentials{}, err
 	}
 
@@ -1310,7 +1302,7 @@ func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, o
 	}
 
 	u := newUserIdentity(cr)
-	if err := sys.store.saveUserIdentity(context.Background(), u.Credentials.AccessKey, srvAccUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), u.Credentials.AccessKey, svcUser, u); err != nil {
 		return err
 	}
 
@@ -1397,12 +1389,8 @@ func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string) e
 	}
 
 	// It is ok to ignore deletion error on the mapped policy
-	err := sys.store.deleteUserIdentity(context.Background(), accessKey, srvAccUser)
-	if err != nil {
-		// ignore if user is already deleted.
-		if err == errNoSuchUser {
-			return nil
-		}
+	err := sys.store.deleteUserIdentity(context.Background(), accessKey, svcUser)
+	if err != nil && err != errNoSuchUser {
 		return err
 	}
 
@@ -1448,7 +1436,7 @@ func (sys *IAMSys) CreateUser(accessKey string, uinfo madmin.UserInfo) error {
 		}(),
 	})
 
-	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regUser, u); err != nil {
 		return err
 	}
 
@@ -1456,7 +1444,7 @@ func (sys *IAMSys) CreateUser(accessKey string, uinfo madmin.UserInfo) error {
 
 	// Set policy if specified.
 	if uinfo.PolicyName != "" {
-		return sys.policyDBSet(accessKey, uinfo.PolicyName, regularUser, false)
+		return sys.policyDBSet(accessKey, uinfo.PolicyName, regUser, false)
 	}
 	return nil
 }
@@ -1489,7 +1477,7 @@ func (sys *IAMSys) SetUserSecretKey(accessKey string, secretKey string) error {
 
 	cred.SecretKey = secretKey
 	u := newUserIdentity(cred)
-	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regularUser, u); err != nil {
+	if err := sys.store.saveUserIdentity(context.Background(), accessKey, regUser, u); err != nil {
 		return err
 	}
 
@@ -1501,18 +1489,18 @@ func (sys *IAMSys) loadUserFromStore(accessKey string) {
 	sys.store.lock()
 	// If user is already found proceed.
 	if _, found := sys.iamUsersMap[accessKey]; !found {
-		sys.store.loadUser(context.Background(), accessKey, regularUser, sys.iamUsersMap)
+		sys.store.loadUser(context.Background(), accessKey, regUser, sys.iamUsersMap)
 		if _, found = sys.iamUsersMap[accessKey]; found {
 			// found user, load its mapped policies
-			sys.store.loadMappedPolicy(context.Background(), accessKey, regularUser, false, sys.iamUserPolicyMap)
+			sys.store.loadMappedPolicy(context.Background(), accessKey, regUser, false, sys.iamUserPolicyMap)
 		} else {
-			sys.store.loadUser(context.Background(), accessKey, srvAccUser, sys.iamUsersMap)
+			sys.store.loadUser(context.Background(), accessKey, svcUser, sys.iamUsersMap)
 			if svc, found := sys.iamUsersMap[accessKey]; found {
 				// Found service account, load its parent user and its mapped policies.
 				if sys.usersSysType == MinIOUsersSysType {
-					sys.store.loadUser(context.Background(), svc.ParentUser, regularUser, sys.iamUsersMap)
+					sys.store.loadUser(context.Background(), svc.ParentUser, regUser, sys.iamUsersMap)
 				}
-				sys.store.loadMappedPolicy(context.Background(), svc.ParentUser, regularUser, false, sys.iamUserPolicyMap)
+				sys.store.loadMappedPolicy(context.Background(), svc.ParentUser, regUser, false, sys.iamUserPolicyMap)
 			} else {
 				// None found fall back to STS users.
 				sys.store.loadUser(context.Background(), accessKey, stsUser, sys.iamUsersMap)
@@ -1535,6 +1523,54 @@ func (sys *IAMSys) loadUserFromStore(accessKey string) {
 	sys.store.unlock()
 }
 
+// purgeExpiredCredentialsForExternalSSO - validates if local credentials are still valid
+// by checking remote IDP if the relevant users are still active and present.
+func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) {
+	sys.store.lock()
+	parentUsersMap := make(map[string]auth.Credentials, len(sys.iamUsersMap))
+	for _, cred := range sys.iamUsersMap {
+		if cred.IsServiceAccount() || cred.IsTemp() {
+			userid, err := parseOpenIDParentUser(cred.ParentUser)
+			if err == errSkipFile {
+				continue
+			}
+			parentUsersMap[userid] = cred
+		}
+	}
+	sys.store.unlock()
+
+	expiredUsers := make([]auth.Credentials, 0, len(parentUsersMap))
+	for userid, cred := range parentUsersMap {
+		u, err := globalOpenIDConfig.LookupUser(userid)
+		if err != nil {
+			logger.LogIf(GlobalContext, err)
+			continue
+		}
+		// Disabled parentUser purge the entries locally
+		if !u.Enabled {
+			expiredUsers = append(expiredUsers, cred)
+		}
+	}
+
+	for _, cred := range expiredUsers {
+		userType := regUser
+		if cred.IsServiceAccount() {
+			userType = svcUser
+		} else if cred.IsTemp() {
+			userType = stsUser
+		}
+		sys.store.deleteIAMConfig(ctx, getUserIdentityPath(cred.AccessKey, userType))
+		sys.store.deleteIAMConfig(ctx, getMappedPolicyPath(cred.AccessKey, userType, false))
+	}
+
+	sys.store.lock()
+	for _, cred := range expiredUsers {
+		delete(sys.iamUsersMap, cred.AccessKey)
+		delete(sys.iamUserPolicyMap, cred.AccessKey)
+	}
+	sys.store.unlock()
+}
+
 // GetUser - get user credentials
 func (sys *IAMSys) GetUser(accessKey string) (cred auth.Credentials, ok bool) {
 	if !sys.Initialized() {
@@ -1693,7 +1729,7 @@ func (sys *IAMSys) RemoveUsersFromGroup(group string, members []string) error {
 
 		// Remove the group from storage. First delete the
 		// mapped policy. No-mapped-policy case is ignored.
-		if err := sys.store.deleteMappedPolicy(context.Background(), group, regularUser, true); err != nil && err != errNoSuchPolicy {
+		if err := sys.store.deleteMappedPolicy(context.Background(), group, regUser, true); err != nil && err != errNoSuchPolicy {
 			return err
 		}
 		if err := sys.store.deleteGroupInfo(context.Background(), group); err != nil && err != errNoSuchGroup {
@@ -1839,7 +1875,7 @@ func (sys *IAMSys) PolicyDBSet(name, policy string, isGroup bool) error {
 		return sys.policyDBSet(name, policy, stsUser, isGroup)
 	}
 
-	return sys.policyDBSet(name, policy, regularUser, isGroup)
+	return sys.policyDBSet(name, policy, regUser, isGroup)
 }
 
 // policyDBSet - sets a policy for user in the policy db. Assumes that caller
@@ -1867,7 +1903,7 @@ func (sys *IAMSys) policyDBSet(name, policyName string, userType IAMUserType, is
 			// Add a fallback removal towards previous content that may come back
 			// as a ghost user due to lack of delete, this change occurred
 			// introduced in PR #11840
-			sys.store.deleteMappedPolicy(context.Background(), name, regularUser, false)
+			sys.store.deleteMappedPolicy(context.Background(), name, regUser, false)
 		}
 		err := sys.store.deleteMappedPolicy(context.Background(), name, userType, isGroup)
 		if err != nil && err != errNoSuchPolicy {
@@ -2002,7 +2038,7 @@ func (sys *IAMSys) policyDBGet(name string, isGroup bool) (policies []string, er
 
 // IsAllowedServiceAccount - checks if the given service account is allowed to perform
 // actions. The permission of the parent user is checked first
-func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) bool {
+func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parentUser string) bool {
 	// Now check if we have a subject claim
 	p, ok := args.Claims[parentClaim]
 	if ok {
@@ -2013,7 +2049,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 		}
 		// The parent claim in the session token should be equal
 		// to the parent detected in the backend
-		if parentInClaim != parent {
+		if parentInClaim != parentUser {
 			return false
 		}
 	} else {
@@ -2024,7 +2060,7 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 	}
 
 	// Check policy for this service account.
-	svcPolicies, err := sys.PolicyDBGet(parent, false, args.Groups...)
+	svcPolicies, err := sys.PolicyDBGet(parentUser, false, args.Groups...)
 	if err != nil {
 		logger.LogIf(GlobalContext, err)
 		return false
@@ -2057,7 +2093,10 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 	}
 
 	parentArgs := args
-	parentArgs.AccountName = parent
+	parentArgs.AccountName = parentUser
+	// These are dynamic values set them appropriately.
+	parentArgs.ConditionValues["username"] = []string{parentUser}
+	parentArgs.ConditionValues["userid"] = []string{parentUser}
 
 	saPolicyClaim, ok := args.Claims[iamPolicyClaimNameSA()]
 	if !ok {
@@ -2096,7 +2135,11 @@ func (sys *IAMSys) IsAllowedServiceAccount(args iampolicy.Args, parent string) b
 		return false
 	}
 
-	// Policy without Version string value reject it.
+	// This can only happen if policy was set but with an empty JSON.
+	if subPolicy.Version == "" && len(subPolicy.Statements) == 0 {
+		return combinedPolicy.IsAllowed(parentArgs)
+	}
+
 	if subPolicy.Version == "" {
 		return false
 	}
@@ -2156,6 +2199,11 @@ func (sys *IAMSys) IsAllowedLDAPSTS(args iampolicy.Args, parentUser string) bool
 				availablePolicies[i].Statements...)
 	}
 
+	hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicy(args)
+	if hasSessionPolicy {
+		return isAllowedSP && combinedPolicy.IsAllowed(args)
+	}
+
 	return combinedPolicy.IsAllowed(args)
 }
 
@@ -2215,31 +2263,14 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 			availablePolicies[i].Statements...)
 	}
 
+	// These are dynamic values set them appropriately.
+	args.ConditionValues["username"] = []string{parentUser}
+	args.ConditionValues["userid"] = []string{parentUser}
+
 	// Now check if we have a sessionPolicy.
-	spolicy, ok := args.Claims[iampolicy.SessionPolicyName]
-	if ok {
-		spolicyStr, ok := spolicy.(string)
-		if !ok {
-			// Sub policy if set, should be a string reject
-			// malformed/malicious requests.
-			return false
-		}
-
-		// Check if policy is parseable.
-		subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
-		if err != nil {
-			// Log any error in input session policy config.
-			logger.LogIf(GlobalContext, err)
-			return false
-		}
-
-		// Policy without Version string value reject it.
-		if subPolicy.Version == "" {
-			return false
-		}
-
-		// Sub policy is set and valid.
-		return combinedPolicy.IsAllowed(args) && subPolicy.IsAllowed(args)
+	hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicy(args)
+	if hasSessionPolicy {
+		return isAllowedSP && combinedPolicy.IsAllowed(args)
 	}
 
 	// Sub policy not set, this is most common since subPolicy
@@ -2247,6 +2278,49 @@ func (sys *IAMSys) IsAllowedSTS(args iampolicy.Args, parentUser string) bool {
 	return combinedPolicy.IsAllowed(args)
 }
 
+func isAllowedBySessionPolicy(args iampolicy.Args) (hasSessionPolicy bool, isAllowed bool) {
+	hasSessionPolicy = false
+	isAllowed = false
+
+	// Now check if we have a sessionPolicy.
+	spolicy, ok := args.Claims[iampolicy.SessionPolicyName]
+	if !ok {
+		return
+	}
+
+	hasSessionPolicy = true
+
+	spolicyStr, ok := spolicy.(string)
+	if !ok {
+		// Sub policy if set, should be a string reject
+		// malformed/malicious requests.
+		return
+	}
+
+	policyBytes, err := base64.StdEncoding.DecodeString(spolicyStr)
+	if err != nil {
+		// Got a malformed base64 string
+		return
+	}
+	spolicyStr = string(policyBytes)
+
+	// Check if policy is parseable.
+	subPolicy, err := iampolicy.ParseConfig(bytes.NewReader([]byte(spolicyStr)))
+	if err != nil {
+		// Log any error in input session policy config.
+		logger.LogIf(GlobalContext, err)
+		return
+	}
+
+	// Policy without Version string value reject it.
+	if subPolicy.Version == "" {
+		return
+	}
+
+	// Sub policy is set and valid.
+	return hasSessionPolicy, subPolicy.IsAllowed(args)
+}
+
 // GetCombinedPolicy returns a combined policy combining all policies
 func (sys *IAMSys) GetCombinedPolicy(policies ...string) iampolicy.Policy {
 	// Policies were found, evaluate all of them.

cmd/metacache-entries.go

@@ -600,7 +600,9 @@ func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<
 			out <- *best
 			last = best.name
 		} else {
-			console.Debugln("mergeEntryChannels: discarding duplicate", best.name, "<=", last)
+			if serverDebugLog {
+				console.Debugln("mergeEntryChannels: discarding duplicate", best.name, "<=", last)
+			}
 		}
 		// Replace entry we just sent.
 		if err := selectFrom(bestIdx); err != nil {

cmd/metacache-set.go

@@ -866,7 +866,8 @@ func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) {
 			var combinedErr []string
 			for i, err := range errs {
 				if err != nil {
-					combinedErr = append(combinedErr, fmt.Sprintf("disk %d returned: %s", i, err))
+					combinedErr = append(combinedErr,
+						fmt.Sprintf("disk %s returned: %s", disks[i], err))
 				}
 			}
 			return errors.New(strings.Join(combinedErr, ", "))

cmd/naughty-disk_test.go

@@ -291,3 +291,10 @@ func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi Fi
 	}
 	return d.disk.VerifyFile(ctx, volume, path, fi)
 }
+
+func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+	if err := d.calcError(); err != nil {
+		return stat, err
+	}
+	return d.disk.StatInfoFile(ctx, volume, path)
+}

cmd/net.go

@@ -157,6 +157,9 @@ func sortIPs(ipList []string) []string {
 }
 
 func getConsoleEndpoints() (consoleEndpoints []string) {
+	if globalBrowserRedirectURL != nil {
+		return []string{globalBrowserRedirectURL.String()}
+	}
 	var ipList []string
 	if globalMinioConsoleHost == "" {
 		ipList = sortIPs(mustGetLocalIP4().ToSlice())

cmd/notification.go

@@ -860,7 +860,7 @@ func (sys *NotificationSys) GetNetPerfInfo(ctx context.Context) madmin.NetPerfIn
 		}
 	}
 	return madmin.NetPerfInfo{
-		Addr:        globalLocalNodeName,
+		NodeCommon:  madmin.NodeCommon{Addr: globalLocalNodeName},
 		RemotePeers: netInfos,
 	}
 }
@@ -935,7 +935,7 @@ func (sys *NotificationSys) GetParallelNetPerfInfo(ctx context.Context) madmin.N
 	}
 	wg.Wait()
 	return madmin.NetPerfInfo{
-		Addr:        globalLocalNodeName,
+		NodeCommon:  madmin.NodeCommon{Addr: globalLocalNodeName},
 		RemotePeers: netInfos,
 	}
 }

cmd/os-readdir-common.go

@@ -0,0 +1,36 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+// Options for readDir function call
+type readDirOpts struct {
+	// The maximum number of entries to return
+	count int
+	// Follow directory symlink
+	followDirSymlink bool
+}
+
+// Return all the entries at the directory dirPath.
+func readDir(dirPath string) (entries []string, err error) {
+	return readDirWithOpts(dirPath, readDirOpts{count: -1})
+}
+
+// Return up to count entries at the directory dirPath.
+func readDirN(dirPath string, count int) (entries []string, err error) {
+	return readDirWithOpts(dirPath, readDirOpts{count: count})
+}

cmd/os-readdir_other.go

@@ -30,11 +30,6 @@ func access(name string) error {
 	return err
 }
 
-// Return all the entries at the directory dirPath.
-func readDir(dirPath string) (entries []string, err error) {
-	return readDirN(dirPath, -1)
-}
-
 // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
@@ -90,8 +85,8 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
 	return nil
 }
 
-// Return N entries at the directory dirPath. If count is -1, return all entries
-func readDirN(dirPath string, count int) (entries []string, err error) {
+// Return entries at the directory dirPath.
+func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
 	d, err := Open(dirPath)
 	if err != nil {
 		return nil, osErrToFileErr(err)
@@ -99,12 +94,12 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	defer d.Close()
 
 	maxEntries := 1000
-	if count > 0 && count < maxEntries {
+	if opts.count > 0 && opts.count < maxEntries {
 		maxEntries = count
 	}
 
 	done := false
-	remaining := count
+	remaining := opts.count
 
 	for !done {
 		// Read up to max number of entries.
@@ -115,7 +110,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 			}
 			return nil, osErrToFileErr(err)
 		}
-		if count > -1 {
+		if opts.count > -1 {
 			if remaining <= len(fis) {
 				fis = fis[:remaining]
 				done = true
@@ -136,7 +131,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 				}
 
 				// Ignore symlinked directories.
-				if fi.IsDir() {
+				if !opts.followDirSymlink && fi.IsDir() {
 					continue
 				}
 			}
@@ -147,7 +142,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 			} else if fi.Mode().IsRegular() {
 				entries = append(entries, fi.Name())
 			}
-			if count > 0 {
+			if opts.count > 0 {
 				remaining--
 			}
 		}

cmd/os-readdir_unix.go

@@ -98,11 +98,6 @@ func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err er
 	return consumed, nameBuf[:nameLen], typ, nil
 }
 
-// Return all the entries at the directory dirPath.
-func readDir(dirPath string) (entries []string, err error) {
-	return readDirN(dirPath, -1)
-}
-
 // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
@@ -184,7 +179,7 @@ func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) erro
 
 // Return count entries at the directory dirPath and all entries
 // if count is set to -1
-func readDirN(dirPath string, count int) (entries []string, err error) {
+func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
 	f, err := Open(dirPath)
 	if err != nil {
 		return nil, osErrToFileErr(err)
@@ -202,6 +197,8 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	boff := 0 // starting read position in buf
 	nbuf := 0 // end valid data in buf
 
+	count := opts.count
+
 	for count != 0 {
 		if boff >= nbuf {
 			boff = 0
@@ -242,7 +239,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 			}
 
 			// Ignore symlinked directories.
-			if typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
+			if !opts.followDirSymlink && typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() {
 				continue
 			}
 

cmd/os-readdir_windows.go

@@ -30,11 +30,6 @@ func access(name string) error {
 	return err
 }
 
-// Return all the entries at the directory dirPath.
-func readDir(dirPath string) (entries []string, err error) {
-	return readDirN(dirPath, -1)
-}
-
 // readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into
 // the directory itself, if the dirPath doesn't exist this function doesn't return
 // an error.
@@ -118,8 +113,8 @@ func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error)
 	return nil
 }
 
-// Return N entries at the directory dirPath. If count is -1, return all entries
-func readDirN(dirPath string, count int) (entries []string, err error) {
+// Return N entries at the directory dirPath.
+func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) {
 	f, err := os.Open(dirPath)
 	if err != nil {
 		return nil, osErrToFileErr(err)
@@ -138,6 +133,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 	data := &syscall.Win32finddata{}
 	handle := syscall.Handle(f.Fd())
 
+	count := opts.count
 	for count != 0 {
 		e := syscall.FindNextFile(handle, data)
 		if e != nil {
@@ -172,7 +168,7 @@ func readDirN(dirPath string, count int) (entries []string, err error) {
 				return nil, err
 			}
 
-			if fi.IsDir() {
+			if !opts.followDirSymlink && fi.IsDir() {
 				// directory symlinks are ignored.
 				continue
 			}

cmd/peer-rest-server.go

@@ -129,8 +129,8 @@ func (s *peerRESTServer) LoadPolicyMappingHandler(w http.ResponseWriter, r *http
 		s.writeErrorResponse(w, errors.New("user-or-group is missing"))
 		return
 	}
-	_, isGroup := vars[peerRESTIsGroup]
 
+	_, isGroup := r.URL.Query()[peerRESTIsGroup]
 	if err := globalIAMSys.LoadPolicyMapping(objAPI, userOrGroup, isGroup); err != nil {
 		s.writeErrorResponse(w, err)
 		return
@@ -249,7 +249,7 @@ func (s *peerRESTServer) LoadUserHandler(w http.ResponseWriter, r *http.Request)
 		return
 	}
 
-	var userType = regularUser
+	var userType = regUser
 	if temp {
 		userType = stsUser
 	}
@@ -384,7 +384,6 @@ func (s *peerRESTServer) NetInfoHandler(w http.ResponseWriter, r *http.Request)
 	}
 	if n != r.ContentLength {
 		err := fmt.Errorf("Subnet health: short read: expected %d found %d", r.ContentLength, n)
-
 		logger.LogIf(ctx, err)
 		w.Header().Set("FinalStatus", err.Error())
 		return

cmd/postpolicyform.go

@@ -45,7 +45,7 @@ var startsWithConds = map[string]bool{
 	"$key":                     true,
 	"$success_action_redirect": true,
 	"$redirect":                true,
-	"$success_action_status":   false,
+	"$success_action_status":   true,
 	"$x-amz-algorithm":         false,
 	"$x-amz-credential":        false,
 	"$x-amz-date":              false,

cmd/routers.go

@@ -90,7 +90,7 @@ func configureServerHandler(endpointServerPools EndpointServerPools) (http.Handl
 	}
 
 	// Add Admin router, all APIs are enabled in server mode.
-	registerAdminRouter(router, true, true)
+	registerAdminRouter(router, true)
 
 	// Add healthcheck router
 	registerHealthCheckRouter(router)

cmd/signature-v4-utils.go

@@ -131,11 +131,12 @@ func checkKeyValid(accessKey string) (auth.Credentials, bool, APIErrorCode) {
 	var cred = globalActiveCred
 	if cred.AccessKey != accessKey {
 		// Check if the access key is part of users credentials.
-		var ok bool
-		if cred, ok = globalIAMSys.GetUser(accessKey); !ok {
+		ucred, ok := globalIAMSys.GetUser(accessKey)
+		if !ok {
 			return cred, false, ErrInvalidAccessKeyID
 		}
-		owner = false
+		owner = cred.AccessKey == ucred.ParentUser
+		cred = ucred
 	}
 	return cred, owner, ErrNone
 }

cmd/storage-interface.go

@@ -74,6 +74,7 @@ type StorageAPI interface {
 	CheckFile(ctx context.Context, volume string, path string) (err error)
 	Delete(ctx context.Context, volume string, path string, recursive bool) (err error)
 	VerifyFile(ctx context.Context, volume, path string, fi FileInfo) error
+	StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error)
 
 	// Write all data, syncs the data to disk.
 	// Should be used for smaller payloads.

cmd/storage-rest-client.go

@@ -684,6 +684,23 @@ func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path st
 	return toStorageErr(verifyResp.Err)
 }
 
+func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+	values := make(url.Values)
+	values.Set(storageRESTVolume, volume)
+	values.Set(storageRESTFilePath, path)
+	respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1)
+	if err != nil {
+		return stat, err
+	}
+	defer xhttp.DrainBody(respBody)
+	respReader, err := waitForHTTPResponse(respBody)
+	if err != nil {
+		return stat, err
+	}
+	err = stat.DecodeMsg(msgpNewReader(respReader))
+	return stat, err
+}
+
 // Close - marks the client as closed.
 func (client *storageRESTClient) Close() error {
 	client.restClient.Close()

cmd/storage-rest-common.go

@@ -52,6 +52,7 @@ const (
 	storageRESTMethodRenameFile     = "/renamefile"
 	storageRESTMethodVerifyFile     = "/verifyfile"
 	storageRESTMethodWalkDir        = "/walkdir"
+	storageRESTMethodStatInfoFile   = "/statfile"
 )
 
 const (

cmd/storage-rest-server.go

@@ -282,7 +282,7 @@ func (s *storageRESTServer) DeleteVolHandler(w http.ResponseWriter, r *http.Requ
 	}
 	vars := mux.Vars(r)
 	volume := vars[storageRESTVolume]
-	forceDelete := vars[storageRESTForceDelete] == "true"
+	forceDelete := r.URL.Query().Get(storageRESTForceDelete) == "true"
 	err := s.storage.DeleteVol(r.Context(), volume, forceDelete)
 	if err != nil {
 		s.writeErrorResponse(w, err)
@@ -1038,6 +1038,23 @@ func logFatalErrs(err error, endpoint Endpoint, exit bool) {
 	}
 }
 
+// StatInfoFile returns file stat info.
+func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request) {
+	if !s.IsValid(w, r) {
+		return
+	}
+	vars := mux.Vars(r)
+	volume := vars[storageRESTVolume]
+	filePath := vars[storageRESTFilePath]
+	done := keepHTTPResponseAlive(w)
+	si, err := s.storage.StatInfoFile(r.Context(), volume, filePath)
+	done(err)
+	if err != nil {
+		return
+	}
+	msgp.Encode(w, &si)
+}
+
 // registerStorageRPCRouter - register storage rpc router.
 func registerStorageRESTHandlers(router *mux.Router, endpointServerPools EndpointServerPools) {
 	storageDisks := make([][]*xlStorage, len(endpointServerPools))
@@ -1129,6 +1146,8 @@ func registerStorageRESTHandlers(router *mux.Router, endpointServerPools Endpoin
 				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodWalkDir).HandlerFunc(httpTraceHdrs(server.WalkDirHandler)).
 				Queries(restQueries(storageRESTVolume, storageRESTDirPath, storageRESTRecursive)...)
+			subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(httpTraceHdrs(server.StatInfoFile)).
+				Queries(restQueries(storageRESTVolume, storageRESTFilePath)...)
 		}
 	}
 }

cmd/storagemetric_string.go

@@ -32,12 +32,13 @@ func _() {
 	_ = x[storageMetricUpdateMetadata-21]
 	_ = x[storageMetricReadVersion-22]
 	_ = x[storageMetricReadAll-23]
-	_ = x[storageMetricLast-24]
+	_ = x[storageStatInfoFile-24]
+	_ = x[storageMetricLast-25]
 }
 
-const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsCheckFileDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadAllLast"
+const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsCheckFileDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadAllstorageStatInfoFileLast"
 
-var _storageMetric_index = [...]uint8{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 137, 143, 157, 167, 175, 188, 201, 215, 226, 233, 237}
+var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 137, 143, 157, 167, 175, 188, 201, 215, 226, 233, 252, 256}
 
 func (i storageMetric) String() string {
 	if i >= storageMetric(len(_storageMetric_index)-1) {

cmd/sts-handlers.go

@@ -64,10 +64,20 @@ const (
 	parentClaim = "parent"
 
 	// LDAP claim keys
-	ldapUser     = "ldapUser"
-	ldapUsername = "ldapUsername"
+	ldapUser  = "ldapUser"
+	ldapUserN = "ldapUsername"
 )
 
+func parseOpenIDParentUser(parentUser string) (userID string, err error) {
+	if strings.HasPrefix(parentUser, "openid:") {
+		tokens := strings.SplitN(strings.TrimPrefix(parentUser, "openid:"), ":", 2)
+		if len(tokens) == 2 {
+			return tokens[0], nil
+		}
+	}
+	return "", errSkipFile
+}
+
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
 type stsAPIHandlers struct{}
 
@@ -398,7 +408,7 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 	// this is to ensure that ParentUser doesn't change and we get to use
 	// parentUser as per the requirements for service accounts for OpenID
 	// based logins.
-	cred.ParentUser = "jwt:" + subFromToken + ":" + issFromToken
+	cred.ParentUser = "openid:" + subFromToken + ":" + issFromToken
 
 	// Set the newly generated credentials.
 	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
@@ -533,9 +543,9 @@ func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *
 
 	expiryDur := globalLDAPConfig.GetExpiryDuration()
 	m := map[string]interface{}{
-		expClaim:     UTCNow().Add(expiryDur).Unix(),
-		ldapUsername: ldapUsername,
-		ldapUser:     ldapUserDN,
+		expClaim:  UTCNow().Add(expiryDur).Unix(),
+		ldapUser:  ldapUserDN,
+		ldapUserN: ldapUsername,
 	}
 
 	if len(sessionPolicyStr) > 0 {

cmd/xl-storage-disk-id-check.go

@@ -58,6 +58,7 @@ const (
 	storageMetricUpdateMetadata
 	storageMetricReadVersion
 	storageMetricReadAll
+	storageStatInfoFile
 
 	// .... add more
 
@@ -611,6 +612,22 @@ func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path
 	return p.storage.ReadAll(ctx, volume, path)
 }
 
+func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+	defer p.updateStorageMetrics(storageStatInfoFile, volume, path)()
+
+	select {
+	case <-ctx.Done():
+		return StatInfo{}, ctx.Err()
+	default:
+	}
+
+	if err = p.checkDiskStale(); err != nil {
+		return StatInfo{}, err
+	}
+
+	return p.storage.StatInfoFile(ctx, volume, path)
+}
+
 func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string) madmin.TraceInfo {
 	return madmin.TraceInfo{
 		TraceType: madmin.TraceStorage,

cmd/xl-storage-format-v2.go

@@ -784,7 +784,9 @@ func (z *xlMetaV2) UpdateObjectVersion(fi FileInfo) error {
 				return nil
 			}
 		case DeleteType:
-			return errMethodNotAllowed
+			if version.DeleteMarker.VersionID == uv {
+				return errMethodNotAllowed
+			}
 		}
 	}
 

cmd/xl-storage.go

@@ -2284,3 +2284,32 @@ func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi File
 
 	return nil
 }
+
+func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string) (stat StatInfo, err error) {
+	volumeDir, err := s.getVolDir(volume)
+	if err != nil {
+		return stat, err
+	}
+
+	// Stat a volume entry.
+	if err = Access(volumeDir); err != nil {
+		if osIsNotExist(err) {
+			return stat, errVolumeNotFound
+		} else if isSysErrIO(err) {
+			return stat, errFaultyDisk
+		} else if osIsPermission(err) {
+			return stat, errVolumeAccessDenied
+		}
+		return stat, err
+	}
+	filePath := pathJoin(volumeDir, path)
+	if err := checkPathLength(filePath); err != nil {
+		return stat, err
+	}
+	st, _ := Lstat(filePath)
+	if st == nil {
+		return stat, errPathNotFound
+	}
+
+	return StatInfo{ModTime: st.ModTime(), Size: st.Size()}, nil
+}

dockerscripts/docker-entrypoint.sh

@@ -8,6 +8,20 @@ if [ "${1}" != "minio" ]; then
     fi
 fi
 
+## look for specific a `config.env` file to load all the
+## minio settings from
+docker_minio_env() {
+    if [ -f "$MINIO_CONFIG_ENV_FILE" ]; then
+        config_env_file="${MINIO_CONFIG_ENV_FILE}"
+    else
+        config_env_file="/run/secrets/${MINIO_CONFIG_ENV_FILE}"
+    fi
+    if [ -f "$config_env_file" ]; then
+        # shellcheck source=/dev/null
+        . "${config_env_file}"
+    fi
+}
+
 ## Look for docker secrets at given absolute path or in default documented location.
 docker_secrets_env_old() {
     if [ -f "$MINIO_ACCESS_KEY_FILE" ]; then
@@ -73,15 +87,16 @@ docker_kms_secret_encryption_env() {
 
 # su-exec to requested user, if service cannot run exec will fail.
 docker_switch_user() {
-    if [ ! -z "${MINIO_USERNAME}" ] && [ ! -z "${MINIO_GROUPNAME}" ]; then
-        if [ ! -z "${MINIO_UID}" ] && [ ! -z "${MINIO_GID}" ]; then
+    if [ -n "${MINIO_USERNAME}" ] && [ -n "${MINIO_GROUPNAME}" ]; then
+        if [ -n "${MINIO_UID}" ] && [ -n "${MINIO_GID}" ]; then
             groupadd -g "$MINIO_GID" "$MINIO_GROUPNAME" && \
                 useradd -u "$MINIO_UID" -g "$MINIO_GROUPNAME" "$MINIO_USERNAME"
         else
             groupadd "$MINIO_GROUPNAME" && \
                 useradd -g "$MINIO_GROUPNAME" "$MINIO_USERNAME"
         fi
-        exec setpriv --reuid="${MINIO_USERNAME}" --regid="${MINIO_GROUPNAME}" --keep-groups "$@"
+        exec setpriv --reuid="${MINIO_USERNAME}" \
+             --regid="${MINIO_GROUPNAME}" --keep-groups "$@"
     else
         exec "$@"
     fi
@@ -96,5 +111,10 @@ docker_secrets_env
 ## Set kms encryption from secrets if necessary. Override
 docker_kms_secret_encryption_env
 
+## Set all config environment variables from 'config.env' if necessary.
+## Overrides all previous settings and also overrides all
+## environment values passed from 'podman run -e ENV=value'
+docker_minio_env
+
 ## Switch to user if applicable.
 docker_switch_user "$@"

docs/debugging/inspect/main.go

@@ -0,0 +1,108 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bufio"
+	"encoding/binary"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"hash/crc32"
+	"io"
+	"log"
+	"os"
+	"strings"
+
+	"github.com/secure-io/sio-go"
+)
+
+var (
+	key = flag.String("key", "", "decryption string")
+	//js = flag.Bool("json", false, "expect json input")
+)
+
+func main() {
+	flag.Parse()
+	args := flag.Args()
+	switch len(flag.Args()) {
+	case 0:
+		// Read from stdin, write to stdout.
+		decrypt(*key, os.Stdin, os.Stdout)
+		return
+	case 1:
+		r, err := os.Open(args[0])
+		fatalErr(err)
+		defer r.Close()
+		dstName := strings.TrimSuffix(args[0], ".enc") + ".zip"
+		w, err := os.Create(dstName)
+		fatalErr(err)
+		defer w.Close()
+		if len(*key) == 0 {
+			reader := bufio.NewReader(os.Stdin)
+			fmt.Print("Enter Decryption Key: ")
+
+			text, _ := reader.ReadString('\n')
+			// convert CRLF to LF
+			*key = strings.Replace(text, "\n", "", -1)
+		}
+		decrypt(*key, r, w)
+		fmt.Println("Output decrypted to", dstName)
+		return
+	default:
+		fatalIf(true, "Only 1 file can be decrypted")
+		os.Exit(1)
+	}
+}
+
+func decrypt(keyHex string, r io.Reader, w io.Writer) {
+	keyHex = strings.TrimSpace(keyHex)
+	fatalIf(len(keyHex) != 72, "Unexpected key length: %d, want 72", len(keyHex))
+	id, err := hex.DecodeString(keyHex[:8])
+	fatalErr(err)
+	key, err := hex.DecodeString(keyHex[8:])
+	fatalErr(err)
+
+	// Verify that CRC is ok.
+	want := binary.LittleEndian.Uint32(id)
+	got := crc32.ChecksumIEEE(key)
+	fatalIf(want != got, "Invalid key checksum, want %x, got %x", want, got)
+
+	stream, err := sio.AES_256_GCM.Stream(key)
+	fatalErr(err)
+
+	// Zero nonce, we only use each key once, and 32 bytes is plenty.
+	nonce := make([]byte, stream.NonceSize())
+	encr := stream.DecryptReader(r, nonce, nil)
+	_, err = io.Copy(w, encr)
+	fatalErr(err)
+}
+
+func fatalErr(err error) {
+	if err == nil {
+		return
+	}
+	log.Fatalln(err)
+}
+
+func fatalIf(b bool, msg string, v ...interface{}) {
+	if !b {
+		return
+	}
+	log.Fatalf(msg, v...)
+}

docs/disk-caching/DESIGN.md

@@ -61,7 +61,7 @@ sudo mount -o relatime /tmp/data /mnt/cache
 podman run --net=host -e MINIO_ROOT_USER={s3-access-key} -e MINIO_ROOT_PASSWORD={s3-secret-key} \
     -e MINIO_CACHE_DRIVES=/cache -e MINIO_CACHE_QUOTA=99 -e MINIO_CACHE_AFTER=0 \
     -e MINIO_CACHE_WATERMARK_LOW=90 -e MINIO_CACHE_WATERMARK_HIGH=95 \
-    -v /mnt/cache:/cache  minio/minio:latest gateway s3
+    -v /mnt/cache:/cache  minio/minio:latest gateway s3 --console-address ":9001"
 ```
 
 ## Assumptions

docs/docker/README.md

@@ -7,32 +7,38 @@ Docker installed on your machine. Download the relevant installer from [here](ht
 MinIO needs a persistent volume to store configuration and application data. However, for testing purposes, you can launch MinIO by simply passing a directory (`/data` in the example below). This directory gets created in the container filesystem at the time of container start. But all the data is lost after container exits.
 
 ```sh
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 To create a MinIO container with persistent storage, you need to map local persistent directories from the host OS to virtual config `~/.minio` and export `/data` directories. To do this, run the below commands
 
 #### GNU/Linux and macOS
 ```sh
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   --name minio1 \
   -v /mnt/data:/data \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 #### Windows
 ```sh
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   --name minio1 \
   -v D:\data:/data \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 ## Run Distributed MinIO on Docker
@@ -45,20 +51,26 @@ To override MinIO's auto-generated keys, you may pass secret and access keys exp
 
 #### GNU/Linux and macOS
 ```sh
-docker run -p 9000:9000 --name minio1 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  --name minio1 \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   -v /mnt/data:/data \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 #### Windows
 ```powershell
-docker run -p 9000:9000 --name minio1 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  --name minio1 \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
   -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \
   -v D:\data:/data \
-  minio/minio server /data
+  minio/minio server /data --console-address ":9001"
 ```
 
 ### Run MinIO Docker as a regular user
@@ -70,7 +82,9 @@ On Linux and macOS you can use `--user` to run the container as regular user.
 > NOTE: make sure --user has write permission to *${HOME}/data* prior to using `--user`.
 ```sh
 mkdir -p ${HOME}/data
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   --user $(id -u):$(id -g) \
   --name minio1 \
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \
@@ -85,7 +99,9 @@ On windows you would need to use [Docker integrated windows authentication](http
 > NOTE: make sure your AD/Windows user has write permissions to *D:\data* prior to using `credentialspec=`.
 
 ```powershell
-docker run -p 9000:9000 \
+docker run \
+  -p 9000:9000 \
+  -p 9001:9001 \
   --name minio1 \
   --security-opt "credentialspec=file://myuser.json"
   -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \

docs/erasure/README.md

@@ -44,10 +44,13 @@ Example: Start MinIO server in a 12 drives setup, using MinIO binary.
 minio server /data{1...12}
 ```
 
-Example: Start MinIO server in a 8 drives setup, using MinIO Docker image. 
+Example: Start MinIO server in a 8 drives setup, using MinIO Docker image.
 
 ```sh
-podman run -p 9000:9000 --name minio \
+podman run \
+  -p 9000:9000 \
+  -p 9001:9001 \
+  --name minio \
   -v /mnt/data1:/data1 \
   -v /mnt/data2:/data2 \
   -v /mnt/data3:/data3 \
@@ -56,7 +59,7 @@ podman run -p 9000:9000 --name minio \
   -v /mnt/data6:/data6 \
   -v /mnt/data7:/data7 \
   -v /mnt/data8:/data8 \
-  minio/minio server /data{1...8}
+  minio/minio server /data{1...8} --console-address ":9001"
 ```
 
 ### 3. Test your setup

docs/gateway/azure.md

@@ -4,10 +4,13 @@ MinIO Gateway adds Amazon S3 compatibility to Microsoft Azure Blob Storage.
 ## Run MinIO Gateway for Microsoft Azure Blob Storage
 ### Using Docker
 ```
-podman run -p 9000:9000 --name azure-s3 \
+podman run \
+ -p 9000:9000 \
+ -p 9001:9001 \
+ --name azure-s3 \
  -e "MINIO_ROOT_USER=azurestorageaccountname" \
  -e "MINIO_ROOT_PASSWORD=azurestorageaccountkey" \
- minio/minio gateway azure
+ minio/minio gateway azure --console-address ":9001"
 ```
 
 ### Using Binary

docs/gateway/gcs.md

@@ -21,12 +21,15 @@ MinIO GCS Gateway allows you to access Google Cloud Storage (GCS) with Amazon S3
 
 ### 1.2 Run MinIO GCS Gateway Using Docker
 ```sh
-podman run -p 9000:9000 --name gcs-s3 \
+podman run \
+ -p 9000:9000 \
+ -p 9001:9001 \
+ --name gcs-s3 \
  -v /path/to/credentials.json:/credentials.json \
  -e "GOOGLE_APPLICATION_CREDENTIALS=/credentials.json" \
  -e "MINIO_ROOT_USER=minioaccountname" \
  -e "MINIO_ROOT_PASSWORD=minioaccountkey" \
- minio/minio gateway gcs yourprojectid
+ minio/minio gateway gcs yourprojectid --console-address ":9001"
 ```
 
 ### 1.3 Run MinIO GCS Gateway Using the MinIO Binary

docs/gateway/hdfs.md

@@ -23,11 +23,13 @@ minio gateway hdfs hdfs://namenode:8200
 ### Using Docker
 Using docker is experimental, most Hadoop environments are not dockerized and may require additional steps in getting this to work properly. You are better off just using the binary in this situation.
 ```
-podman run -p 9000:9000 \
+podman run \
+ -p 9000:9000 \
+ -p 9001:9001 \
  --name hdfs-s3 \
  -e "MINIO_ROOT_USER=minio" \
  -e "MINIO_ROOT_PASSWORD=minio123" \
- minio/minio gateway hdfs hdfs://namenode:8200
+ minio/minio gateway hdfs hdfs://namenode:8200 --console-address ":9001"
 ```
 
 ### Setup Kerberos

docs/gateway/nas.md

@@ -9,11 +9,14 @@ MinIO Gateway adds Amazon S3 compatibility to NAS storage. You may run multiple
 Please ensure to replace `/shared/nasvol` with actual mount path.
 
 ```
-podman run -p 9000:9000 --name nas-s3 \
+podman run \
+ -p 9000:9000 \
+ -p 9001:9001 \
+ --name nas-s3 \
  -e "MINIO_ROOT_USER=minio" \
  -e "MINIO_ROOT_PASSWORD=minio123" \
  -v /shared/nasvol:/container/vol \
- minio/minio gateway nas /container/vol
+ minio/minio gateway nas /container/vol --console-address ":9001"
 ```
 
 ### Using Binary

docs/gateway/s3.md

@@ -7,10 +7,13 @@ As a prerequisite to run MinIO S3 gateway, you need valid AWS S3 access key and
 
 ### Using Docker
 ```
-podman run -p 9000:9000 --name minio-s3 \
+podman run \
+ -p 9000:9000 \
+ -p 9001:9001 \
+ --name minio-s3 \
  -e "MINIO_ROOT_USER=aws_s3_access_key" \
  -e "MINIO_ROOT_PASSWORD=aws_s3_secret_key" \
- minio/minio gateway s3
+ minio/minio gateway s3 --console-address ":9001"
 ```
 
 ### Using Binary
@@ -54,6 +57,7 @@ Minimum permissions required if you wish to provide restricted access with your
             "Sid": "readonly",
             "Effect": "Allow",
             "Action": [
+                "s3:GetBucketPolicy",
                 "s3:HeadBucket",
                 "s3:ListBucket"
             ],

docs/logging/README.md

@@ -36,9 +36,11 @@ minio server /mnt/data
 
 ## Audit Targets
 Assuming `mc` is already [configured](https://docs.min.io/docs/minio-client-quickstart-guide.html)
+
+### HTTP Target
 ```
 mc admin config get myminio/ audit_webhook
-audit_webhook:name1 enable=off endpoint= auth_token= client_cert= client_key= 
+audit_webhook:name1 enable=off endpoint= auth_token= client_cert= client_key=
 ```
 
 ```
@@ -119,6 +121,83 @@ NOTE:
 }
 ```
 
+### Kafka Target
+Assuming that you already have Apache Kafka configured and running.
+```
+mc admin config set myminio/ audit_kafka
+KEY:
+audit_kafka[:name]  send audit logs to kafka endpoints
+
+ARGS:
+brokers*         (csv)       comma separated list of Kafka broker addresses
+topic            (string)    Kafka topic used for bucket notifications
+sasl_username    (string)    username for SASL/PLAIN or SASL/SCRAM authentication
+sasl_password    (string)    password for SASL/PLAIN or SASL/SCRAM authentication
+sasl_mechanism   (string)    sasl authentication mechanism, default 'plain'
+tls_client_auth  (string)    clientAuth determines the Kafka server's policy for TLS client auth
+sasl             (on|off)    set to 'on' to enable SASL authentication
+tls              (on|off)    set to 'on' to enable TLS
+tls_skip_verify  (on|off)    trust server TLS without verification, defaults to "on" (verify)
+client_tls_cert  (path)      path to client certificate for mTLS auth
+client_tls_key   (path)      path to client key for mTLS auth
+version          (string)    specify the version of the Kafka cluster
+comment          (sentence)  optionally add a comment to this setting
+```
+
+Configure MinIO to send audit logs to locally running Kafka brokers
+```
+mc admin config set myminio/ audit_kafka:target1 brokers=localhost:29092 topic=auditlog
+mc admin service restart myminio/
+```
+
+On another terminal assuming you have `kafkacat` installed
+
+```
+kafkacat -b localhost:29092 -t auditlog  -C
+
+{"version":"1","deploymentid":"8a1d8091-b874-45df-b9ea-e044eede6ace","time":"2021-07-13T02:00:47.020547414Z","trigger":"incoming","api":{"name":"ListBuckets","status":"OK","statusCode":200,"timeToFirstByte":"261795ns","timeToResponse":"312490ns"},"remotehost":"127.0.0.1","requestID":"16913736591C237F","userAgent":"MinIO (linux; amd64) minio-go/v7.0.11 mc/DEVELOPMENT.2021-07-09T02-22-26Z","requestHeader":{"Authorization":"AWS4-HMAC-SHA256 Credential=minio/20210713/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=7fe65c5467e05ca21de64094688da43f96f34fec82e8955612827079f4600527","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.11 mc/DEVELOPMENT.2021-07-09T02-22-26Z","X-Amz-Content-Sha256":"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855","X-Amz-Date":"20210713T020047Z"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"547","Content-Security-Policy":"block-all-mixed-content","Content-Type":"application/xml","Server":"MinIO","Vary":"Origin,Accept-Encoding","X-Amz-Request-Id":"16913736591C237F","X-Xss-Protection":"1; mode=block"}}
+```
+
+MinIO also honors environment variable for Kafka target Audit logging as shown below, this setting will override the endpoint settings in the MinIO server config.
+
+```
+mc admin config set myminio/ audit_kafka --env
+KEY:
+audit_kafka[:name]  send audit logs to kafka endpoints
+
+ARGS:
+MINIO_AUDIT_KAFKA_ENABLE*          (on|off)    enable audit_kafka target, default is 'off'
+MINIO_AUDIT_KAFKA_BROKERS*         (csv)       comma separated list of Kafka broker addresses
+MINIO_AUDIT_KAFKA_TOPIC            (string)    Kafka topic used for bucket notifications
+MINIO_AUDIT_KAFKA_SASL_USERNAME    (string)    username for SASL/PLAIN or SASL/SCRAM authentication
+MINIO_AUDIT_KAFKA_SASL_PASSWORD    (string)    password for SASL/PLAIN or SASL/SCRAM authentication
+MINIO_AUDIT_KAFKA_SASL_MECHANISM   (string)    sasl authentication mechanism, default 'plain'
+MINIO_AUDIT_KAFKA_TLS_CLIENT_AUTH  (string)    clientAuth determines the Kafka server's policy for TLS client auth
+MINIO_AUDIT_KAFKA_SASL             (on|off)    set to 'on' to enable SASL authentication
+MINIO_AUDIT_KAFKA_TLS              (on|off)    set to 'on' to enable TLS
+MINIO_AUDIT_KAFKA_TLS_SKIP_VERIFY  (on|off)    trust server TLS without verification, defaults to "on" (verify)
+MINIO_AUDIT_KAFKA_CLIENT_TLS_CERT  (path)      path to client certificate for mTLS auth
+MINIO_AUDIT_KAFKA_CLIENT_TLS_KEY   (path)      path to client key for mTLS auth
+MINIO_AUDIT_KAFKA_VERSION          (string)    specify the version of the Kafka cluster
+MINIO_AUDIT_KAFKA_COMMENT          (sentence)  optionally add a comment to this setting
+```
+
+```
+export MINIO_AUDIT_KAFKA_ENABLE_target1="on"
+export MINIO_AUDIT_KAFKA_BROKERS_target1="localhost:29092"
+export MINIO_AUDIT_KAFKA_TOPIC_target1="auditlog"
+minio server /mnt/data
+```
+
+Setting this environment variable automatically enables audit logging to the Kafka target. The audit logging is in JSON format as described below.
+
+NOTE:
+- `timeToFirstByte` and `timeToResponse` will be expressed in Nanoseconds.
+- Additionally in the case of the erasure coded setup `tags.objectErasureMap` provides per object details about
+   - Pool number the object operation was performed on.
+   - Set number the object operation was performed on.
+   - The list of disks participating in this operation belong to the set.
+
 ## Explore Further
 * [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide)
 * [Configure MinIO Server with TLS](https://docs.min.io/docs/how-to-secure-access-to-minio-server-with-tls)

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,17 +5,18 @@ version: '3.7'
 # it through port 9000.
 services:
   minio1:
-    image: minio/minio:RELEASE.2021-06-17T00-10-46Z
+    image: minio/minio:RELEASE.2021-07-12T02-44-53Z
     hostname: minio1
     volumes:
       - data1-1:/data1
       - data1-2:/data2
     expose:
       - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
-    command: server http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -23,17 +24,18 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2021-06-17T00-10-46Z
+    image: minio/minio:RELEASE.2021-07-12T02-44-53Z
     hostname: minio2
     volumes:
       - data2-1:/data1
       - data2-2:/data2
     expose:
       - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
-    command: server http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -41,17 +43,18 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2021-06-17T00-10-46Z
+    image: minio/minio:RELEASE.2021-07-12T02-44-53Z
     hostname: minio3
     volumes:
       - data3-1:/data1
       - data3-2:/data2
     expose:
       - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
-    command: server http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -59,17 +62,18 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2021-06-17T00-10-46Z
+    image: minio/minio:RELEASE.2021-07-12T02-44-53Z
     hostname: minio4
     volumes:
       - data4-1:/data1
       - data4-2:/data2
     expose:
       - "9000"
+      - "9001"
     environment:
       MINIO_ROOT_USER: minio
       MINIO_ROOT_PASSWORD: minio123
-    command: server http://minio{1...4}/data{1...2}
+    command: server --console-address ":9001" http://minio{1...4}/data{1...2}
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
       interval: 30s
@@ -83,6 +87,7 @@ services:
       - ./nginx.conf:/etc/nginx/nginx.conf:ro
     ports:
       - "9000:9000"
+      - "9001:9001"
     depends_on:
       - minio1
       - minio2

docs/orchestration/docker-compose/nginx.conf

@@ -29,6 +29,14 @@ http {
         server minio4:9000;
     }
 
+    upstream console {
+        ip_hash;
+        server minio1:9001;
+        server minio2:9001;
+        server minio3:9001;
+        server minio4:9001;
+    }
+
     server {
         listen       9000;
         listen  [::]:9000;
@@ -57,4 +65,37 @@ http {
             proxy_pass http://minio;
         }
     }
+
+    server {
+        listen       9001;
+        listen  [::]:9001;
+        server_name  localhost;
+
+        # To allow special characters in headers
+        ignore_invalid_headers off;
+        # Allow any size file to be uploaded.
+        # Set to a value such as 1000m; to restrict file size to a specific value
+        client_max_body_size 0;
+        # To disable buffering
+        proxy_buffering off;
+
+        location / {
+            proxy_set_header Host $http_host;
+            proxy_set_header X-Real-IP $remote_addr;
+            proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+            proxy_set_header X-Forwarded-Proto $scheme;
+            proxy_set_header X-NginX-Proxy true;
+
+            # This is necessary to pass the correct IP to be hashed
+            real_ip_header X-Real-IP;
+
+            proxy_connect_timeout 300;
+            # Default is HTTP/1, keepalive is only enabled in HTTP/1.1
+            proxy_http_version 1.1;
+            proxy_set_header Connection "";
+            chunked_transfer_encoding off;
+
+            proxy_pass http://console;
+        }
+    }
 }

docs/select/README.md

@@ -24,7 +24,7 @@ To enable Parquet set the environment variable `MINIO_API_SELECT_PARQUET=on`.
 # Example using Python API 
 
 ## 1. Prerequisites
-- Install MinIO Server from [here](http://docs.min.io/docs/minio-quickstart-guide).
+- Install MinIO Server from [here](https://docs.min.io/docs/minio-quickstart-guide).
 - Familiarity with AWS S3 API.
 - Familiarity with Python and installing dependencies.
 

docs/sts/assume-role.md

@@ -7,7 +7,7 @@ Returns a set of temporary security credentials that you can use to access MinIO
 - To be able to reliably use S3 multipart APIs feature of the SDKs without re-inventing the wheel of pre-signing the each URL in multipart API. This is very tedious to implement with all the scenarios of fault tolerance that's already implemented by the client SDK. The general client SDKs don't support multipart with presigned URLs.
 - To be able to easily get the temporary credentials to upload to a prefix. Make it possible for a client to upload a whole folder using the session. The server side applications need not create a presigned URL and serve to the client for each file. Since, the client would have the session it can do it by itself.
 
-The temporary security credentials returned by this API consists of an access key, a secret key, and a security token. Applications can use these temporary security credentials to sign calls to MinIO API operations. The policy applied to these temporary credentials is inherited from the MinIO user credentials. By default, the temporary security credentials created by AssumeRole last for one hour. However, use the optional DurationSeconds parameter to specify the duration of the credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 7 days.
+The temporary security credentials returned by this API consists of an access key, a secret key, and a security token. Applications can use these temporary security credentials to sign calls to MinIO API operations. The policy applied to these temporary credentials is inherited from the MinIO user credentials. By default, the temporary security credentials created by AssumeRole last for one hour. However, use the optional DurationSeconds parameter to specify the duration of the credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 365 days.
 
 ## API Request Parameters
 ### Version
@@ -22,13 +22,13 @@ Indicates STS API version information, the only supported value is '2011-06-15'.
 Indicates STS API Authorization information. If you are familiar with AWS Signature V4 Authorization header, this STS API supports signature V4 authorization as mentioned [here](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html)
 
 ### DurationSeconds
-The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 7 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds.
+The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 365 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds.
 
-| Params        | Value                                            |
-| :--           | :--                                              |
-| *Type*        | *Integer*                                        |
-| *Valid Range* | *Minimum value of 900. Maximum value of 604800.* |
-| *Required*    | *No*                                             |
+| Params        | Value                                              |
+| :--           | :--                                                |
+| *Type*        | *Integer*                                          |
+| *Valid Range* | *Minimum value of 900. Maximum value of 31536000.* |
+| *Required*    | *No*                                               |
 
 ### Policy
 An IAM policy in JSON format that you want to use as an inline session policy. This parameter is optional. Passing policies to this operation returns new temporary credentials. The resulting session's permissions are the intersection of the canned policy name and the policy set here. You cannot use this policy to grant more permissions than those allowed by the canned policy name being assumed.

docs/sts/client-grants.md

@@ -6,7 +6,7 @@ Returns a set of temporary security credentials for applications/clients who hav
 
 Calling AssumeRoleWithClientGrants does not require the use of MinIO default credentials. Therefore, client application can be distributed that requests temporary security credentials without including MinIO default credentials. Instead, the identity of the caller is validated by using a JWT access token from the identity provider. The temporary security credentials returned by this API consists of an access key, a secret key, and a security token. Applications can use these temporary security credentials to sign calls to MinIO API operations.
 
-By default, the temporary security credentials created by AssumeRoleWithClientGrants last for one hour. However, use the optional DurationSeconds parameter to specify the duration of the credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 7 days.
+By default, the temporary security credentials created by AssumeRoleWithClientGrants last for one hour. However, use the optional DurationSeconds parameter to specify the duration of the credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 365 days.
 
 ## API Request Parameters
 ### Token
@@ -27,13 +27,13 @@ Indicates STS API version information, the only supported value is '2011-06-15'.
 | *Required* | *Yes*    |
 
 ### DurationSeconds
-The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 7 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds. If no *DurationSeconds* is specified expiry seconds is obtained from *Token*.
+The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 365 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds. If no *DurationSeconds* is specified expiry seconds is obtained from *Token*.
 
-| Params        | Value                                            |
-| :--           | :--                                              |
-| *Type*        | *Integer*                                        |
-| *Valid Range* | *Minimum value of 900. Maximum value of 604800.* |
-| *Required*    | *No*                                             |
+| Params        | Value                                              |
+| :--           | :--                                                |
+| *Type*        | *Integer*                                          |
+| *Valid Range* | *Minimum value of 900. Maximum value of 31536000.* |
+| *Required*    | *No*                                               |
 
 ### Policy
 An IAM policy in JSON format that you want to use as an inline session policy. This parameter is optional. Passing policies to this operation returns new temporary credentials. The resulting session's permissions are the intersection of the canned policy name and the policy set here. You cannot use this policy to grant more permissions than those allowed by the canned policy name being assumed.

docs/sts/etcd.md

@@ -10,7 +10,7 @@ etcd uses [gcr.io/etcd-development/etcd](https://console.cloud.google.com/gcr/im
 
 ```
 rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \
-  docker rmi gcr.io/etcd-development/etcd:v3.3.9 || true && \
+  podman rmi gcr.io/etcd-development/etcd:v3.3.9 || true && \
   podman run \
   -p 2379:2379 \
   -p 2380:2380 \

docs/sts/keycloak.md

@@ -6,12 +6,15 @@ Keycloak is an open source Identity and Access Management solution aimed at mode
 
 Configure and install keycloak server by following [Keycloak Installation Guide](https://www.keycloak.org/docs/latest/getting_started/index.html) (finish upto section 3.4)
 
-### Configure Keycloak UI
+### Configure Keycloak Realm
 - Go to Clients
   - Click on account
-  - Settings
-  - Enable `Implicit Flow`
-  - Save
+    - Settings
+    - Change `Access Type` to `confidential`.
+    - Save
+  - Click on credentials tab
+    - Copy the `Secret` to clipboard.
+    - This value is needed for `MINIO_IDENTITY_OPENID_CLIENT_SECRET` for MinIO.
 
 - Go to Users
   - Click on the user
@@ -36,6 +39,42 @@ Configure and install keycloak server by following [Keycloak Installation Guide]
 
 - Open http://localhost:8080/auth/realms/minio/.well-known/openid-configuration to verify OpenID discovery document, verify it has `authorization_endpoint` and `jwks_uri`
 
+### Enable Keycloak Admin REST API support
+Before being able to authenticate against the Admin REST API using a client_id and a client_secret you need to make sure the client is configured as it follows:
+
+- `account` client_id is a confidential client that belongs to the realm `{realm}`
+- `account` client_id is has **Service Accounts Enabled** option enabled.
+- `account` client_id has a custom "Audience" mapper, in the Mappers section.
+  - Included Client Audience: security-admin-console
+
+#### Adding 'admin' Role
+
+- Go to Roles
+  - Add new Role `admin` with Description `${role_admin}`.
+  - Add this Role into compositive role named `default-roles-{realm}` - `{realm}` should be replaced with whatever realm you created from `prerequisites` section. This role is automatically trusted in the 'Service Accounts' tab.
+
+- Check that `account` client_id has the role 'admin' assigned in the "Service Account Roles" tab.
+
+After that, you will be able to obtain an access token for the Admin REST API using client_id and client_secret:
+
+```
+curl \
+  -d "client_id=<YOUR_CLIENT_ID>" \
+  -d "client_secret=<YOUR_CLIENT_SECRET>" \
+  -d "grant_type=client_credentials" \
+  "http://localhost:8080/auth/realms/{realm}/protocol/openid-connect/token"
+```
+
+The result will be a JSON document. To invoke the API you need to extract the value of the access_token property. You can then invoke the API by including the value in the Authorization header of requests to the API.
+
+The following example shows how to get the details of the user with `{userid}` from `{realm}` realm:
+
+```
+curl \
+  -H "Authorization: Bearer eyJhbGciOiJSUz..." \
+  "http://localhost:8080/auth/admin/realms/{realm}/users/{userid}"
+```
+
 ### Configure MinIO
 ```
 $ export MINIO_ROOT_USER=minio

docs/sts/ldap.go

@@ -23,8 +23,10 @@ import (
 	"context"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/url"
+	"os"
 
 	"github.com/minio/minio-go/v7"
 	cr "github.com/minio/minio-go/v7/pkg/credentials"
@@ -37,12 +39,24 @@ var (
 	// LDAP credentials
 	ldapUsername string
 	ldapPassword string
+
+	// Display credentials flag
+	displayCreds bool
+
+	// Bucket to list
+	bucketToList string
+
+	// Session policy file
+	sessionPolicyFile string
 )
 
 func init() {
 	flag.StringVar(&stsEndpoint, "sts-ep", "http://localhost:9000", "STS endpoint")
 	flag.StringVar(&ldapUsername, "u", "", "AD/LDAP Username")
 	flag.StringVar(&ldapPassword, "p", "", "AD/LDAP Password")
+	flag.BoolVar(&displayCreds, "d", false, "Only show generated credentials")
+	flag.StringVar(&bucketToList, "b", "", "Bucket to list (defaults to ldap username)")
+	flag.StringVar(&sessionPolicyFile, "s", "", "File containing session policy to apply to the STS request")
 }
 
 func main() {
@@ -56,11 +70,30 @@ func main() {
 	// LDAP STS API.
 
 	// Initialize LDAP credentials
-	li, _ := cr.NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword)
+	var li *cr.Credentials
+	var err error
+	if sessionPolicyFile == "" {
+		li, err = cr.NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword)
+	} else {
+		var policy string
+		if f, err := os.Open(sessionPolicyFile); err != nil {
+			log.Fatalf("Unable to open session policy file: %v", sessionPolicyFile, err)
+		} else {
+			bs, err := ioutil.ReadAll(f)
+			if err != nil {
+				log.Fatalf("Error reading session policy file: %v", err)
+			}
+			policy = string(bs)
+		}
+		li, err = cr.NewLDAPIdentityWithSessionPolicy(stsEndpoint, ldapUsername, ldapPassword, policy)
+	}
+	if err != nil {
+		log.Fatalf("Error initializing LDAP Identity: %v", err)
+	}
 
 	stsEndpointURL, err := url.Parse(stsEndpoint)
 	if err != nil {
-		log.Fatalf("Err: %v", err)
+		log.Fatalf("Error parsing sts endpoint: %v", err)
 	}
 
 	opts := &minio.Options{
@@ -68,22 +101,35 @@ func main() {
 		Secure: stsEndpointURL.Scheme == "https",
 	}
 
-	fmt.Println(li.Get())
+	v, err := li.Get()
+	if err != nil {
+		log.Fatalf("Error retrieving STS credentials: %v", err)
+	}
+
+	if displayCreds {
+		fmt.Println("Only displaying credentials:")
+		fmt.Println("AccessKeyID:", v.AccessKeyID)
+		fmt.Println("SecretAccessKey:", v.SecretAccessKey)
+		fmt.Println("SessionToken:", v.SessionToken)
+		return
+	}
+
 	// Use generated credentials to authenticate with MinIO server
 	minioClient, err := minio.New(stsEndpointURL.Host, opts)
 	if err != nil {
-		log.Fatalln(err)
+		log.Fatalf("Error initializing client: ", err)
 	}
 
 	// Use minIO Client object normally like the regular client.
-	fmt.Println("Calling list objects with temp creds: ")
-	objCh := minioClient.ListObjects(context.Background(), ldapUsername, minio.ListObjectsOptions{})
+	if bucketToList == "" {
+		bucketToList = ldapUsername
+	}
+	fmt.Printf("Calling list objects on bucket named `%s` with temp creds:\n===\n", bucketToList)
+	objCh := minioClient.ListObjects(context.Background(), bucketToList, minio.ListObjectsOptions{})
 	for obj := range objCh {
 		if obj.Err != nil {
-			if err != nil {
-				log.Fatalln(err)
-			}
+			log.Fatalf("Listing error: %v", obj.Err)
 		}
-		fmt.Println(obj)
+		fmt.Printf("Key: %s\nSize: %d\nLast Modified: %s\n===\n", obj.Key, obj.Size, obj.LastModified)
 	}
 }

docs/sts/ldap.md

@@ -95,10 +95,12 @@ export MINIO_IDENTITY_LDAP_SERVER_ADDR=myldapserver.com:636
 export MINIO_IDENTITY_LDAP_USERNAME_FORMAT="uid=%s,cn=accounts,dc=myldapserver,dc=com"
 export MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN="dc=myldapserver,dc=com"
 export MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER="(&(objectclass=groupOfNames)(memberUid=%s)$)"
-export MINIO_IDENTITY_LDAP_STS_EXPIRY=60h
+export MINIO_IDENTITY_LDAP_STS_EXPIRY=720h
 export MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY=on
 ```
 
+> NOTE: In this example STS_EXPIRY is set to 1month, maximum expiry that can be set is 365 days.
+
 ### Variable substitution in AD/LDAP configuration strings ###
 
 In the configuration variables, `%s` is substituted with the *username* from the STS request and `%d` is substituted with the *distinguished username (user DN)* of the LDAP user. Please see the following table for which configuration variables support these substitution variables:

docs/sts/web-identity.md

@@ -4,7 +4,7 @@
 
 Calling AssumeRoleWithWebIdentity does not require the use of MinIO default credentials. Therefore, you can distribute an application (for example, on mobile devices) that requests temporary security credentials without including MinIO default credentials in the application. Instead, the identity of the caller is validated by using a JWT access token from the web identity provider. The temporary security credentials returned by this API consists of an access key, a secret key, and a security token. Applications can use these temporary security credentials to sign calls to MinIO API operations.
 
-By default, the temporary security credentials created by AssumeRoleWithWebIdentity last for one hour. However, use the optional DurationSeconds parameter to specify the duration of the credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 7 days.
+By default, the temporary security credentials created by AssumeRoleWithWebIdentity last for one hour. However, use the optional DurationSeconds parameter to specify the duration of the credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 365 days.
 
 ## API Request Parameters
 ### WebIdentityToken
@@ -25,13 +25,13 @@ Indicates STS API version information, the only supported value is '2011-06-15'.
 | *Required* | *Yes*    |
 
 ### DurationSeconds
-The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 7 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds. If no *DurationSeconds* is specified expiry seconds is obtained from *WebIdentityToken*.
+The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 365 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds. If no *DurationSeconds* is specified expiry seconds is obtained from *WebIdentityToken*.
 
-| Params        | Value                                            |
-| :--           | :--                                              |
-| *Type*        | *Integer*                                        |
-| *Valid Range* | *Minimum value of 900. Maximum value of 604800.* |
-| *Required*    | *No*                                             |
+| Params        | Value                                              |
+| :--           | :--                                                |
+| *Type*        | *Integer*                                          |
+| *Valid Range* | *Minimum value of 900. Maximum value of 31536000.* |
+| *Required*    | *No*                                               |
 
 ### Policy
 An IAM policy in JSON format that you want to use as an inline session policy. This parameter is optional. Passing policies to this operation returns new temporary credentials. The resulting session's permissions are the intersection of the canned policy name and the policy set here. You cannot use this policy to grant more permissions than those allowed by the canned policy name being assumed.

docs/tls/README.md

@@ -9,7 +9,7 @@ This guide explains how to configure MinIO Server with TLS certificates on Linux
 
 ## <a name="install-minio-server"></a>1. Install MinIO Server
 
-Install MinIO Server using the instructions in the [MinIO Quickstart Guide](http://docs.min.io/docs/minio-quickstart-guide).
+Install MinIO Server using the instructions in the [MinIO Quickstart Guide](https://docs.min.io/docs/minio-quickstart-guide).
 
 ## <a name="use-an-existing-key-and-certificate-with-minio"></a>2. Use an Existing Key and Certificate with MinIO
 

go.mod

@@ -32,7 +32,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/jcmturner/gokrb5/v8 v8.4.2
 	github.com/json-iterator/go v1.1.11
-	github.com/klauspost/compress v1.12.2
+	github.com/klauspost/compress v1.13.1
 	github.com/klauspost/cpuid/v2 v2.0.4
 	github.com/klauspost/pgzip v1.2.5
 	github.com/klauspost/readahead v1.3.1
@@ -41,14 +41,14 @@ require (
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/miekg/dns v1.1.35
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.7.5-0.20210628223511-b51d5505f375
+	github.com/minio/console v0.7.6-0.20210715212137-d8840cc81026
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.0.13
-	github.com/minio/minio-go/v7 v7.0.13-0.20210706013812-337aa536abe2
+	github.com/minio/madmin-go v1.0.16-0.20210713203601-e2a5b7c4d8ae
+	github.com/minio/minio-go/v7 v7.0.13-0.20210715203016-9e713532886e
 	github.com/minio/parquet-go v1.0.0
-	github.com/minio/pkg v1.0.8
+	github.com/minio/pkg v1.0.10
 	github.com/minio/rpc v1.0.0
 	github.com/minio/selfupdate v0.3.1
 	github.com/minio/sha256-simd v1.0.0
@@ -80,6 +80,7 @@ require (
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	go.etcd.io/etcd/api/v3 v3.5.0-beta.4
 	go.etcd.io/etcd/client/v3 v3.5.0-beta.4
+	go.uber.org/atomic v1.7.0
 	go.uber.org/zap v1.16.1-0.20210329175301-c23abee72d19
 	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b
 	golang.org/x/sys v0.0.0-20210510120138-977fb7262007
@@ -88,5 +89,3 @@ require (
 	google.golang.org/api v0.31.0
 	gopkg.in/yaml.v2 v2.4.0
 )
-
-replace google.golang.org/grpc => google.golang.org/grpc v1.29.1

go.sum

@@ -1,4 +1,5 @@
 bazil.org/fuse v0.0.0-20180421153158-65cc252bf669/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
 cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts=
@@ -227,6 +228,7 @@ github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp
 github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
 github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ=
 github.com/cockroachdb/cockroach-go/v2 v2.0.3 h1:ZA346ACHIZctef6trOTwBAEvPVm1k0uLm/bb2Atc+S8=
 github.com/cockroachdb/cockroach-go/v2 v2.0.3/go.mod h1:hAuDgiVgDVkfirP9JnhXEfcXEPRKBpYdGz+l7mvYSzw=
@@ -318,7 +320,11 @@ github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkg
 github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
 github.com/emicklei/go-restful v2.9.5+incompatible h1:spTtZBk5DYEvbxMVutUuTyh1Ao2r4iyvLdACqsl/Ljk=
 github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/erikstmartin/go-testdb v0.0.0-20160219214506-8d10e4a1bae5/go.mod h1:a2zkGnVExMxdzMo3M0Hi/3sEU+cWnZpSni0O6/Yb/P0=
 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
@@ -526,6 +532,7 @@ github.com/goccy/go-json v0.4.8/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGF
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/flock v0.0.0-20190320160742-5135e617513b/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
 github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
+github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -873,8 +880,9 @@ github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A=
 github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
-github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8=
 github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ=
+github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
 github.com/klauspost/cpuid v0.0.0-20180405133222-e7e905edc00e/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
 github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
@@ -933,6 +941,7 @@ github.com/lib/pq v1.9.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
 github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM=
 github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4=
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
+github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ=
 github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
@@ -998,8 +1007,8 @@ github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos=
 github.com/minio/colorjson v1.0.1/go.mod h1:oPM3zQQY8Gz9NGtgvuBEjQ+gPZLKAGc7T+kjMlwtOgs=
-github.com/minio/console v0.7.5-0.20210628223511-b51d5505f375 h1:XBnZdMA8ehUQk3RZK+8R50K1BL0DW53ZqME6l+X0qBk=
-github.com/minio/console v0.7.5-0.20210628223511-b51d5505f375/go.mod h1:PrnkbZbBzQzWaBDspDt8oTL7KMdNVQROnltJyNrBuGg=
+github.com/minio/console v0.7.6-0.20210715212137-d8840cc81026 h1:g/J0SaD+xofku31QqkDjCx7//atjsbyBwH9WIAmQoo0=
+github.com/minio/console v0.7.6-0.20210715212137-d8840cc81026/go.mod h1:ZiDruKgGwP/tSWJxeKAUo8Q3vxaDRVW7fusuAugamX4=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf h1:wylCc/PdvdTIqYqVNEU9LJAZBanvfGY1TwTnjM3zQaA=
@@ -1013,8 +1022,8 @@ github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q=
 github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA=
 github.com/minio/madmin-go v1.0.6/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
 github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
-github.com/minio/madmin-go v1.0.13 h1:g0KIqo9zCt6wEG6koWZr7tlWFFozIXW2NRCQx9EsCrw=
-github.com/minio/madmin-go v1.0.13/go.mod h1:4nl9hvLWFnwCjkLfZSsZXEHgDODa2XSG6xGlIZyQ2oA=
+github.com/minio/madmin-go v1.0.16-0.20210713203601-e2a5b7c4d8ae h1:eRwOOnwEoqHL8CLQR69obOxYYshQoQ3470+qgD9psJw=
+github.com/minio/madmin-go v1.0.16-0.20210713203601-e2a5b7c4d8ae/go.mod h1:4nl9hvLWFnwCjkLfZSsZXEHgDODa2XSG6xGlIZyQ2oA=
 github.com/minio/mc v0.0.0-20210626002108-cebf3318546f h1:hyFvo5hSFw2K417YvDr/vAKlgCG69uTuhZW/5LNdL0U=
 github.com/minio/mc v0.0.0-20210626002108-cebf3318546f/go.mod h1:tuaonkPjVApCXkbtKENHBtsqUf7YTV33qmFrC+Pgp5g=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@@ -1023,9 +1032,8 @@ github.com/minio/md5-simd v1.1.1/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77Z
 github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
 github.com/minio/minio-go/v7 v7.0.11-0.20210607181445-e162fdb8e584/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA=
-github.com/minio/minio-go/v7 v7.0.12-0.20210617160455-b7103728fb87/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
-github.com/minio/minio-go/v7 v7.0.13-0.20210706013812-337aa536abe2 h1:Mleb9L1AnLrlHzrvb2eKjRflzLb5P3q+nS3wS4tci/Q=
-github.com/minio/minio-go/v7 v7.0.13-0.20210706013812-337aa536abe2/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
+github.com/minio/minio-go/v7 v7.0.13-0.20210715203016-9e713532886e h1:aVnxKPpUI1gVeEf9vC+QEt8OxMXiiNMeUWcrBM62oDU=
+github.com/minio/minio-go/v7 v7.0.13-0.20210715203016-9e713532886e/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
 github.com/minio/operator v0.0.0-20210616045941-65f31f5f78ae h1:GONmqbjCi/KTEc1CGujnS/m1qeJeghcQ8dUBLh19qQo=
 github.com/minio/operator v0.0.0-20210616045941-65f31f5f78ae/go.mod h1:8/mIXK+CFdL6VqyxRn1SwD+PEX0jsN8uqjoadaw/Np0=
 github.com/minio/operator/logsearchapi v0.0.0-20210604224119-7e256f98cf90 h1:Qu6j6oE7+QNuq7Kr2DLyVYq3fqMdqFd/T8NAeNp47og=
@@ -1034,8 +1042,9 @@ github.com/minio/parquet-go v1.0.0 h1:fcWsEvub04Nsl/4hiRBDWlbqd6jhacQieV07a+nhiI
 github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTacxxk/oHA=
 github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
-github.com/minio/pkg v1.0.8 h1:lWQwHSeYlvnRoPpO+wS0I4mL6c00ABxBgbGjSmjwOi4=
 github.com/minio/pkg v1.0.8/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
+github.com/minio/pkg v1.0.10 h1:fohpAm/0ttQFf4BzmzH5r6A9JUIfg63AyGCPM0f9/9U=
+github.com/minio/pkg v1.0.10/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
 github.com/minio/rpc v1.0.0 h1:tJCHyLfQF6k6HlMQFpKy2FO/7lc2WP8gLDGMZp18E70=
 github.com/minio/rpc v1.0.0/go.mod h1:b9xqF7J0xeMXr0cM4pnBlP7Te7PDsG5JrRxl5dG6Ldk=
 github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=
@@ -1541,6 +1550,7 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
@@ -1646,6 +1656,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180816055513-1c9583448a9c/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1753,11 +1764,13 @@ golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiT
 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181117154741-2ddaf7f79a09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190110163146-51295c7ec13a/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190125232054-d66bd3c5d5a6/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190206041539-40960b6deb8e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190221204921-83362c3779f5/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
@@ -1928,8 +1941,29 @@ google.golang.org/genproto v0.0.0-20200831141814-d751682dd103/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200901141002-b3bf27a9dbd1/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8=
 google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/grpc v1.29.1 h1:EC2SB8S04d2r73uptxphDSUG+kTKVgjRPF+N3xpxRB4=
+google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
 google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.37.0 h1:uSZWeQJX5j11bIQ4AJoj+McDBo29cY1MCoC1wO3ts+c=
+google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -2003,6 +2037,7 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C
 gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/config/config.go

@@ -74,6 +74,7 @@ const (
 	CompressionSubSys    = "compression"
 	LoggerWebhookSubSys  = "logger_webhook"
 	AuditWebhookSubSys   = "audit_webhook"
+	AuditKafkaSubSys     = "audit_kafka"
 	HealSubSys           = "heal"
 	ScannerSubSys        = "scanner"
 	CrawlerSubSys        = "crawler"
@@ -108,6 +109,7 @@ var SubSystems = set.CreateStringSet(
 	CompressionSubSys,
 	LoggerWebhookSubSys,
 	AuditWebhookSubSys,
+	AuditKafkaSubSys,
 	PolicyOPASubSys,
 	IdentityLDAPSubSys,
 	IdentityOpenIDSubSys,
@@ -247,6 +249,23 @@ func (kvs KVS) String() string {
 	return s.String()
 }
 
+// Merge environment values with on disk KVS, environment values overrides
+// anything on the disk.
+func Merge(cfgKVS map[string]KVS, envname string, defaultKVS KVS) map[string]KVS {
+	newCfgKVS := make(map[string]KVS)
+	for _, e := range env.List(envname) {
+		tgt := strings.TrimPrefix(e, envname+Default)
+		if tgt == envname {
+			tgt = Default
+		}
+		newCfgKVS[tgt] = defaultKVS
+	}
+	for tgt, kv := range cfgKVS {
+		newCfgKVS[tgt] = kv
+	}
+	return newCfgKVS
+}
+
 // Set sets a value, if not sets a default value.
 func (kvs *KVS) Set(key, value string) {
 	for i, kv := range *kvs {
@@ -355,9 +374,11 @@ func (c Config) RedactSensitiveInfo() Config {
 				}
 			}
 		}
-		nc[configName] = configVals
 	}
 
+	// Remove the server credentials altogether
+	nc.DelKVS(CredentialsSubSys)
+
 	return nc
 }
 

internal/config/constants.go

@@ -37,7 +37,8 @@ const (
 	EnvArgs       = "MINIO_ARGS"
 	EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT"
 
-	EnvRootDiskThresholdSize = "MINIO_ROOTDISK_THRESHOLD_SIZE"
+	EnvMinIOBrowserRedirectURL = "MINIO_BROWSER_REDIRECT_URL"
+	EnvRootDiskThresholdSize   = "MINIO_ROOTDISK_THRESHOLD_SIZE"
 
 	EnvUpdate = "MINIO_UPDATE"
 

internal/config/identity/openid/help.go

@@ -62,6 +62,24 @@ var (
 			Optional:    true,
 			Type:        "csv",
 		},
+		config.HelpKV{
+			Key:         Vendor,
+			Description: `Specify vendor type for vendor specific behavior to checking validity of temporary credentials and service accounts on MinIO`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         KeyCloakRealm,
+			Description: `Specify Keycloak 'realm' name, only honored if vendor was set to 'keycloak' as value, if no realm is specified 'master' is default`,
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         KeyCloakAdminURL,
+			Description: `Specify Keycloak 'admin' REST API endpoint e.g. http://localhost:8080/auth/admin/`,
+			Optional:    true,
+			Type:        "string",
+		},
 		config.HelpKV{
 			Key:         config.Comment,
 			Description: config.DefaultComment,

internal/config/identity/openid/jwt.go

@@ -32,6 +32,7 @@ import (
 	jwtgo "github.com/golang-jwt/jwt"
 	"github.com/minio/minio/internal/auth"
 	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/config/identity/openid/provider"
 	"github.com/minio/pkg/env"
 	iampolicy "github.com/minio/pkg/iam/policy"
 	xnet "github.com/minio/pkg/net"
@@ -40,7 +41,10 @@ import (
 // Config - OpenID Config
 // RSA authentication target arguments
 type Config struct {
-	JWKS struct {
+	*sync.RWMutex
+
+	Enabled bool `json:"enabled"`
+	JWKS    struct {
 		URL *xnet.URL `json:"url"`
 	} `json:"jwks"`
 	URL          *xnet.URL `json:"url,omitempty"`
@@ -50,16 +54,76 @@ type Config struct {
 	DiscoveryDoc DiscoveryDoc
 	ClientID     string
 	ClientSecret string
-	publicKeys   map[string]crypto.PublicKey
-	transport    *http.Transport
-	closeRespFn  func(io.ReadCloser)
-	mutex        *sync.Mutex
+
+	provider    provider.Provider
+	publicKeys  map[string]crypto.PublicKey
+	transport   *http.Transport
+	closeRespFn func(io.ReadCloser)
+}
+
+// LookupUser lookup userid for the provider
+func (r Config) LookupUser(userid string) (provider.User, error) {
+	if r.provider != nil {
+		user, err := r.provider.LookupUser(userid)
+		if err != nil && err != provider.ErrAccessTokenExpired {
+			return user, err
+		}
+		if err == provider.ErrAccessTokenExpired {
+			if err = r.provider.LoginWithClientID(r.ClientID, r.ClientSecret); err != nil {
+				return user, err
+			}
+			user, err = r.provider.LookupUser(userid)
+		}
+		return user, err
+	}
+	// Without any specific logic for a provider, all accounts
+	// are always enabled.
+	return provider.User{ID: userid, Enabled: true}, nil
+}
+
+const (
+	keyCloakVendor = "keycloak"
+)
+
+// InitializeProvider initializes if any additional vendor specific
+// information was provided, initialization will return an error
+// initial login fails.
+func (r Config) InitializeProvider(kvs config.KVS) error {
+	vendor := env.Get(EnvIdentityOpenIDVendor, kvs.Get(Vendor))
+	if vendor == "" {
+		return nil
+	}
+	switch vendor {
+	case keyCloakVendor:
+		adminURL := env.Get(EnvIdentityOpenIDKeyCloakAdminURL, kvs.Get(KeyCloakAdminURL))
+		realm := env.Get(EnvIdentityOpenIDKeyCloakRealm, kvs.Get(KeyCloakRealm))
+		return r.InitializeKeycloakProvider(adminURL, realm)
+	default:
+		return fmt.Errorf("Unsupport vendor %s", keyCloakVendor)
+	}
+}
+
+// ProviderEnabled returns true if any vendor specific provider is enabled.
+func (r Config) ProviderEnabled() bool {
+	return r.Enabled && r.provider != nil
+}
+
+// InitializeKeycloakProvider - initializes keycloak provider
+func (r *Config) InitializeKeycloakProvider(adminURL, realm string) error {
+	var err error
+	r.provider, err = provider.KeyCloak(
+		provider.WithAdminURL(adminURL),
+		provider.WithOpenIDConfig(provider.DiscoveryDoc(r.DiscoveryDoc)),
+		provider.WithTransport(r.transport),
+		provider.WithRealm(realm),
+	)
+	return err
 }
 
 // PopulatePublicKey - populates a new publickey from the JWKS URL.
 func (r *Config) PopulatePublicKey() error {
-	r.mutex.Lock()
-	defer r.mutex.Unlock()
+	r.Lock()
+	defer r.Unlock()
 
 	if r.JWKS.URL == nil || r.JWKS.URL.String() == "" {
 		return nil
@@ -115,11 +179,6 @@ func (r *Config) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-// JWT - rs client grants provider details.
-type JWT struct {
-	Config
-}
-
 // GetDefaultExpiration - returns the expiration seconds expected.
 func GetDefaultExpiration(dsecs string) (time.Duration, error) {
 	defaultExpiryDuration := time.Duration(60) * time.Minute // Defaults to 1hr.
@@ -131,8 +190,8 @@ func GetDefaultExpiration(dsecs string) (time.Duration, error) {
 
 		// The duration, in seconds, of the role session.
 		// The value can range from 900 seconds (15 minutes)
-		// up to 7 days.
-		if expirySecs < 900 || expirySecs > 604800 {
+		// up to 365 days.
+		if expirySecs < 900 || expirySecs > 31536000 {
 			return 0, auth.ErrInvalidDuration
 		}
 
@@ -175,7 +234,7 @@ func updateClaimsExpiry(dsecs string, claims map[string]interface{}) error {
 }
 
 // Validate - validates the access token.
-func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
+func (r *Config) Validate(token, dsecs string) (map[string]interface{}, error) {
 	jp := new(jwtgo.Parser)
 	jp.ValidMethods = []string{
 		"RS256", "RS384", "RS512", "ES256", "ES384", "ES512",
@@ -187,7 +246,9 @@ func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
 		if !ok {
 			return nil, fmt.Errorf("Invalid kid value %v", jwtToken.Header["kid"])
 		}
-		return p.publicKeys[kid], nil
+		r.RLock()
+		defer r.RUnlock()
+		return r.publicKeys[kid], nil
 	}
 
 	var claims jwtgo.MapClaims
@@ -195,7 +256,7 @@ func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
 	if err != nil {
 		// Re-populate the public key in-case the JWKS
 		// pubkeys are refreshed
-		if err = p.PopulatePublicKey(); err != nil {
+		if err = r.PopulatePublicKey(); err != nil {
 			return nil, err
 		}
 		jwtToken, err = jwtgo.ParseWithClaims(token, &claims, keyFuncCallback)
@@ -216,7 +277,7 @@ func (p *JWT) Validate(token, dsecs string) (map[string]interface{}, error) {
 }
 
 // ID returns the provider name and authentication type.
-func (p *JWT) ID() ID {
+func (Config) ID() ID {
 	return "jwt"
 }
 
@@ -228,9 +289,15 @@ const (
 	ClaimPrefix  = "claim_prefix"
 	ClientID     = "client_id"
 	ClientSecret = "client_secret"
+	Vendor       = "vendor"
 	Scopes       = "scopes"
 	RedirectURI  = "redirect_uri"
 
+	// Vendor specific ENV only enabled if the Vendor matches == "vendor"
+	KeyCloakRealm    = "keycloak_realm"
+	KeyCloakAdminURL = "keycloak_admin_url"
+
+	EnvIdentityOpenIDVendor       = "MINIO_IDENTITY_OPENID_VENDOR"
 	EnvIdentityOpenIDClientID     = "MINIO_IDENTITY_OPENID_CLIENT_ID"
 	EnvIdentityOpenIDClientSecret = "MINIO_IDENTITY_OPENID_CLIENT_SECRET"
 	EnvIdentityOpenIDJWKSURL      = "MINIO_IDENTITY_OPENID_JWKS_URL"
@@ -239,6 +306,10 @@ const (
 	EnvIdentityOpenIDClaimPrefix  = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
 	EnvIdentityOpenIDRedirectURI  = "MINIO_IDENTITY_OPENID_REDIRECT_URI"
 	EnvIdentityOpenIDScopes       = "MINIO_IDENTITY_OPENID_SCOPES"
+
+	// Vendor specific ENVs only enabled if the Vendor matches == "vendor"
+	EnvIdentityOpenIDKeyCloakRealm    = "MINIO_IDENTITY_OPENID_KEYCLOAK_REALM"
+	EnvIdentityOpenIDKeyCloakAdminURL = "MINIO_IDENTITY_OPENID_KEYCLOAK_ADMIN_URL"
 )
 
 // DiscoveryDoc - parses the output from openid-configuration
@@ -339,6 +410,7 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 	}
 
 	c = Config{
+		RWMutex:      &sync.RWMutex{},
 		ClaimName:    env.Get(EnvIdentityOpenIDClaimName, kvs.Get(ClaimName)),
 		ClaimPrefix:  env.Get(EnvIdentityOpenIDClaimPrefix, kvs.Get(ClaimPrefix)),
 		RedirectURI:  env.Get(EnvIdentityOpenIDRedirectURI, kvs.Get(RedirectURI)),
@@ -347,7 +419,6 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 		ClientSecret: env.Get(EnvIdentityOpenIDClientSecret, kvs.Get(ClientSecret)),
 		transport:    transport,
 		closeRespFn:  closeRespFn,
-		mutex:        &sync.Mutex{}, // allocate for copying
 	}
 
 	configURL := env.Get(EnvIdentityOpenIDURL, kvs.Get(ConfigURL))
@@ -397,10 +468,11 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 		return c, err
 	}
 
+	if err = c.InitializeProvider(kvs); err != nil {
+		return c, err
+	}
+
+	c.Enabled = true
+
 	return c, nil
 }
-
-// NewJWT - initialize new jwt authenticator.
-func NewJWT(c Config) *JWT {
-	return &JWT{c}
-}

internal/config/identity/openid/jwt_test.go

@@ -90,16 +90,17 @@ func TestJWTAzureFail(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cfg := Config{}
-	cfg.mutex = &sync.Mutex{}
+	cfg := Config{
+		RWMutex: &sync.RWMutex{},
+		Enabled: true,
+	}
 	cfg.JWKS.URL = u1
 	cfg.publicKeys = keys
-	jwt := NewJWT(cfg)
-	if jwt.ID() != "jwt" {
-		t.Fatalf("Uexpected id %s for the validator", jwt.ID())
+	if cfg.ID() != "jwt" {
+		t.Fatalf("Unexpected id %s for the validator", cfg.ID())
 	}
 
-	if _, err := jwt.Validate(jwtToken, ""); err == nil {
+	if _, err := cfg.Validate(jwtToken, ""); err == nil {
 		// Azure should fail due to non OIDC compliant JWT
 		// generated by Azure AD
 		t.Fatal(err)
@@ -138,13 +139,14 @@ func TestJWT(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cfg := Config{}
-	cfg.mutex = &sync.Mutex{}
+	cfg := Config{
+		RWMutex: &sync.RWMutex{},
+		Enabled: true,
+	}
 	cfg.JWKS.URL = u1
 	cfg.publicKeys = keys
-	jwt := NewJWT(cfg)
-	if jwt.ID() != "jwt" {
-		t.Fatalf("Uexpected id %s for the validator", jwt.ID())
+	if cfg.ID() != "jwt" {
+		t.Fatalf("Unexpected id %s for the validator", cfg.ID())
 	}
 
 	u, err := url.Parse("http://localhost:8443/?Token=invalid")
@@ -152,7 +154,7 @@ func TestJWT(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := jwt.Validate(u.Query().Get("Token"), ""); err == nil {
+	if _, err := cfg.Validate(u.Query().Get("Token"), ""); err == nil {
 		t.Fatal(err)
 	}
 }
@@ -172,7 +174,7 @@ func TestDefaultExpiryDuration(t *testing.T) {
 			expectErr: true,
 		},
 		{
-			reqURL:    "http://localhost:8443/?DurationSeconds=604801",
+			reqURL:    "http://localhost:8443/?DurationSeconds=31536001",
 			expectErr: true,
 		},
 		{

internal/config/identity/openid/provider/keycloak.go

@@ -0,0 +1,184 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package provider
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/url"
+	"strings"
+	"sync"
+)
+
+// Token - parses the output from IDP access token.
+type Token struct {
+	AccessToken string `json:"access_token"`
+	Expiry      int    `json:"expires_in"`
+}
+
+// KeycloakProvider implements Provider interface for KeyCloak Identity Provider.
+type KeycloakProvider struct {
+	sync.Mutex
+
+	oeConfig DiscoveryDoc
+	client   http.Client
+	adminURL string
+	realm    string
+
+	// internal value refreshed
+	accessToken Token
+}
+
+// LoginWithUser authenticates username/password, not needed for Keycloak
+func (k *KeycloakProvider) LoginWithUser(username, password string) error {
+	return ErrNotImplemented
+}
+
+// LoginWithClientID is implemented by Keycloak service account support
+func (k *KeycloakProvider) LoginWithClientID(clientID, clientSecret string) error {
+	values := url.Values{}
+	values.Set("client_id", clientID)
+	values.Set("client_secret", clientSecret)
+	values.Set("grant_type", "client_credentials")
+
+	req, err := http.NewRequest(http.MethodPost, k.oeConfig.TokenEndpoint, strings.NewReader(values.Encode()))
+	if err != nil {
+		return err
+	}
+	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
+
+	resp, err := k.client.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	var accessToken Token
+	if err = json.NewDecoder(resp.Body).Decode(&accessToken); err != nil {
+		return err
+	}
+
+	k.Lock()
+	k.accessToken = accessToken
+	k.Unlock()
+	return nil
+}
+
+// LookupUser lookup user by their userid.
+func (k *KeycloakProvider) LookupUser(userid string) (User, error) {
+	lookupUserID := k.adminURL + "/realms" + k.realm + "/users/" + userid
+	req, err := http.NewRequest(http.MethodGet, lookupUserID, nil)
+	if err != nil {
+		return User{}, err
+	}
+	k.Lock()
+	accessToken := k.accessToken
+	k.Unlock()
+	if accessToken.AccessToken == "" {
+		return User{}, ErrAccessTokenExpired
+	}
+	req.Header.Set("Authorization", "Bearer "+accessToken.AccessToken)
+	resp, err := k.client.Do(req)
+	if err != nil {
+		return User{}, err
+	}
+	defer resp.Body.Close()
+	switch resp.StatusCode {
+	case http.StatusOK, http.StatusPartialContent:
+		var u User
+		if err = json.NewDecoder(resp.Body).Decode(&u); err != nil {
+			return User{}, err
+		}
+		return u, nil
+	case http.StatusNotFound:
+		return User{
+			ID:      userid,
+			Enabled: false,
+		}, nil
+	case http.StatusUnauthorized:
+		return User{}, ErrAccessTokenExpired
+	}
+	return User{}, fmt.Errorf("Unable to lookup %s - keycloak user lookup returned %v", userid, resp.Status)
+}
+
+// Option is a function type that accepts a pointer Target
+type Option func(*KeycloakProvider)
+
+// WithTransport provide custom transport
+func WithTransport(transport *http.Transport) Option {
+	return func(p *KeycloakProvider) {
+		p.client = http.Client{
+			Transport: transport,
+		}
+	}
+}
+
+// WithOpenIDConfig provide OpenID Endpoint configuration discovery document
+func WithOpenIDConfig(oeConfig DiscoveryDoc) Option {
+	return func(p *KeycloakProvider) {
+		p.oeConfig = oeConfig
+	}
+}
+
+// WithAdminURL provide admin URL configuration for Keycloak
+func WithAdminURL(url string) Option {
+	return func(p *KeycloakProvider) {
+		p.adminURL = url
+	}
+}
+
+// WithRealm provide realm configuration for Keycloak
+func WithRealm(realm string) Option {
+	return func(p *KeycloakProvider) {
+		p.realm = realm
+	}
+}
+
+// KeyCloak initializes a new keycloak provider
+func KeyCloak(opts ...Option) (Provider, error) {
+	p := &KeycloakProvider{}
+
+	for _, opt := range opts {
+		opt(p)
+	}
+
+	if p.adminURL == "" {
+		return nil, errors.New("Admin URL cannot be empty")
+	}
+
+	_, err := url.Parse(p.adminURL)
+	if err != nil {
+		return nil, fmt.Errorf("Unable to parse the adminURL %s: %w", p.adminURL, err)
+	}
+
+	if p.client.Transport == nil {
+		p.client.Transport = http.DefaultTransport
+	}
+
+	if p.oeConfig.TokenEndpoint == "" {
+		return nil, errors.New("missing OpenID token endpoint")
+	}
+
+	if p.realm == "" {
+		p.realm = "master" // default realm
+	}
+
+	return p, nil
+}

internal/config/identity/openid/provider/provider.go

@@ -0,0 +1,59 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package provider
+
+import "errors"
+
+// DiscoveryDoc - parses the output from openid-configuration
+// for example https://accounts.google.com/.well-known/openid-configuration
+type DiscoveryDoc struct {
+	Issuer                           string   `json:"issuer,omitempty"`
+	AuthEndpoint                     string   `json:"authorization_endpoint,omitempty"`
+	TokenEndpoint                    string   `json:"token_endpoint,omitempty"`
+	UserInfoEndpoint                 string   `json:"userinfo_endpoint,omitempty"`
+	RevocationEndpoint               string   `json:"revocation_endpoint,omitempty"`
+	JwksURI                          string   `json:"jwks_uri,omitempty"`
+	ResponseTypesSupported           []string `json:"response_types_supported,omitempty"`
+	SubjectTypesSupported            []string `json:"subject_types_supported,omitempty"`
+	IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported,omitempty"`
+	ScopesSupported                  []string `json:"scopes_supported,omitempty"`
+	TokenEndpointAuthMethods         []string `json:"token_endpoint_auth_methods_supported,omitempty"`
+	ClaimsSupported                  []string `json:"claims_supported,omitempty"`
+	CodeChallengeMethodsSupported    []string `json:"code_challenge_methods_supported,omitempty"`
+}
+
+// User represents information about user.
+type User struct {
+	Name    string `json:"username"`
+	ID      string `json:"id"`
+	Enabled bool   `json:"enabled"`
+}
+
+// Standard errors.
+var (
+	ErrNotImplemented     = errors.New("function not implemented")
+	ErrAccessTokenExpired = errors.New("access_token expired or unauthorized")
+)
+
+// Provider implements indentity provider specific admin operations, such as
+// looking up users, fetching additional attributes etc.
+type Provider interface {
+	LoginWithUser(username, password string) error
+	LoginWithClientID(clientID, clientSecret string) error
+	LookupUser(userid string) (User, error)
+}

internal/config/identity/openid/validators_test.go

@@ -95,7 +95,7 @@ func TestValidators(t *testing.T) {
 
 	cfg := Config{}
 	cfg.JWKS.URL = u
-	if err = vrs.Add(NewJWT(cfg)); err != nil {
+	if err = vrs.Add(&cfg); err != nil {
 		t.Fatal(err)
 	}
 

internal/config/notify/parse.go

@@ -410,21 +410,6 @@ func checkValidNotificationKeys(cfg config.Config) error {
 	return nil
 }
 
-func mergeTargets(cfgTargets map[string]config.KVS, envname string, defaultKVS config.KVS) map[string]config.KVS {
-	newCfgTargets := make(map[string]config.KVS)
-	for _, e := range env.List(envname) {
-		tgt := strings.TrimPrefix(e, envname+config.Default)
-		if tgt == envname {
-			tgt = config.Default
-		}
-		newCfgTargets[tgt] = defaultKVS
-	}
-	for tgt, kv := range cfgTargets {
-		newCfgTargets[tgt] = kv
-	}
-	return newCfgTargets
-}
-
 // DefaultKakfaKVS - default KV for kafka target
 var (
 	DefaultKafkaKVS = config.KVS{
@@ -494,7 +479,7 @@ var (
 // GetNotifyKafka - returns a map of registered notification 'kafka' targets
 func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs, error) {
 	kafkaTargets := make(map[string]target.KafkaArgs)
-	for k, kv := range mergeTargets(kafkaKVS, target.EnvKafkaEnable, DefaultKafkaKVS) {
+	for k, kv := range config.Merge(kafkaKVS, target.EnvKafkaEnable, DefaultKafkaKVS) {
 		enableEnv := target.EnvKafkaEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -675,7 +660,7 @@ var (
 // GetNotifyMQTT - returns a map of registered notification 'mqtt' targets
 func GetNotifyMQTT(mqttKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[string]target.MQTTArgs, error) {
 	mqttTargets := make(map[string]target.MQTTArgs)
-	for k, kv := range mergeTargets(mqttKVS, target.EnvMQTTEnable, DefaultMQTTKVS) {
+	for k, kv := range config.Merge(mqttKVS, target.EnvMQTTEnable, DefaultMQTTKVS) {
 		enableEnv := target.EnvMQTTEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -818,7 +803,7 @@ var (
 // GetNotifyMySQL - returns a map of registered notification 'mysql' targets
 func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs, error) {
 	mysqlTargets := make(map[string]target.MySQLArgs)
-	for k, kv := range mergeTargets(mysqlKVS, target.EnvMySQLEnable, DefaultMySQLKVS) {
+	for k, kv := range config.Merge(mysqlKVS, target.EnvMySQLEnable, DefaultMySQLKVS) {
 		enableEnv := target.EnvMySQLEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -969,7 +954,7 @@ var (
 // GetNotifyNATS - returns a map of registered notification 'nats' targets
 func GetNotifyNATS(natsKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[string]target.NATSArgs, error) {
 	natsTargets := make(map[string]target.NATSArgs)
-	for k, kv := range mergeTargets(natsKVS, target.EnvNATSEnable, DefaultNATSKVS) {
+	for k, kv := range config.Merge(natsKVS, target.EnvNATSEnable, DefaultNATSKVS) {
 		enableEnv := target.EnvNATSEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -1157,7 +1142,7 @@ var (
 // GetNotifyNSQ - returns a map of registered notification 'nsq' targets
 func GetNotifyNSQ(nsqKVS map[string]config.KVS) (map[string]target.NSQArgs, error) {
 	nsqTargets := make(map[string]target.NSQArgs)
-	for k, kv := range mergeTargets(nsqKVS, target.EnvNSQEnable, DefaultNSQKVS) {
+	for k, kv := range config.Merge(nsqKVS, target.EnvNSQEnable, DefaultNSQKVS) {
 		enableEnv := target.EnvNSQEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -1262,7 +1247,7 @@ var (
 // GetNotifyPostgres - returns a map of registered notification 'postgres' targets
 func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.PostgreSQLArgs, error) {
 	psqlTargets := make(map[string]target.PostgreSQLArgs)
-	for k, kv := range mergeTargets(postgresKVS, target.EnvPostgresEnable, DefaultPostgresKVS) {
+	for k, kv := range config.Merge(postgresKVS, target.EnvPostgresEnable, DefaultPostgresKVS) {
 		enableEnv := target.EnvPostgresEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -1371,7 +1356,7 @@ var (
 // GetNotifyRedis - returns a map of registered notification 'redis' targets
 func GetNotifyRedis(redisKVS map[string]config.KVS) (map[string]target.RedisArgs, error) {
 	redisTargets := make(map[string]target.RedisArgs)
-	for k, kv := range mergeTargets(redisKVS, target.EnvRedisEnable, DefaultRedisKVS) {
+	for k, kv := range config.Merge(redisKVS, target.EnvRedisEnable, DefaultRedisKVS) {
 		enableEnv := target.EnvRedisEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -1472,7 +1457,7 @@ var (
 func GetNotifyWebhook(webhookKVS map[string]config.KVS, transport *http.Transport) (
 	map[string]target.WebhookArgs, error) {
 	webhookTargets := make(map[string]target.WebhookArgs)
-	for k, kv := range mergeTargets(webhookKVS, target.EnvWebhookEnable, DefaultWebhookKVS) {
+	for k, kv := range config.Merge(webhookKVS, target.EnvWebhookEnable, DefaultWebhookKVS) {
 		enableEnv := target.EnvWebhookEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -1577,7 +1562,7 @@ var (
 // GetNotifyES - returns a map of registered notification 'elasticsearch' targets
 func GetNotifyES(esKVS map[string]config.KVS, transport *http.Transport) (map[string]target.ElasticsearchArgs, error) {
 	esTargets := make(map[string]target.ElasticsearchArgs)
-	for k, kv := range mergeTargets(esKVS, target.EnvElasticEnable, DefaultESKVS) {
+	for k, kv := range config.Merge(esKVS, target.EnvElasticEnable, DefaultESKVS) {
 		enableEnv := target.EnvElasticEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k
@@ -1719,7 +1704,7 @@ var (
 // GetNotifyAMQP - returns a map of registered notification 'amqp' targets
 func GetNotifyAMQP(amqpKVS map[string]config.KVS) (map[string]target.AMQPArgs, error) {
 	amqpTargets := make(map[string]target.AMQPArgs)
-	for k, kv := range mergeTargets(amqpKVS, target.EnvAMQPEnable, DefaultAMQPKVS) {
+	for k, kv := range config.Merge(amqpKVS, target.EnvAMQPEnable, DefaultAMQPKVS) {
 		enableEnv := target.EnvAMQPEnable
 		if k != config.Default {
 			enableEnv = enableEnv + config.Default + k

internal/event/target/webhook.go

@@ -30,6 +30,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/minio/minio/internal/event"
@@ -163,7 +164,15 @@ func (target *WebhookTarget) send(eventData event.Event) error {
 		return err
 	}
 
-	if target.args.AuthToken != "" {
+	// Verify if the authToken already contains
+	// <Key> <Token> like format, if this is
+	// already present we can blindly use the
+	// authToken as is instead of adding 'Bearer'
+	tokens := strings.Fields(target.args.AuthToken)
+	switch len(tokens) {
+	case 2:
+		req.Header.Set("Authorization", target.args.AuthToken)
+	case 1:
 		req.Header.Set("Authorization", "Bearer "+target.args.AuthToken)
 	}
 

internal/logger/config.go

@@ -18,10 +18,16 @@
 package logger
 
 import (
+	"crypto/tls"
+	"strconv"
 	"strings"
 
-	"github.com/minio/minio/internal/config"
 	"github.com/minio/pkg/env"
+	xnet "github.com/minio/pkg/net"
+
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/logger/target/http"
+	"github.com/minio/minio/internal/logger/target/kafka"
 )
 
 // Console logger target
@@ -29,29 +35,26 @@ type Console struct {
 	Enabled bool `json:"enabled"`
 }
 
-// HTTP logger target
-type HTTP struct {
-	Enabled    bool   `json:"enabled"`
-	Endpoint   string `json:"endpoint"`
-	AuthToken  string `json:"authToken"`
-	ClientCert string `json:"clientCert"`
-	ClientKey  string `json:"clientKey"`
-}
-
-// Config console and http logger targets
-type Config struct {
-	Console Console         `json:"console"`
-	HTTP    map[string]HTTP `json:"http"`
-	Audit   map[string]HTTP `json:"audit"`
-}
-
-// HTTP endpoint logger
+// Audit/Logger constants
 const (
 	Endpoint   = "endpoint"
 	AuthToken  = "auth_token"
 	ClientCert = "client_cert"
 	ClientKey  = "client_key"
 
+	KafkaBrokers       = "brokers"
+	KafkaTopic         = "topic"
+	KafkaTLS           = "tls"
+	KafkaTLSSkipVerify = "tls_skip_verify"
+	KafkaTLSClientAuth = "tls_client_auth"
+	KafkaSASL          = "sasl"
+	KafkaSASLUsername  = "sasl_username"
+	KafkaSASLPassword  = "sasl_password"
+	KafkaSASLMechanism = "sasl_mechanism"
+	KafkaClientTLSCert = "client_tls_cert"
+	KafkaClientTLSKey  = "client_tls_key"
+	KafkaVersion       = "version"
+
 	EnvLoggerWebhookEnable    = "MINIO_LOGGER_WEBHOOK_ENABLE"
 	EnvLoggerWebhookEndpoint  = "MINIO_LOGGER_WEBHOOK_ENDPOINT"
 	EnvLoggerWebhookAuthToken = "MINIO_LOGGER_WEBHOOK_AUTH_TOKEN"
@@ -61,6 +64,20 @@ const (
 	EnvAuditWebhookAuthToken  = "MINIO_AUDIT_WEBHOOK_AUTH_TOKEN"
 	EnvAuditWebhookClientCert = "MINIO_AUDIT_WEBHOOK_CLIENT_CERT"
 	EnvAuditWebhookClientKey  = "MINIO_AUDIT_WEBHOOK_CLIENT_KEY"
+
+	EnvKafkaEnable        = "MINIO_AUDIT_KAFKA_ENABLE"
+	EnvKafkaBrokers       = "MINIO_AUDIT_KAFKA_BROKERS"
+	EnvKafkaTopic         = "MINIO_AUDIT_KAFKA_TOPIC"
+	EnvKafkaTLS           = "MINIO_AUDIT_KAFKA_TLS"
+	EnvKafkaTLSSkipVerify = "MINIO_AUDIT_KAFKA_TLS_SKIP_VERIFY"
+	EnvKafkaTLSClientAuth = "MINIO_AUDIT_KAFKA_TLS_CLIENT_AUTH"
+	EnvKafkaSASLEnable    = "MINIO_AUDIT_KAFKA_SASL"
+	EnvKafkaSASLUsername  = "MINIO_AUDIT_KAFKA_SASL_USERNAME"
+	EnvKafkaSASLPassword  = "MINIO_AUDIT_KAFKA_SASL_PASSWORD"
+	EnvKafkaSASLMechanism = "MINIO_AUDIT_KAFKA_SASL_MECHANISM"
+	EnvKafkaClientTLSCert = "MINIO_AUDIT_KAFKA_CLIENT_TLS_CERT"
+	EnvKafkaClientTLSKey  = "MINIO_AUDIT_KAFKA_CLIENT_TLS_KEY"
+	EnvKafkaVersion       = "MINIO_AUDIT_KAFKA_VERSION"
 )
 
 // Default KVS for loggerHTTP and loggerAuditHTTP
@@ -79,7 +96,8 @@ var (
 			Value: "",
 		},
 	}
-	DefaultAuditKVS = config.KVS{
+
+	DefaultAuditWebhookKVS = config.KVS{
 		config.KV{
 			Key:   config.Enable,
 			Value: config.EnableOff,
@@ -101,8 +119,71 @@ var (
 			Value: "",
 		},
 	}
+
+	DefaultAuditKafkaKVS = config.KVS{
+		config.KV{
+			Key:   config.Enable,
+			Value: config.EnableOff,
+		},
+		config.KV{
+			Key:   KafkaTopic,
+			Value: "",
+		},
+		config.KV{
+			Key:   KafkaBrokers,
+			Value: "",
+		},
+		config.KV{
+			Key:   KafkaSASLUsername,
+			Value: "",
+		},
+		config.KV{
+			Key:   KafkaSASLPassword,
+			Value: "",
+		},
+		config.KV{
+			Key:   KafkaSASLMechanism,
+			Value: "plain",
+		},
+		config.KV{
+			Key:   KafkaClientTLSCert,
+			Value: "",
+		},
+		config.KV{
+			Key:   KafkaClientTLSKey,
+			Value: "",
+		},
+		config.KV{
+			Key:   KafkaTLSClientAuth,
+			Value: "0",
+		},
+		config.KV{
+			Key:   KafkaSASL,
+			Value: config.EnableOff,
+		},
+		config.KV{
+			Key:   KafkaTLS,
+			Value: config.EnableOff,
+		},
+		config.KV{
+			Key:   KafkaTLSSkipVerify,
+			Value: config.EnableOff,
+		},
+		config.KV{
+			Key:   KafkaVersion,
+			Value: "",
+		},
+	}
 )
 
+// Config console and http logger targets
+type Config struct {
+	Console      Console                 `json:"console"`
+	HTTP         map[string]http.Config  `json:"http"`
+	AuditWebhook map[string]http.Config  `json:"audit"`
+	AuditKafka   map[string]kafka.Config `json:"audit_kafka"`
+}
+
 // NewConfig - initialize new logger config.
 func NewConfig() Config {
 	cfg := Config{
@@ -110,18 +191,9 @@ func NewConfig() Config {
 		Console: Console{
 			Enabled: true,
 		},
-		HTTP:  make(map[string]HTTP),
-		Audit: make(map[string]HTTP),
-	}
-
-	// Create an example HTTP logger
-	cfg.HTTP[config.Default] = HTTP{
-		Endpoint: "https://username:password@example.com/api",
-	}
-
-	// Create an example Audit logger
-	cfg.Audit[config.Default] = HTTP{
-		Endpoint: "https://username:password@example.com/api/audit",
+		HTTP:         make(map[string]http.Config),
+		AuditWebhook: make(map[string]http.Config),
+		AuditKafka:   make(map[string]kafka.Config),
 	}
 
 	return cfg
@@ -150,7 +222,7 @@ func lookupLegacyConfig() (Config, error) {
 		if endpoint == "" {
 			continue
 		}
-		cfg.HTTP[target] = HTTP{
+		cfg.HTTP[target] = http.Config{
 			Enabled:  true,
 			Endpoint: endpoint,
 		}
@@ -176,7 +248,7 @@ func lookupLegacyConfig() (Config, error) {
 		if endpoint == "" {
 			continue
 		}
-		cfg.Audit[target] = HTTP{
+		cfg.AuditWebhook[target] = http.Config{
 			Enabled:  true,
 			Endpoint: endpoint,
 		}
@@ -186,6 +258,121 @@ func lookupLegacyConfig() (Config, error) {
 
 }
 
+// GetAuditKafka - returns a map of registered notification 'kafka' targets
+func GetAuditKafka(kafkaKVS map[string]config.KVS) (map[string]kafka.Config, error) {
+	kafkaTargets := make(map[string]kafka.Config)
+	for k, kv := range config.Merge(kafkaKVS, EnvKafkaEnable, DefaultAuditKafkaKVS) {
+		enableEnv := EnvKafkaEnable
+		if k != config.Default {
+			enableEnv = enableEnv + config.Default + k
+		}
+		enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable)))
+		if err != nil {
+			return nil, err
+		}
+		if !enabled {
+			continue
+		}
+		var brokers []xnet.Host
+		brokersEnv := EnvKafkaBrokers
+		if k != config.Default {
+			brokersEnv = brokersEnv + config.Default + k
+		}
+		kafkaBrokers := env.Get(brokersEnv, kv.Get(KafkaBrokers))
+		if len(kafkaBrokers) == 0 {
+			return nil, config.Errorf("kafka 'brokers' cannot be empty")
+		}
+		for _, s := range strings.Split(kafkaBrokers, config.ValueSeparator) {
+			var host *xnet.Host
+			host, err = xnet.ParseHost(s)
+			if err != nil {
+				break
+			}
+			brokers = append(brokers, *host)
+		}
+		if err != nil {
+			return nil, err
+		}
+
+		clientAuthEnv := EnvKafkaTLSClientAuth
+		if k != config.Default {
+			clientAuthEnv = clientAuthEnv + config.Default + k
+		}
+		clientAuth, err := strconv.Atoi(env.Get(clientAuthEnv, kv.Get(KafkaTLSClientAuth)))
+		if err != nil {
+			return nil, err
+		}
+
+		topicEnv := EnvKafkaTopic
+		if k != config.Default {
+			topicEnv = topicEnv + config.Default + k
+		}
+
+		versionEnv := EnvKafkaVersion
+		if k != config.Default {
+			versionEnv = versionEnv + config.Default + k
+		}
+
+		kafkaArgs := kafka.Config{
+			Enabled: enabled,
+			Brokers: brokers,
+			Topic:   env.Get(topicEnv, kv.Get(KafkaTopic)),
+			Version: env.Get(versionEnv, kv.Get(KafkaVersion)),
+		}
+
+		tlsEnableEnv := EnvKafkaTLS
+		if k != config.Default {
+			tlsEnableEnv = tlsEnableEnv + config.Default + k
+		}
+		tlsSkipVerifyEnv := EnvKafkaTLSSkipVerify
+		if k != config.Default {
+			tlsSkipVerifyEnv = tlsSkipVerifyEnv + config.Default + k
+		}
+
+		tlsClientTLSCertEnv := EnvKafkaClientTLSCert
+		if k != config.Default {
+			tlsClientTLSCertEnv = tlsClientTLSCertEnv + config.Default + k
+		}
+
+		tlsClientTLSKeyEnv := EnvKafkaClientTLSKey
+		if k != config.Default {
+			tlsClientTLSKeyEnv = tlsClientTLSKeyEnv + config.Default + k
+		}
+
+		kafkaArgs.TLS.Enable = env.Get(tlsEnableEnv, kv.Get(KafkaTLS)) == config.EnableOn
+		kafkaArgs.TLS.SkipVerify = env.Get(tlsSkipVerifyEnv, kv.Get(KafkaTLSSkipVerify)) == config.EnableOn
+		kafkaArgs.TLS.ClientAuth = tls.ClientAuthType(clientAuth)
+
+		kafkaArgs.TLS.ClientTLSCert = env.Get(tlsClientTLSCertEnv, kv.Get(KafkaClientTLSCert))
+		kafkaArgs.TLS.ClientTLSKey = env.Get(tlsClientTLSKeyEnv, kv.Get(KafkaClientTLSKey))
+
+		saslEnableEnv := EnvKafkaSASLEnable
+		if k != config.Default {
+			saslEnableEnv = saslEnableEnv + config.Default + k
+		}
+		saslUsernameEnv := EnvKafkaSASLUsername
+		if k != config.Default {
+			saslUsernameEnv = saslUsernameEnv + config.Default + k
+		}
+		saslPasswordEnv := EnvKafkaSASLPassword
+		if k != config.Default {
+			saslPasswordEnv = saslPasswordEnv + config.Default + k
+		}
+		saslMechanismEnv := EnvKafkaSASLMechanism
+		if k != config.Default {
+			saslMechanismEnv = saslMechanismEnv + config.Default + k
+		}
+		kafkaArgs.SASL.Enable = env.Get(saslEnableEnv, kv.Get(KafkaSASL)) == config.EnableOn
+		kafkaArgs.SASL.User = env.Get(saslUsernameEnv, kv.Get(KafkaSASLUsername))
+		kafkaArgs.SASL.Password = env.Get(saslPasswordEnv, kv.Get(KafkaSASLPassword))
+		kafkaArgs.SASL.Mechanism = env.Get(saslMechanismEnv, kv.Get(KafkaSASLMechanism))
+
+		kafkaTargets[k] = kafkaArgs
+	}
+
+	return kafkaTargets, nil
+}
+
 // LookupConfig - lookup logger config, override with ENVs if set.
 func LookupConfig(scfg config.Config) (Config, error) {
 	// Lookup for legacy environment variables first
@@ -237,7 +424,7 @@ func LookupConfig(scfg config.Config) (Config, error) {
 		if target != config.Default {
 			authTokenEnv = EnvLoggerWebhookAuthToken + config.Default + target
 		}
-		cfg.HTTP[target] = HTTP{
+		cfg.HTTP[target] = http.Config{
 			Enabled:   true,
 			Endpoint:  env.Get(endpointEnv, ""),
 			AuthToken: env.Get(authTokenEnv, ""),
@@ -245,7 +432,7 @@ func LookupConfig(scfg config.Config) (Config, error) {
 	}
 
 	for _, target := range loggerAuditTargets {
-		if v, ok := cfg.Audit[target]; ok && v.Enabled {
+		if v, ok := cfg.AuditWebhook[target]; ok && v.Enabled {
 			// This target is already enabled using the
 			// legacy environment variables, ignore.
 			continue
@@ -278,7 +465,7 @@ func LookupConfig(scfg config.Config) (Config, error) {
 		if err != nil {
 			return cfg, err
 		}
-		cfg.Audit[target] = HTTP{
+		cfg.AuditWebhook[target] = http.Config{
 			Enabled:    true,
 			Endpoint:   env.Get(endpointEnv, ""),
 			AuthToken:  env.Get(authTokenEnv, ""),
@@ -308,7 +495,7 @@ func LookupConfig(scfg config.Config) (Config, error) {
 		if !enabled {
 			continue
 		}
-		cfg.HTTP[starget] = HTTP{
+		cfg.HTTP[starget] = http.Config{
 			Enabled:   true,
 			Endpoint:  kv.Get(Endpoint),
 			AuthToken: kv.Get(AuthToken),
@@ -316,7 +503,7 @@ func LookupConfig(scfg config.Config) (Config, error) {
 	}
 
 	for starget, kv := range scfg[config.AuditWebhookSubSys] {
-		if l, ok := cfg.Audit[starget]; ok && l.Enabled {
+		if l, ok := cfg.AuditWebhook[starget]; ok && l.Enabled {
 			// Ignore this audit config since another target
 			// with the same name is already loaded and enabled
 			// in the shell environment.
@@ -326,7 +513,7 @@ func LookupConfig(scfg config.Config) (Config, error) {
 		if starget != config.Default {
 			subSysTarget = config.AuditWebhookSubSys + config.SubSystemSeparator + starget
 		}
-		if err := config.CheckValidKeys(subSysTarget, kv, DefaultAuditKVS); err != nil {
+		if err := config.CheckValidKeys(subSysTarget, kv, DefaultAuditWebhookKVS); err != nil {
 			return cfg, err
 		}
 		enabled, err := config.ParseBool(kv.Get(config.Enable))
@@ -340,7 +527,7 @@ func LookupConfig(scfg config.Config) (Config, error) {
 		if err != nil {
 			return cfg, err
 		}
-		cfg.Audit[starget] = HTTP{
+		cfg.AuditWebhook[starget] = http.Config{
 			Enabled:    true,
 			Endpoint:   kv.Get(Endpoint),
 			AuthToken:  kv.Get(AuthToken),
@@ -349,5 +536,10 @@ func LookupConfig(scfg config.Config) (Config, error) {
 		}
 	}
 
+	cfg.AuditKafka, err = GetAuditKafka(scfg[config.AuditKafkaSubSys])
+	if err != nil {
+		return cfg, err
+	}
+
 	return cfg, nil
 }

internal/logger/help.go

@@ -45,7 +45,7 @@ var (
 		},
 	}
 
-	HelpAudit = config.HelpKVS{
+	HelpWebhook = config.HelpKVS{
 		config.HelpKV{
 			Key:         Endpoint,
 			Description: `HTTP(s) endpoint e.g. "http://localhost:8080/minio/logs/audit"`,
@@ -59,12 +59,6 @@ var (
 			Type:        "string",
 			Sensitive:   true,
 		},
-		config.HelpKV{
-			Key:         config.Comment,
-			Description: config.DefaultComment,
-			Optional:    true,
-			Type:        "sentence",
-		},
 		config.HelpKV{
 			Key:         ClientCert,
 			Description: "mTLS certificate for Audit Webhook authentication",
@@ -79,5 +73,95 @@ var (
 			Type:        "string",
 			Sensitive:   true,
 		},
+		config.HelpKV{
+			Key:         config.Comment,
+			Description: config.DefaultComment,
+			Optional:    true,
+			Type:        "sentence",
+		},
+	}
+
+	HelpKafka = config.HelpKVS{
+		config.HelpKV{
+			Key:         KafkaBrokers,
+			Description: "comma separated list of Kafka broker addresses",
+			Type:        "csv",
+		},
+		config.HelpKV{
+			Key:         KafkaTopic,
+			Description: "Kafka topic used for bucket notifications",
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         KafkaSASLUsername,
+			Description: "username for SASL/PLAIN or SASL/SCRAM authentication",
+			Optional:    true,
+			Type:        "string",
+			Sensitive:   true,
+		},
+		config.HelpKV{
+			Key:         KafkaSASLPassword,
+			Description: "password for SASL/PLAIN or SASL/SCRAM authentication",
+			Optional:    true,
+			Type:        "string",
+			Sensitive:   true,
+		},
+		config.HelpKV{
+			Key:         KafkaSASLMechanism,
+			Description: "sasl authentication mechanism, default 'plain'",
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         KafkaTLSClientAuth,
+			Description: "clientAuth determines the Kafka server's policy for TLS client auth",
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         KafkaSASL,
+			Description: "set to 'on' to enable SASL authentication",
+			Optional:    true,
+			Type:        "on|off",
+		},
+		config.HelpKV{
+			Key:         KafkaTLS,
+			Description: "set to 'on' to enable TLS",
+			Optional:    true,
+			Type:        "on|off",
+		},
+		config.HelpKV{
+			Key:         KafkaTLSSkipVerify,
+			Description: `trust server TLS without verification, defaults to "on" (verify)`,
+			Optional:    true,
+			Type:        "on|off",
+		},
+		config.HelpKV{
+			Key:         KafkaClientTLSCert,
+			Description: "path to client certificate for mTLS auth",
+			Optional:    true,
+			Type:        "path",
+			Sensitive:   true,
+		},
+		config.HelpKV{
+			Key:         KafkaClientTLSKey,
+			Description: "path to client key for mTLS auth",
+			Optional:    true,
+			Type:        "path",
+			Sensitive:   true,
+		},
+		config.HelpKV{
+			Key:         KafkaVersion,
+			Description: "specify the version of the Kafka cluster",
+			Optional:    true,
+			Type:        "string",
+		},
+		config.HelpKV{
+			Key:         config.Comment,
+			Description: config.DefaultComment,
+			Optional:    true,
+			Type:        "sentence",
+		},
 	}
 )

internal/logger/legacy.go

@@ -17,7 +17,10 @@
 
 package logger
 
-import "github.com/minio/minio/internal/config"
+import (
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/minio/internal/logger/target/http"
+)
 
 // Legacy envs
 const (
@@ -26,7 +29,7 @@ const (
 )
 
 // SetLoggerHTTPAudit - helper for migrating older config to newer KV format.
-func SetLoggerHTTPAudit(scfg config.Config, k string, args HTTP) {
+func SetLoggerHTTPAudit(scfg config.Config, k string, args http.Config) {
 	if !args.Enabled {
 		// Do not enable audit targets, if not enabled
 		return
@@ -48,7 +51,7 @@ func SetLoggerHTTPAudit(scfg config.Config, k string, args HTTP) {
 }
 
 // SetLoggerHTTP helper for migrating older config to newer KV format.
-func SetLoggerHTTP(scfg config.Config, k string, args HTTP) {
+func SetLoggerHTTP(scfg config.Config, k string, args http.Config) {
 	if !args.Enabled {
 		// Do not enable logger http targets, if not enabled
 		return

internal/logger/target/http/http.go

@@ -28,12 +28,26 @@ import (
 	"time"
 
 	xhttp "github.com/minio/minio/internal/http"
-	"github.com/minio/minio/internal/logger"
 )
 
 // Timeout for the webhook http call
 const webhookCallTimeout = 5 * time.Second
 
+// Config http logger target
+type Config struct {
+	Enabled    bool              `json:"enabled"`
+	Name       string            `json:"name"`
+	UserAgent  string            `json:"userAgent"`
+	Endpoint   string            `json:"endpoint"`
+	AuthToken  string            `json:"authToken"`
+	ClientCert string            `json:"clientCert"`
+	ClientKey  string            `json:"clientKey"`
+	Transport  http.RoundTripper `json:"-"`
+
+	// Custom logger
+	LogOnce func(ctx context.Context, err error, id interface{}, errKind ...interface{}) `json:"-"`
+}
+
 // Target implements logger.Target and sends the json
 // format of a log entry to the configured http endpoint.
 // An internal buffer of logs is maintained but when the
@@ -43,32 +57,24 @@ type Target struct {
 	// Channel of log entries
 	logCh chan interface{}
 
-	name string
-	// HTTP(s) endpoint
-	endpoint string
-	// Authorization token for `endpoint`
-	authToken string
-	// User-Agent to be set on each log to `endpoint`
-	userAgent string
-	logKind   string
-	client    http.Client
+	config Config
 }
 
 // Endpoint returns the backend endpoint
 func (h *Target) Endpoint() string {
-	return h.endpoint
+	return h.config.Endpoint
 }
 
 func (h *Target) String() string {
-	return h.name
+	return h.config.Name
 }
 
-// Validate validate the http target
-func (h *Target) Validate() error {
+// Init validate and initialize the http target
+func (h *Target) Init() error {
 	ctx, cancel := context.WithTimeout(context.Background(), 2*webhookCallTimeout)
 	defer cancel()
 
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.endpoint, strings.NewReader(`{}`))
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, h.config.Endpoint, strings.NewReader(`{}`))
 	if err != nil {
 		return err
 	}
@@ -77,13 +83,14 @@ func (h *Target) Validate() error {
 
 	// Set user-agent to indicate MinIO release
 	// version to the configured log endpoint
-	req.Header.Set("User-Agent", h.userAgent)
+	req.Header.Set("User-Agent", h.config.UserAgent)
 
-	if h.authToken != "" {
-		req.Header.Set("Authorization", h.authToken)
+	if h.config.AuthToken != "" {
+		req.Header.Set("Authorization", h.config.AuthToken)
 	}
 
-	resp, err := h.client.Do(req)
+	client := http.Client{Transport: h.config.Transport}
+	resp, err := client.Do(req)
 	if err != nil {
 		return err
 	}
@@ -95,12 +102,13 @@ func (h *Target) Validate() error {
 		switch resp.StatusCode {
 		case http.StatusForbidden:
 			return fmt.Errorf("%s returned '%s', please check if your auth token is correctly set",
-				h.endpoint, resp.Status)
+				h.config.Endpoint, resp.Status)
 		}
 		return fmt.Errorf("%s returned '%s', please check your endpoint configuration",
-			h.endpoint, resp.Status)
+			h.config.Endpoint, resp.Status)
 	}
 
+	go h.startHTTPLogger()
 	return nil
 }
 
@@ -116,7 +124,7 @@ func (h *Target) startHTTPLogger() {
 
 			ctx, cancel := context.WithTimeout(context.Background(), webhookCallTimeout)
 			req, err := http.NewRequestWithContext(ctx, http.MethodPost,
-				h.endpoint, bytes.NewReader(logJSON))
+				h.config.Endpoint, bytes.NewReader(logJSON))
 			if err != nil {
 				cancel()
 				continue
@@ -125,17 +133,17 @@ func (h *Target) startHTTPLogger() {
 
 			// Set user-agent to indicate MinIO release
 			// version to the configured log endpoint
-			req.Header.Set("User-Agent", h.userAgent)
+			req.Header.Set("User-Agent", h.config.UserAgent)
 
-			if h.authToken != "" {
-				req.Header.Set("Authorization", h.authToken)
+			if h.config.AuthToken != "" {
+				req.Header.Set("Authorization", h.config.AuthToken)
 			}
 
-			resp, err := h.client.Do(req)
+			client := http.Client{Transport: h.config.Transport}
+			resp, err := client.Do(req)
 			cancel()
 			if err != nil {
-				logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%w', please check your endpoint configuration",
-					h.endpoint, err), h.endpoint)
+				h.config.LogOnce(ctx, fmt.Errorf("%s returned '%w', please check your endpoint configuration", h.config.Endpoint, err), h.config.Endpoint)
 				continue
 			}
 
@@ -145,88 +153,28 @@ func (h *Target) startHTTPLogger() {
 			if resp.StatusCode != http.StatusOK {
 				switch resp.StatusCode {
 				case http.StatusForbidden:
-					logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%s', please check if your auth token is correctly set",
-						h.endpoint, resp.Status), h.endpoint)
+					h.config.LogOnce(ctx, fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", h.config.Endpoint, resp.Status), h.config.Endpoint)
 				default:
-					logger.LogOnceIf(ctx, fmt.Errorf("%s returned '%s', please check your endpoint configuration",
-						h.endpoint, resp.Status), h.endpoint)
+					h.config.LogOnce(ctx, fmt.Errorf("%s returned '%s', please check your endpoint configuration", h.config.Endpoint, resp.Status), h.config.Endpoint)
 				}
 			}
 		}
 	}()
 }
 
-// Option is a function type that accepts a pointer Target
-type Option func(*Target)
-
-// WithTargetName target name
-func WithTargetName(name string) Option {
-	return func(t *Target) {
-		t.name = name
-	}
-}
-
-// WithEndpoint adds a new endpoint
-func WithEndpoint(endpoint string) Option {
-	return func(t *Target) {
-		t.endpoint = endpoint
-	}
-}
-
-// WithLogKind adds a log type for this target
-func WithLogKind(logKind string) Option {
-	return func(t *Target) {
-		t.logKind = strings.ToUpper(logKind)
-	}
-}
-
-// WithUserAgent adds a custom user-agent sent to the target.
-func WithUserAgent(userAgent string) Option {
-	return func(t *Target) {
-		t.userAgent = userAgent
-	}
-}
-
-// WithAuthToken adds a new authorization header to be sent to target.
-func WithAuthToken(authToken string) Option {
-	return func(t *Target) {
-		t.authToken = authToken
-	}
-}
-
-// WithTransport adds a custom transport with custom timeouts and tuning.
-func WithTransport(transport *http.Transport) Option {
-	return func(t *Target) {
-		t.client = http.Client{
-			Transport: transport,
-		}
-	}
-}
-
 // New initializes a new logger target which
 // sends log over http to the specified endpoint
-func New(opts ...Option) *Target {
+func New(config Config) *Target {
 	h := &Target{
-		logCh: make(chan interface{}, 10000),
+		logCh:  make(chan interface{}, 10000),
+		config: config,
 	}
 
-	// Loop through each option
-	for _, opt := range opts {
-		// Call the option giving the instantiated
-		// *Target as the argument
-		opt(h)
-	}
-
-	h.startHTTPLogger()
 	return h
 }
 
 // Send log message 'e' to http target.
 func (h *Target) Send(entry interface{}, errKind string) error {
-	if h.logKind != errKind && h.logKind != "ALL" {
-		return nil
-	}
-
 	select {
 	case h.logCh <- entry:
 	default:

internal/logger/target/kafka/kafka.go

@@ -0,0 +1,208 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package kafka
+
+import (
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"errors"
+	"net"
+
+	sarama "github.com/Shopify/sarama"
+	saramatls "github.com/Shopify/sarama/tools/tls"
+
+	"github.com/minio/minio/internal/logger/message/audit"
+	xnet "github.com/minio/pkg/net"
+)
+
+// Target - Kafka target.
+type Target struct {
+	// Channel of log entries
+	logCh chan interface{}
+
+	producer sarama.SyncProducer
+	kconfig  Config
+	config   *sarama.Config
+}
+
+// Send log message 'e' to kafka target.
+func (h *Target) Send(entry interface{}, errKind string) error {
+	select {
+	case h.logCh <- entry:
+	default:
+		// log channel is full, do not wait and return
+		// an error immediately to the caller
+		return errors.New("log buffer full")
+	}
+
+	return nil
+}
+
+func (h *Target) startKakfaLogger() {
+	// Create a routine which sends json logs received
+	// from an internal channel.
+	go func() {
+		for entry := range h.logCh {
+			logJSON, err := json.Marshal(&entry)
+			if err != nil {
+				continue
+			}
+
+			ae, ok := entry.(audit.Entry)
+			if ok {
+				msg := sarama.ProducerMessage{
+					Topic: h.kconfig.Topic,
+					Key:   sarama.StringEncoder(ae.RequestID),
+					Value: sarama.ByteEncoder(logJSON),
+				}
+
+				_, _, err = h.producer.SendMessage(&msg)
+				if err != nil {
+					h.kconfig.LogOnce(context.Background(), err, h.kconfig.Topic)
+					continue
+				}
+			}
+		}
+	}()
+}
+
+// Config - kafka target arguments.
+type Config struct {
+	Enabled bool        `json:"enable"`
+	Brokers []xnet.Host `json:"brokers"`
+	Topic   string      `json:"topic"`
+	Version string      `json:"version"`
+	TLS     struct {
+		Enable        bool               `json:"enable"`
+		RootCAs       *x509.CertPool     `json:"-"`
+		SkipVerify    bool               `json:"skipVerify"`
+		ClientAuth    tls.ClientAuthType `json:"clientAuth"`
+		ClientTLSCert string             `json:"clientTLSCert"`
+		ClientTLSKey  string             `json:"clientTLSKey"`
+	} `json:"tls"`
+	SASL struct {
+		Enable    bool   `json:"enable"`
+		User      string `json:"username"`
+		Password  string `json:"password"`
+		Mechanism string `json:"mechanism"`
+	} `json:"sasl"`
+
+	// Custom logger
+	LogOnce func(ctx context.Context, err error, id interface{}, errKind ...interface{}) `json:"-"`
+}
+
+// Check if atleast one broker in cluster is active
+func (k Config) pingBrokers() error {
+	var err error
+	for _, broker := range k.Brokers {
+		_, err1 := net.Dial("tcp", broker.String())
+		if err1 != nil {
+			if err == nil {
+				// Set first error
+				err = err1
+			}
+		}
+	}
+	return err
+}
+
+// Endpoint - return kafka target
+func (h *Target) Endpoint() string {
+	return "kafka"
+}
+
+// String - kafka string
+func (h *Target) String() string {
+	return "kafka"
+}
+
+// Init initialize kafka target
+func (h *Target) Init() error {
+	if !h.kconfig.Enabled {
+		return nil
+	}
+	if len(h.kconfig.Brokers) == 0 {
+		return errors.New("no broker address found")
+	}
+	for _, b := range h.kconfig.Brokers {
+		if _, err := xnet.ParseHost(b.String()); err != nil {
+			return err
+		}
+	}
+	if err := h.kconfig.pingBrokers(); err != nil {
+		return err
+	}
+
+	sconfig := sarama.NewConfig()
+	if h.kconfig.Version != "" {
+		kafkaVersion, err := sarama.ParseKafkaVersion(h.kconfig.Version)
+		if err != nil {
+			return err
+		}
+		sconfig.Version = kafkaVersion
+	}
+
+	sconfig.Net.SASL.User = h.kconfig.SASL.User
+	sconfig.Net.SASL.Password = h.kconfig.SASL.Password
+	initScramClient(h.kconfig, sconfig) // initializes configured scram client.
+	sconfig.Net.SASL.Enable = h.kconfig.SASL.Enable
+
+	tlsConfig, err := saramatls.NewConfig(h.kconfig.TLS.ClientTLSCert, h.kconfig.TLS.ClientTLSKey)
+	if err != nil {
+		return err
+	}
+
+	sconfig.Net.TLS.Enable = h.kconfig.TLS.Enable
+	sconfig.Net.TLS.Config = tlsConfig
+	sconfig.Net.TLS.Config.InsecureSkipVerify = h.kconfig.TLS.SkipVerify
+	sconfig.Net.TLS.Config.ClientAuth = h.kconfig.TLS.ClientAuth
+	sconfig.Net.TLS.Config.RootCAs = h.kconfig.TLS.RootCAs
+
+	sconfig.Producer.RequiredAcks = sarama.WaitForAll
+	sconfig.Producer.Retry.Max = 10
+	sconfig.Producer.Return.Successes = true
+
+	h.config = sconfig
+
+	var brokers []string
+	for _, broker := range h.kconfig.Brokers {
+		brokers = append(brokers, broker.String())
+	}
+
+	producer, err := sarama.NewSyncProducer(brokers, sconfig)
+	if err != nil {
+		return err
+	}
+
+	h.producer = producer
+
+	go h.startKakfaLogger()
+	return nil
+}
+
+// New initializes a new logger target which
+// sends log over http to the specified endpoint
+func New(config Config) *Target {
+	target := &Target{
+		logCh:   make(chan interface{}, 10000),
+		kconfig: config,
+	}
+	return target
+}

internal/logger/target/kafka/kafka_scram_client_contrib.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package kafka
+
+import (
+	"crypto/sha256"
+	"crypto/sha512"
+
+	"github.com/Shopify/sarama"
+	"github.com/xdg/scram"
+)
+
+func initScramClient(cfg Config, config *sarama.Config) {
+	if cfg.SASL.Mechanism == "sha512" {
+		config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA512} }
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA512)
+	} else if cfg.SASL.Mechanism == "sha256" {
+		config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA256} }
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA256)
+	} else {
+		// default to PLAIN
+		config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypePlaintext)
+	}
+}
+
+// KafkaSHA256 is a function that returns a crypto/sha256 hasher and should be used
+// to create Client objects configured for SHA-256 hashing.
+var KafkaSHA256 scram.HashGeneratorFcn = sha256.New
+
+// KafkaSHA512 is a function that returns a crypto/sha512 hasher and should be used
+// to create Client objects configured for SHA-512 hashing.
+var KafkaSHA512 scram.HashGeneratorFcn = sha512.New
+
+// XDGSCRAMClient implements the client-side of an authentication
+// conversation with a server.  A new conversation must be created for
+// each authentication attempt.
+type XDGSCRAMClient struct {
+	*scram.Client
+	*scram.ClientConversation
+	scram.HashGeneratorFcn
+}
+
+// Begin constructs a SCRAM client component based on a given hash.Hash
+// factory receiver.  This constructor will normalize the username, password
+// and authzID via the SASLprep algorithm, as recommended by RFC-5802.  If
+// SASLprep fails, the method returns an error.
+func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) {
+	x.Client, err = x.HashGeneratorFcn.NewClient(userName, password, authzID)
+	if err != nil {
+		return err
+	}
+	x.ClientConversation = x.Client.NewConversation()
+	return nil
+}
+
+// Step takes a string provided from a server (or just an empty string for the
+// very first conversation step) and attempts to move the authentication
+// conversation forward.  It returns a string to be sent to the server or an
+// error if the server message is invalid.  Calling Step after a conversation
+// completes is also an error.
+func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) {
+	response, err = x.ClientConversation.Step(challenge)
+	return
+}
+
+// Done returns true if the conversation is completed or has errored.
+func (x *XDGSCRAMClient) Done() bool {
+	return x.ClientConversation.Done()
+}

internal/logger/targets.go

@@ -23,7 +23,7 @@ package logger
 type Target interface {
 	String() string
 	Endpoint() string
-	Validate() error
+	Init() error
 	Send(entry interface{}, errKind string) error
 }
 
@@ -36,7 +36,7 @@ var AuditTargets = []Target{}
 // AddAuditTarget adds a new audit logger target to the
 // list of enabled loggers
 func AddAuditTarget(t Target) error {
-	if err := t.Validate(); err != nil {
+	if err := t.Init(); err != nil {
 		return err
 	}
 
@@ -47,7 +47,7 @@ func AddAuditTarget(t Target) error {
 // AddTarget adds a new logger target to the
 // list of enabled loggers
 func AddTarget(t Target) error {
-	if err := t.Validate(); err != nil {
+	if err := t.Init(); err != nil {
 		return err
 	}
 	Targets = append(Targets, t)