Add config to store subnet license (#13194)

Command to set subnet license:

`mc admin config set {alias} subnet license={token}`

Signed-off-by: Shireesh Anjal <shireesh@minio.io>
Co-authored-by: Harshavardhana <harsha@minio.io>

Dockerfile.release

@@ -18,7 +18,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=$PATH:/opt/bin
 
 COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
@@ -30,11 +31,12 @@ RUN \
      microdnf install curl ca-certificates shadow-utils util-linux iproute iputils --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /usr/bin/minio.minisig && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /opt/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /opt/bin/minio.minisig && \
      microdnf clean all && \
-     chmod +x /usr/bin/minio && \
+     chmod +x /opt/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
      /usr/bin/verify-minio.sh

Dockerfile.release.fips

@@ -18,7 +18,8 @@ ENV MINIO_ACCESS_KEY_FILE=access_key \
     MINIO_ROOT_PASSWORD_FILE=secret_key \
     MINIO_KMS_SECRET_KEY_FILE=kms_master_key \
     MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \
-    MINIO_CONFIG_ENV_FILE=config.env
+    MINIO_CONFIG_ENV_FILE=config.env \
+    PATH=$PATH:/opt/bin
 
 COPY dockerscripts/verify-minio.sh /usr/bin/verify-minio.sh
 COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh
@@ -30,11 +31,12 @@ RUN \
      microdnf install curl ca-certificates shadow-utils util-linux iproute iputils --nodocs && \
      rpm -Uvh https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm && \
      microdnf install minisign --nodocs && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /usr/bin/minio && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /usr/bin/minio.sha256sum && \
-     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /usr/bin/minio.minisig && \
+     mkdir -p /opt/bin && chmod -R 777 /opt/bin && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips -o /opt/bin/minio && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.sha256sum -o /opt/bin/minio.sha256sum && \
+     curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.fips.minisig -o /opt/bin/minio.minisig && \
      microdnf clean all && \
-     chmod +x /usr/bin/minio && \
+     chmod +x /opt/bin/minio && \
      chmod +x /usr/bin/docker-entrypoint.sh && \
      chmod +x /usr/bin/verify-minio.sh && \
      /usr/bin/verify-minio.sh

Dockerfile.scratch

@@ -0,0 +1,5 @@
+FROM scratch
+
+COPY minio /minio
+
+CMD ["/minio"]

cmd/admin-handlers.go

@@ -925,6 +925,12 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 	sizeStr := r.Form.Get(peerRESTSize)
 	durationStr := r.Form.Get(peerRESTDuration)
 	concurrentStr := r.Form.Get(peerRESTConcurrent)
+	autotuneStr := r.Form.Get("autotune")
+
+	var autotune bool
+	if autotuneStr != "" {
+		autotune = true
+	}
 
 	size, err := strconv.Atoi(sizeStr)
 	if err != nil {
@@ -941,9 +947,47 @@ func (a adminAPIHandlers) SpeedtestHandler(w http.ResponseWriter, r *http.Reques
 		duration = time.Second * 10
 	}
 
-	results := globalNotificationSys.Speedtest(ctx, size, concurrent, duration)
+	throughputSize := size
+	iopsSize := size
 
-	if err := json.NewEncoder(w).Encode(results); err != nil {
+	if autotune {
+		iopsSize = 4 * humanize.KiByte
+	}
+
+	keepAliveTicker := time.NewTicker(500 * time.Millisecond)
+	defer keepAliveTicker.Stop()
+
+	endBlankRepliesCh := make(chan error)
+
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				endBlankRepliesCh <- nil
+				return
+			case <-keepAliveTicker.C:
+				// Write a blank entry to prevent client from disconnecting
+				if err := json.NewEncoder(w).Encode(madmin.SpeedTestResult{}); err != nil {
+					endBlankRepliesCh <- err
+					return
+				}
+				w.(http.Flusher).Flush()
+			case endBlankRepliesCh <- nil:
+				return
+			}
+		}
+	}()
+
+	result, err := speedTest(ctx, throughputSize, iopsSize, concurrent, duration, autotune)
+	if <-endBlankRepliesCh != nil {
+		return
+	}
+	if err != nil {
+		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
+		return
+	}
+
+	if err := json.NewEncoder(w).Encode(result); err != nil {
 		writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL)
 		return
 	}

cmd/auth-handler.go

@@ -138,7 +138,7 @@ func validateAdminSignature(ctx context.Context, r *http.Request, region string)
 	var owner bool
 	s3Err := ErrAccessDenied
 	if _, ok := r.Header[xhttp.AmzContentSha256]; ok &&
-		getRequestAuthType(r) == authTypeSigned && !skipContentSha256Cksum(r) {
+		getRequestAuthType(r) == authTypeSigned {
 		// We only support admin credentials to access admin APIs.
 		cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3)
 		if s3Err != ErrNone {

cmd/common-main.go

@@ -144,6 +144,9 @@ func minioConfigToConsoleFeatures() {
 		os.Setenv("CONSOLE_IDP_HMAC_SALT", globalDeploymentID)
 		os.Setenv("CONSOLE_IDP_HMAC_PASSPHRASE", globalOpenIDConfig.ClientID)
 		os.Setenv("CONSOLE_IDP_SCOPES", strings.Join(globalOpenIDConfig.DiscoveryDoc.ScopesSupported, ","))
+		if globalOpenIDConfig.ClaimUserinfo {
+			os.Setenv("CONSOLE_IDP_USERINFO", "on")
+		}
 		if globalOpenIDConfig.RedirectURI != "" {
 			os.Setenv("CONSOLE_IDP_CALLBACK", globalOpenIDConfig.RedirectURI)
 		} else {
@@ -152,8 +155,8 @@ func minioConfigToConsoleFeatures() {
 	}
 	os.Setenv("CONSOLE_MINIO_REGION", globalServerRegion)
 	os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", ""))
-	if globalSubnetLicense != "" {
-		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetLicense)
+	if globalSubnetConfig.License != "" {
+		os.Setenv("CONSOLE_SUBNET_LICENSE", globalSubnetConfig.License)
 	}
 }
 
@@ -599,8 +602,6 @@ func handleCommonEnvVars() {
 	if tiers := env.Get("_MINIO_DEBUG_REMOTE_TIERS_IMMEDIATELY", ""); tiers != "" {
 		globalDebugRemoteTiersImmediately = strings.Split(tiers, ",")
 	}
-
-	globalSubnetLicense = env.Get(config.EnvMinIOSubnetLicense, "")
 }
 
 func logStartupMessage(msg string) {

cmd/config-current.go

@@ -39,6 +39,7 @@ import (
 	"github.com/minio/minio/internal/config/policy/opa"
 	"github.com/minio/minio/internal/config/scanner"
 	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/config/subnet"
 	"github.com/minio/minio/internal/crypto"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/kms"
@@ -65,6 +66,7 @@ func initHelp() {
 		config.AuditKafkaSubSys:     logger.DefaultAuditKafkaKVS,
 		config.HealSubSys:           heal.DefaultKVS,
 		config.ScannerSubSys:        scanner.DefaultKVS,
+		config.SubnetSubSys:         subnet.DefaultKVS,
 	}
 	for k, v := range notify.DefaultNotificationKVS {
 		kvs[k] = v
@@ -185,6 +187,12 @@ func initHelp() {
 			Description:     "publish bucket notifications to Redis datastores",
 			MultipleTargets: true,
 		},
+		config.HelpKV{
+			Key:         config.SubnetSubSys,
+			Type:        "string",
+			Description: "set subnet config for the cluster e.g. license token",
+			Optional:    true,
+		},
 	}
 
 	if globalIsErasure {
@@ -223,6 +231,7 @@ func initHelp() {
 		config.NotifyRedisSubSys:    notify.HelpRedis,
 		config.NotifyWebhookSubSys:  notify.HelpWebhook,
 		config.NotifyESSubSys:       notify.HelpES,
+		config.SubnetSubSys:         subnet.HelpLicense,
 	}
 
 	config.RegisterHelpSubSys(helpMap)
@@ -508,6 +517,11 @@ func lookupConfigs(s config.Config, objAPI ObjectLayer) {
 		logger.LogIf(ctx, fmt.Errorf("Unable to parse LDAP configuration: %w", err))
 	}
 
+	globalSubnetConfig, err = subnet.LookupConfig(s[config.SubnetSubSys][config.Default])
+	if err != nil {
+		logger.LogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err))
+	}
+
 	// Load logger targets based on user's configuration
 	loggerUserAgent := getUserAgent(getMinioMode())
 

cmd/erasure-sets.go

@@ -1292,9 +1292,12 @@ func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.H
 			if s.erasureDisks[m][n] != nil {
 				s.erasureDisks[m][n].Close()
 			}
-			storageDisks[index].SetDiskLoc(s.poolIndex, m, n)
-			s.erasureDisks[m][n] = storageDisks[index]
-			s.endpointStrings[m*s.setDriveCount+n] = storageDisks[index].String()
+
+			if storageDisks[index] != nil {
+				storageDisks[index].SetDiskLoc(s.poolIndex, m, n)
+				s.erasureDisks[m][n] = storageDisks[index]
+				s.endpointStrings[m*s.setDriveCount+n] = storageDisks[index].String()
+			}
 		}
 
 		// Replace reference format with what was loaded from disks.

cmd/gateway-main.go

@@ -280,7 +280,6 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		SecretKey: globalActiveCred.SecretKey,
 	})
 	if err != nil {
-		globalHTTPServer.Shutdown()
 		logger.FatalIf(err, "Unable to initialize gateway backend")
 	}
 	newObject = NewGatewayLayerWithLocker(newObject)

cmd/gateway/s3/gateway-s3.go

@@ -23,6 +23,7 @@ import (
 	"math/rand"
 	"net/http"
 	"net/url"
+	"os"
 	"strings"
 	"time"
 
@@ -34,9 +35,11 @@ import (
 	"github.com/minio/minio-go/v7/pkg/s3utils"
 	"github.com/minio/minio-go/v7/pkg/tags"
 	minio "github.com/minio/minio/cmd"
+	"github.com/minio/minio/internal/config"
 	xhttp "github.com/minio/minio/internal/http"
 	"github.com/minio/minio/internal/logger"
 	"github.com/minio/pkg/bucket/policy"
+	"github.com/minio/pkg/env"
 )
 
 func init() {
@@ -94,12 +97,16 @@ func s3GatewayMain(ctx *cli.Context) {
 	logger.FatalIf(minio.ValidateGatewayArguments(serverAddr, args.First()), "Invalid argument")
 
 	// Start the gateway..
-	minio.StartGateway(ctx, &S3{args.First()})
+	minio.StartGateway(ctx, &S3{
+		host:  args.First(),
+		debug: env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn,
+	})
 }
 
 // S3 implements Gateway.
 type S3 struct {
-	host string
+	host  string
+	debug bool
 }
 
 // Name implements Gateway interface.
@@ -219,6 +226,10 @@ func (g *S3) NewGatewayLayer(creds madmin.Credentials) (minio.ObjectLayer, error
 		return nil, err
 	}
 
+	if g.debug {
+		clnt.Client.TraceOn(os.Stderr)
+	}
+
 	probeBucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "probe-bucket-sign-")
 
 	// Check if the provided keys are valid.

cmd/globals.go

@@ -41,6 +41,7 @@ import (
 	xtls "github.com/minio/minio/internal/config/identity/tls"
 	"github.com/minio/minio/internal/config/policy/opa"
 	"github.com/minio/minio/internal/config/storageclass"
+	"github.com/minio/minio/internal/config/subnet"
 	xhttp "github.com/minio/minio/internal/http"
 	etcd "go.etcd.io/etcd/client/v3"
 
@@ -219,8 +220,8 @@ var (
 	// The name of this local node, fetched from arguments
 	globalLocalNodeName string
 
-	// The global subnet license
-	globalSubnetLicense string
+	// The global subnet config
+	globalSubnetConfig subnet.Config
 
 	globalRemoteEndpoints map[string]Endpoint
 

cmd/metacache-set.go

@@ -599,7 +599,7 @@ type metaCacheRPC struct {
 
 func (m *metaCacheRPC) setErr(err string) {
 	m.mu.Lock()
-	defer m.mu.Lock()
+	defer m.mu.Unlock()
 	meta := *m.meta
 	if meta.status != scanStateError {
 		meta.error = err

cmd/notification.go

@@ -1505,8 +1505,13 @@ func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) chan Metric {
 
 // Speedtest run GET/PUT tests at input concurrency for requested object size,
 // optionally you can extend the tests longer with time.Duration.
-func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent int, duration time.Duration) []madmin.SpeedtestResult {
-	results := make([]madmin.SpeedtestResult, len(sys.allPeerClients))
+func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent int, duration time.Duration) []SpeedtestResult {
+	length := len(sys.allPeerClients)
+	if length == 0 {
+		// For single node erasure setup.
+		length = 1
+	}
+	results := make([]SpeedtestResult, length)
 
 	scheme := "http"
 	if globalIsTLS {
@@ -1526,12 +1531,12 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent
 				Scheme: scheme,
 				Host:   sys.peerClients[index].host.String(),
 			}
-			results[index].Endpoint = u.String()
-			results[index].Err = err
-			if err == nil {
-				results[index].Uploads = r.Uploads
-				results[index].Downloads = r.Downloads
+			if err != nil {
+				results[index].Error = err.Error()
+			} else {
+				results[index] = r
 			}
+			results[index].Endpoint = u.String()
 		}(index)
 	}
 
@@ -1543,12 +1548,12 @@ func (sys *NotificationSys) Speedtest(ctx context.Context, size int, concurrent
 			Scheme: scheme,
 			Host:   globalLocalNodeName,
 		}
-		results[len(results)-1].Endpoint = u.String()
-		results[len(results)-1].Err = err
-		if err == nil {
-			results[len(results)-1].Uploads = r.Uploads
-			results[len(results)-1].Downloads = r.Downloads
+		if err != nil {
+			results[len(results)-1].Error = err.Error()
+		} else {
+			results[len(results)-1] = r
 		}
+		results[len(results)-1].Endpoint = u.String()
 	}()
 	wg.Wait()
 

cmd/object-handlers.go

@@ -916,9 +916,18 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
+	if _, ok := crypto.IsRequested(r.Header); ok {
+		if globalIsGateway {
+			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		} else {
+			if !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		}
 	}
 
 	vars := mux.Vars(r)
@@ -1467,9 +1476,18 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
+	if _, ok := crypto.IsRequested(r.Header); ok {
+		if globalIsGateway {
+			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		} else {
+			if !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		}
 	}
 
 	vars := mux.Vars(r)
@@ -1798,9 +1816,18 @@ func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *h
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
+	if _, ok := crypto.IsRequested(r.Header); ok {
+		if globalIsGateway {
+			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		} else {
+			if !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		}
 	}
 
 	vars := mux.Vars(r)
@@ -2075,9 +2102,18 @@ func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
-		return
+	if _, ok := crypto.IsRequested(r.Header); ok {
+		if globalIsGateway {
+			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		} else {
+			if !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		}
 	}
 
 	vars := mux.Vars(r)
@@ -2517,8 +2553,18 @@ func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http
 		return
 	}
 
-	if _, ok := crypto.IsRequested(r.Header); ok && !objectAPI.IsEncryptionSupported() {
-		writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+	if _, ok := crypto.IsRequested(r.Header); ok {
+		if globalIsGateway {
+			if crypto.SSEC.IsRequested(r.Header) && !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		} else {
+			if !objectAPI.IsEncryptionSupported() {
+				writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL)
+				return
+			}
+		}
 	}
 
 	vars := mux.Vars(r)

cmd/peer-rest-server.go

@@ -1156,6 +1156,7 @@ func (s *peerRESTServer) GetPeerMetrics(w http.ResponseWriter, r *http.Request)
 
 // SpeedtestResult return value of the speedtest function
 type SpeedtestResult struct {
+	Endpoint  string
 	Uploads   uint64
 	Downloads uint64
 	Error     string
@@ -1163,8 +1164,9 @@ type SpeedtestResult struct {
 
 // SpeedtestObject implements "random-read" object reader
 type SpeedtestObject struct {
-	buf       []byte
-	remaining int
+	buf               []byte
+	remaining         int
+	totalBytesWritten *uint64
 }
 
 func (bo *SpeedtestObject) Read(b []byte) (int, error) {
@@ -1182,17 +1184,20 @@ func (bo *SpeedtestObject) Read(b []byte) (int, error) {
 	}
 	copy(b, bo.buf)
 	bo.remaining -= len(b)
+
+	atomic.AddUint64(bo.totalBytesWritten, uint64(len(b)))
 	return len(b), nil
 }
 
 // Runs the speedtest on local MinIO process.
 func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Duration) (SpeedtestResult, error) {
-	var result SpeedtestResult
 	objAPI := newObjectLayerFn()
 	if objAPI == nil {
-		return result, errServerNotInitialized
+		return SpeedtestResult{}, errServerNotInitialized
 	}
 
+	var retError string
+
 	bucket := minioMetaSpeedTestBucket
 
 	buf := make([]byte, humanize.MiByte)
@@ -1200,15 +1205,16 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 
 	objCountPerThread := make([]uint64, concurrent)
 
-	var objUploadCount uint64
-	var objDownloadCount uint64
+	uploadsStopped := false
+	var totalBytesWritten uint64
+	uploadsCtx, uploadsCancel := context.WithCancel(context.Background())
 
 	var wg sync.WaitGroup
 
-	doneCh1 := make(chan struct{})
 	go func() {
 		time.Sleep(duration)
-		close(doneCh1)
+		uploadsStopped = true
+		uploadsCancel()
 	}()
 
 	objNamePrefix := minioMetaSpeedTestBucketPrefix + uuid.New().String()
@@ -1218,35 +1224,37 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 		go func(i int) {
 			defer wg.Done()
 			for {
-				hashReader, err := hash.NewReader(&SpeedtestObject{buf, size},
+				hashReader, err := hash.NewReader(&SpeedtestObject{buf, size, &totalBytesWritten},
 					int64(size), "", "", int64(size))
 				if err != nil {
+					retError = err.Error()
 					logger.LogIf(ctx, err)
 					break
 				}
 				reader := NewPutObjReader(hashReader)
-				_, err = objAPI.PutObject(ctx, bucket, fmt.Sprintf("%s.%d.%d",
+				_, err = objAPI.PutObject(uploadsCtx, bucket, fmt.Sprintf("%s.%d.%d",
 					objNamePrefix, i, objCountPerThread[i]), reader, ObjectOptions{})
-				if err != nil {
+				if err != nil && !uploadsStopped {
+					retError = err.Error()
 					logger.LogIf(ctx, err)
+				}
+				if err != nil {
 					break
 				}
 				objCountPerThread[i]++
-				atomic.AddUint64(&objUploadCount, 1)
-				select {
-				case <-doneCh1:
-					return
-				default:
-				}
 			}
 		}(i)
 	}
 	wg.Wait()
 
-	doneCh2 := make(chan struct{})
+	downloadsStopped := false
+	var totalBytesRead uint64
+	downloadsCtx, downloadsCancel := context.WithCancel(context.Background())
+
 	go func() {
 		time.Sleep(duration)
-		close(doneCh2)
+		downloadsStopped = true
+		downloadsCancel()
 	}()
 
 	wg.Add(concurrent)
@@ -1254,34 +1262,39 @@ func selfSpeedtest(ctx context.Context, size, concurrent int, duration time.Dura
 		go func(i int) {
 			defer wg.Done()
 			var j uint64
+			if objCountPerThread[i] == 0 {
+				return
+			}
 			for {
 				if objCountPerThread[i] == j {
 					j = 0
 				}
-				r, err := objAPI.GetObjectNInfo(ctx, bucket, fmt.Sprintf("%s.%d.%d",
+				r, err := objAPI.GetObjectNInfo(downloadsCtx, bucket, fmt.Sprintf("%s.%d.%d",
 					objNamePrefix, i, j), nil, nil, noLock, ObjectOptions{})
-				if err != nil {
+				if err != nil && !downloadsStopped {
+					retError = err.Error()
 					logger.LogIf(ctx, err)
+				}
+				if err != nil {
 					break
 				}
-				_, err = io.Copy(ioutil.Discard, r)
+				n, err := io.Copy(ioutil.Discard, r)
 				r.Close()
-				if err != nil {
+
+				atomic.AddUint64(&totalBytesRead, uint64(n))
+				if err != nil && !downloadsStopped {
+					retError = err.Error()
 					logger.LogIf(ctx, err)
+				}
+				if err != nil {
 					break
 				}
 				j++
-				atomic.AddUint64(&objDownloadCount, 1)
-				select {
-				case <-doneCh2:
-					return
-				default:
-				}
 			}
 		}(i)
 	}
 	wg.Wait()
-	return SpeedtestResult{Uploads: objUploadCount, Downloads: objDownloadCount}, nil
+	return SpeedtestResult{Uploads: totalBytesWritten, Downloads: totalBytesRead, Error: retError}, nil
 }
 
 func (s *peerRESTServer) SpeedtestHandler(w http.ResponseWriter, r *http.Request) {

cmd/sts-handlers.go

@@ -39,15 +39,16 @@ import (
 
 const (
 	// STS API version.
-	stsAPIVersion       = "2011-06-15"
-	stsVersion          = "Version"
-	stsAction           = "Action"
-	stsPolicy           = "Policy"
-	stsToken            = "Token"
-	stsWebIdentityToken = "WebIdentityToken"
-	stsDurationSeconds  = "DurationSeconds"
-	stsLDAPUsername     = "LDAPUsername"
-	stsLDAPPassword     = "LDAPPassword"
+	stsAPIVersion             = "2011-06-15"
+	stsVersion                = "Version"
+	stsAction                 = "Action"
+	stsPolicy                 = "Policy"
+	stsToken                  = "Token"
+	stsWebIdentityToken       = "WebIdentityToken"
+	stsWebIdentityAccessToken = "WebIdentityAccessToken" // only valid if UserInfo is enabled.
+	stsDurationSeconds        = "DurationSeconds"
+	stsLDAPUsername           = "LDAPUsername"
+	stsLDAPPassword           = "LDAPPassword"
 
 	// STS API action constants
 	clientGrants      = "AssumeRoleWithClientGrants"
@@ -336,7 +337,9 @@ func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Requ
 		token = r.Form.Get(stsWebIdentityToken)
 	}
 
-	m, err := v.Validate(token, r.Form.Get(stsDurationSeconds))
+	accessToken := r.Form.Get(stsWebIdentityAccessToken)
+
+	m, err := v.Validate(token, accessToken, r.Form.Get(stsDurationSeconds))
 	if err != nil {
 		switch err {
 		case openid.ErrTokenExpired:

cmd/utils.go

@@ -37,6 +37,7 @@ import (
 	"runtime"
 	"runtime/pprof"
 	"runtime/trace"
+	"sort"
 	"strings"
 	"sync"
 	"time"
@@ -970,3 +971,163 @@ func auditLogInternal(ctx context.Context, bucket, object string, opts AuditLogO
 	ctx = logger.SetAuditEntry(ctx, &entry)
 	logger.AuditLog(ctx, nil, nil, nil)
 }
+
+// Get the max throughput and iops numbers.
+func speedTest(ctx context.Context, throughputSize, iopsSize int, concurrencyStart int, duration time.Duration, autotune bool) (madmin.SpeedTestResult, error) {
+	var result madmin.SpeedTestResult
+
+	objAPI := newObjectLayerFn()
+	if objAPI == nil {
+		return result, errServerNotInitialized
+	}
+
+	concurrency := concurrencyStart
+
+	throughputHighestGet := uint64(0)
+	throughputHighestPut := uint64(0)
+	var throughputHighestPutResults []SpeedtestResult
+	var throughputHighestGetResults []SpeedtestResult
+
+	for {
+		select {
+		case <-ctx.Done():
+			// If the client got disconnected stop the speedtest.
+			return result, errUnexpected
+		default:
+		}
+
+		results := globalNotificationSys.Speedtest(ctx, throughputSize, concurrency, duration)
+		sort.Slice(results, func(i, j int) bool {
+			return results[i].Endpoint < results[j].Endpoint
+		})
+		totalPut := uint64(0)
+		totalGet := uint64(0)
+		for _, result := range results {
+			totalPut += result.Uploads
+			totalGet += result.Downloads
+		}
+		if totalPut < throughputHighestPut && totalGet < throughputHighestGet {
+			break
+		}
+
+		if totalPut > throughputHighestPut {
+			throughputHighestPut = totalPut
+			throughputHighestPutResults = results
+		}
+		if totalGet > throughputHighestGet {
+			throughputHighestGet = totalGet
+			throughputHighestGetResults = results
+		}
+		if !autotune {
+			break
+		}
+		// Try with a higher concurrency to see if we get better throughput
+		concurrency += (concurrency + 1) / 2
+	}
+
+	concurrency = concurrencyStart
+	iopsHighestPut := uint64(0)
+	iopsHighestGet := uint64(0)
+	var iopsHighestPutResults []SpeedtestResult
+	var iopsHighestGetResults []SpeedtestResult
+
+	if autotune {
+		for {
+			select {
+			case <-ctx.Done():
+				// If the client got disconnected stop the speedtest.
+				return result, errUnexpected
+			default:
+			}
+			results := globalNotificationSys.Speedtest(ctx, iopsSize, concurrency, duration)
+			sort.Slice(results, func(i, j int) bool {
+				return results[i].Endpoint < results[j].Endpoint
+			})
+			totalPut := uint64(0)
+			totalGet := uint64(0)
+			for _, result := range results {
+				totalPut += result.Uploads
+				totalGet += result.Downloads
+			}
+			if totalPut < iopsHighestPut && totalGet < iopsHighestGet {
+				break
+			}
+			if totalPut > iopsHighestPut {
+				iopsHighestPut = totalPut
+				iopsHighestPutResults = results
+			}
+			if totalGet > iopsHighestGet {
+				iopsHighestGet = totalGet
+				iopsHighestGetResults = results
+			}
+			if !autotune {
+				break
+			}
+			// Try with a higher concurrency to see if we get better throughput
+			concurrency += (concurrency + 1) / 2
+		}
+	} else {
+		iopsHighestPut = throughputHighestPut
+		iopsHighestGet = throughputHighestGet
+		iopsHighestPutResults = throughputHighestPutResults
+		iopsHighestGetResults = throughputHighestGetResults
+	}
+
+	if len(throughputHighestPutResults) != len(iopsHighestPutResults) {
+		return result, errors.New("throughput and iops differ in number of nodes")
+	}
+
+	if len(throughputHighestGetResults) != len(iopsHighestGetResults) {
+		return result, errors.New("throughput and iops differ in number of nodes")
+	}
+
+	durationSecs := duration.Seconds()
+
+	result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs)
+	result.PUTStats.ObjectsPerSec = iopsHighestPut / uint64(iopsSize) / uint64(durationSecs)
+	for i := 0; i < len(throughputHighestPutResults); i++ {
+		errStr := ""
+		if throughputHighestPutResults[i].Error != "" {
+			errStr = throughputHighestPutResults[i].Error
+		}
+		if iopsHighestPutResults[i].Error != "" {
+			errStr = iopsHighestPutResults[i].Error
+		}
+		result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{
+			Endpoint:         throughputHighestPutResults[i].Endpoint,
+			ThroughputPerSec: throughputHighestPutResults[i].Uploads / uint64(durationSecs),
+			ObjectsPerSec:    iopsHighestPutResults[i].Uploads / uint64(iopsSize) / uint64(durationSecs),
+			Err:              errStr,
+		})
+	}
+
+	result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs)
+	result.GETStats.ObjectsPerSec = iopsHighestGet / uint64(iopsSize) / uint64(durationSecs)
+	for i := 0; i < len(throughputHighestGetResults); i++ {
+		errStr := ""
+		if throughputHighestGetResults[i].Error != "" {
+			errStr = throughputHighestGetResults[i].Error
+		}
+		if iopsHighestGetResults[i].Error != "" {
+			errStr = iopsHighestGetResults[i].Error
+		}
+		result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{
+			Endpoint:         throughputHighestGetResults[i].Endpoint,
+			ThroughputPerSec: throughputHighestGetResults[i].Downloads / uint64(durationSecs),
+			ObjectsPerSec:    iopsHighestGetResults[i].Downloads / uint64(iopsSize) / uint64(durationSecs),
+			Err:              errStr,
+		})
+	}
+
+	numDisks := 0
+	if pools, ok := objAPI.(*erasureServerPools); ok {
+		for _, set := range pools.serverPools {
+			numDisks = set.setCount * set.setDriveCount
+		}
+	}
+	result.Disks = numDisks
+	result.Servers = len(globalNotificationSys.peerClients) + 1
+	result.Version = Version
+
+	return result, nil
+}

cmd/xl-storage.go

@@ -991,6 +991,7 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
 
 	var xlMeta xlMetaV2
 	if !isXL2V1Format(buf) {
+		// This is both legacy and without proper version.
 		err = xlMeta.AddVersion(fi)
 		if err != nil {
 			logger.LogIf(ctx, err)
@@ -1006,7 +1007,8 @@ func (s *xlStorage) WriteMetadata(ctx context.Context, volume, path string, fi F
 	} else {
 		if err = xlMeta.Load(buf); err != nil {
 			logger.LogIf(ctx, err)
-			return err
+			// Corrupted data, reset and write.
+			xlMeta = xlMetaV2{}
 		}
 
 		if err = xlMeta.AddVersion(fi); err != nil {
@@ -1982,7 +1984,8 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 		if isXL2V1Format(dstBuf) {
 			if err = xlMeta.Load(dstBuf); err != nil {
 				logger.LogIf(s.ctx, err)
-				return err
+				// Data appears corrupt. Drop data.
+				xlMeta = xlMetaV2{}
 			}
 		} else {
 			// This code-path is to preserve the legacy data.
@@ -1990,13 +1993,13 @@ func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, f
 			var json = jsoniter.ConfigCompatibleWithStandardLibrary
 			if err := json.Unmarshal(dstBuf, xlMetaLegacy); err != nil {
 				logger.LogIf(s.ctx, err)
-				return errFileCorrupt
+				// Data appears corrupt. Drop data.
+			} else {
+				if err = xlMeta.AddLegacy(xlMetaLegacy); err != nil {
+					logger.LogIf(s.ctx, err)
+				}
+				legacyPreserved = true
 			}
-			if err = xlMeta.AddLegacy(xlMetaLegacy); err != nil {
-				logger.LogIf(s.ctx, err)
-				return errFileCorrupt
-			}
-			legacyPreserved = true
 		}
 	} else {
 		s.RLock()

dockerscripts/verify-minio.sh

@@ -3,14 +3,14 @@
 
 set -e
 
-if [ ! -x "/usr/bin/minio" ]; then
+if [ ! -x "/opt/bin/minio" ]; then
     echo "minio executable binary not found refusing to proceed"
     exit 1
 fi
 
 verify_sha256sum() {
     echo "verifying binary checksum"
-    echo "$(awk '{print $1}' /usr/bin/minio.sha256sum)  /usr/bin/minio" | sha256sum -c
+    echo "$(awk '{print $1}' /opt/bin/minio.sha256sum)  /opt/bin/minio" | sha256sum -c
 }
 
 verify_signature() {
@@ -19,7 +19,7 @@ verify_signature() {
         return
     fi
     echo "verifying binary signature"
-    minisign -VQm /usr/bin/minio -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
+    minisign -VQm /opt/bin/minio -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav
 }
 
 main() {

docs/metrics/prometheus/grafana/minio-overview.json

@@ -1539,7 +1539,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "sum by (server,api) (rate(minio_s3_requests_total{job=\"$scrape_jobs\"}[$__rate_interval]))",
+          "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=\"$scrape_jobs\"}[$__rate_interval]))",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server,api}}",
@@ -1635,7 +1635,7 @@
       "targets": [
         {
           "exemplar": true,
-          "expr": "rate(minio_s3_requests_errors_total{job=\"$scrape_jobs\"}[$__rate_interval])",
+          "expr": "sum by (server,api) (increase(minio_s3_requests_errors_total{job=\"$scrape_jobs\"}[$__rate_interval]))",
           "interval": "1m",
           "intervalFactor": 2,
           "legendFormat": "{{server,api}}",
@@ -2671,4 +2671,4 @@
   "title": "MinIO Overview",
   "uid": "TgmJnqnnk",
   "version": 18
-}
+}

docs/orchestration/docker-compose/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 # Settings and configurations that are common for all containers
 x-minio-common: &minio-common
-  image: quay.io/minio/minio:RELEASE.2021-09-03T03-56-13Z
+  image: quay.io/minio/minio:RELEASE.2021-09-09T21-37-07Z
   command: server --console-address ":9001" http://minio{1...4}/data{1...2}
   expose:
     - "9000"

docs/sts/web-identity.md

@@ -16,6 +16,14 @@ The OAuth 2.0 id_token that is provided by the web identity provider. Applicatio
 | *Length Constraints* | *Minimum length of 4. Maximum length of 2048.* |
 | *Required*           | *Yes*                                          |
 
+### WebIdentityAccessToken (MinIO Extension)
+There are situations when identity provider does not provide user claims in `id_token` instead it needs to be retrieved from UserInfo endpoint, this extension is only useful in this scenario. This is rare so use it accordingly depending on your Identity provider implementation. `access_token` is available as part of the OIDC authentication flow similar to `id_token`.
+
+| Params     | Value    |
+| :--        | :--      |
+| *Type*     | *String* |
+| *Required* | *No*     |
+
 ### Version
 Indicates STS API version information, the only supported value is '2011-06-15'. This value is borrowed from AWS STS API documentation for compatibility reasons.
 

go.mod

@@ -41,14 +41,14 @@ require (
 	github.com/mattn/go-runewidth v0.0.13 // indirect
 	github.com/miekg/dns v1.1.35
 	github.com/minio/cli v1.22.0
-	github.com/minio/console v0.9.8
+	github.com/minio/console v0.10.0
 	github.com/minio/csvparser v1.0.0
 	github.com/minio/highwayhash v1.0.2
 	github.com/minio/kes v0.14.0
-	github.com/minio/madmin-go v1.1.0
-	github.com/minio/minio-go/v7 v7.0.14-0.20210908194250-617d530ffac5
+	github.com/minio/madmin-go v1.1.5
+	github.com/minio/minio-go/v7 v7.0.14
 	github.com/minio/parquet-go v1.0.0
-	github.com/minio/pkg v1.1.2
+	github.com/minio/pkg v1.1.3
 	github.com/minio/selfupdate v0.3.1
 	github.com/minio/sha256-simd v1.0.0
 	github.com/minio/simdjson-go v0.2.1

go.sum

@@ -235,8 +235,6 @@ github.com/cockroachdb/cockroach-go/v2 v2.0.3 h1:ZA346ACHIZctef6trOTwBAEvPVm1k0u
 github.com/cockroachdb/cockroach-go/v2 v2.0.3/go.mod h1:hAuDgiVgDVkfirP9JnhXEfcXEPRKBpYdGz+l7mvYSzw=
 github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
-github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY=
-github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0=
 github.com/colinmarc/hdfs/v2 v2.2.0 h1:4AaIlTq+/sWmeqYhI0dX8bD4YrMQM990tRjm636FkGM=
 github.com/colinmarc/hdfs/v2 v2.2.0/go.mod h1:Wss6n3mtaZyRwWaqtSH+6ge01qT0rw9dJJmvoUnIQ/E=
 github.com/container-storage-interface/spec v1.1.0/go.mod h1:6URME8mwIBbpVyZV93Ce5St17xBiQJQY67NDsuohiy4=
@@ -1009,8 +1007,8 @@ github.com/minio/cli v1.22.0 h1:VTQm7lmXm3quxO917X3p+el1l0Ca5X3S4PM2ruUYO68=
 github.com/minio/cli v1.22.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
 github.com/minio/colorjson v1.0.1 h1:+hvfP8C1iMB95AT+ZFDRE+Knn9QPd9lg0CRJY9DRpos=
 github.com/minio/colorjson v1.0.1/go.mod h1:oPM3zQQY8Gz9NGtgvuBEjQ+gPZLKAGc7T+kjMlwtOgs=
-github.com/minio/console v0.9.8 h1:a3qrusK5EEAdL2o+fqOTECgSn2gf7QADatnpa+5IcAs=
-github.com/minio/console v0.9.8/go.mod h1:OWT3/G+s2/Umrvz4+DjUoyChdf4PjDwkVmJ93RezRG4=
+github.com/minio/console v0.10.0 h1:P1uxtmdIUAnLTlXJQtUFf/sGcKHlKNjFw1JGgt+8wZE=
+github.com/minio/console v0.10.0/go.mod h1:RO83E/L1fC/Cb8i+M/RZabBz5/m/AF5RJylKWlyk/Kw=
 github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU=
 github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM=
 github.com/minio/direct-csi v1.3.5-0.20210601185811-f7776f7961bf h1:wylCc/PdvdTIqYqVNEU9LJAZBanvfGY1TwTnjM3zQaA=
@@ -1024,9 +1022,8 @@ github.com/minio/kes v0.11.0/go.mod h1:mTF1Bv8YVEtQqF/B7Felp4tLee44Pp+dgI0rhCvgN
 github.com/minio/kes v0.14.0 h1:plCGm4LwR++T1P1sXsJbyFRX54CE1WRuo9PAPj6MC3Q=
 github.com/minio/kes v0.14.0/go.mod h1:OUensXz2BpgMfiogslKxv7Anyx/wj+6bFC6qA7BQcfA=
 github.com/minio/madmin-go v1.0.12/go.mod h1:BK+z4XRx7Y1v8SFWXsuLNqQqnq5BO/axJ8IDJfgyvfs=
-github.com/minio/madmin-go v1.0.17/go.mod h1:4nl9hvLWFnwCjkLfZSsZXEHgDODa2XSG6xGlIZyQ2oA=
-github.com/minio/madmin-go v1.1.0 h1:Y7YOPQafIVBpW0kwYzlt7FGi0GV23dlr/u3jRRrB4cw=
-github.com/minio/madmin-go v1.1.0/go.mod h1:4nl9hvLWFnwCjkLfZSsZXEHgDODa2XSG6xGlIZyQ2oA=
+github.com/minio/madmin-go v1.1.5 h1:xfzHwQ/KeKDQZKLqllNSyexwOPM/tvc13UdCeVMzADY=
+github.com/minio/madmin-go v1.1.5/go.mod h1:xIPJHUbyYhNDgeD9Wov5Fz5/p7DIW0u+q6Rs/+Xu2TM=
 github.com/minio/mc v0.0.0-20210626002108-cebf3318546f h1:hyFvo5hSFw2K417YvDr/vAKlgCG69uTuhZW/5LNdL0U=
 github.com/minio/mc v0.0.0-20210626002108-cebf3318546f/go.mod h1:tuaonkPjVApCXkbtKENHBtsqUf7YTV33qmFrC+Pgp5g=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
@@ -1035,9 +1032,8 @@ github.com/minio/md5-simd v1.1.1/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77Z
 github.com/minio/minio-go/v7 v7.0.10/go.mod h1:td4gW1ldOsj1PbSNS+WYK43j+P1XVhX/8W8awaYlBFo=
 github.com/minio/minio-go/v7 v7.0.11-0.20210302210017-6ae69c73ce78/go.mod h1:mTh2uJuAbEqdhMVl6CMIIZLUeiMiWtJR4JB8/5g2skw=
 github.com/minio/minio-go/v7 v7.0.11-0.20210607181445-e162fdb8e584/go.mod h1:WoyW+ySKAKjY98B9+7ZbI8z8S3jaxaisdcvj9TGlazA=
-github.com/minio/minio-go/v7 v7.0.13-0.20210715203016-9e713532886e/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
-github.com/minio/minio-go/v7 v7.0.14-0.20210908194250-617d530ffac5 h1:c6okzYcdOLPP9tHoOE/JxiWi5qSQpvFC6VqbA4FB/Iw=
-github.com/minio/minio-go/v7 v7.0.14-0.20210908194250-617d530ffac5/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
+github.com/minio/minio-go/v7 v7.0.14 h1:T7cw8P586gVwEEd0y21kTYtloD576XZgP62N8pE130s=
+github.com/minio/minio-go/v7 v7.0.14/go.mod h1:S23iSP5/gbMwtxeY5FM71R+TkAYyzEdoNEDDwpt8yWs=
 github.com/minio/operator v0.0.0-20210812082324-26350f153661 h1:dGAJHpfmhNukFg0M0wDqH+G1OB2YPgZCcT6uv4n9YQk=
 github.com/minio/operator v0.0.0-20210812082324-26350f153661/go.mod h1:zQqn6VGT46xlSpVXh1I/VZRv+eSgHtVu6URdg71YKX8=
 github.com/minio/operator/logsearchapi v0.0.0-20210812082324-26350f153661 h1:tJw15hS3b1dVTf5PwA4roXZ/oRNnHyZ/8Y+yNTmQ5rA=
@@ -1047,8 +1043,8 @@ github.com/minio/parquet-go v1.0.0/go.mod h1:aQlkSOfOq2AtQKkuou3mosNVMwNokd+faTa
 github.com/minio/pkg v1.0.3/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.4/go.mod h1:obU54TZ9QlMv0TRaDgQ/JTzf11ZSXxnSfLrm4tMtBP8=
 github.com/minio/pkg v1.0.8/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
-github.com/minio/pkg v1.1.2 h1:A7zSlVIQCf71DPYM1kqBpybq6fmsRY3RuTMD0ZNcqZA=
-github.com/minio/pkg v1.1.2/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
+github.com/minio/pkg v1.1.3 h1:J4vGnlNSxc/o9gDOQMZ3k0L3koA7ZgBQ7GRMrUpt/OY=
+github.com/minio/pkg v1.1.3/go.mod h1:32x/3OmGB0EOi1N+3ggnp+B5VFkSBBB9svPMVfpnf14=
 github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=
 github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
@@ -1423,10 +1419,12 @@ github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4A
 github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
 github.com/ultraware/funlen v0.0.2/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lPbfaF6xhA=
 github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA=
-github.com/unrolled/secure v1.0.7 h1:BcQHp3iKZyZCKj5gRqwQG+5urnGBF00wGgoPPwtheVQ=
-github.com/unrolled/secure v1.0.7/go.mod h1:uGc1OcRF8gCVBA+ANksKmvM85Hka6SZtQIbrKc3sHS4=
+github.com/unrolled/secure v1.0.9 h1:BWRuEb1vDrBFFDdbCnKkof3gZ35I/bnHGyt0LB0TNyQ=
+github.com/unrolled/secure v1.0.9/go.mod h1:fO+mEan+FLB0CdEnHf6Q4ZZVNqG+5fuLFnP8p0BXDPI=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc=
+github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4=
 github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM=
 github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
 github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=

helm/minio/Chart.yaml

@@ -1,8 +1,8 @@
 apiVersion: v1
 description: Multi-Cloud Object Storage
 name: minio
-version: 3.0.2
-appVersion: RELEASE.2021-09-03T03-56-13Z
+version: 3.1.1
+appVersion: RELEASE.2021-09-09T21-37-07Z
 keywords:
   - minio
   - storage

helm/minio/templates/serviceaccount.yaml

@@ -0,0 +1,7 @@
+{{- if .Values.serviceAccount.create -}}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.serviceAccount.name | quote }}
+  namespace: {{ .Release.Namespace | quote }}
+{{- end -}}

helm/minio/templates/servicemonitor.yaml

@@ -0,0 +1,41 @@
+{{- if .Values.metrics.serviceMonitor.enabled }}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: {{ template "minio.fullname" . }}
+  {{- if .Values.metrics.serviceMonitor.namespace }}
+  namespace: {{ .Values.metrics.serviceMonitor.namespace }}
+  {{- end }}
+  labels:
+    app: {{ template "minio.name" . }}
+    chart: {{ template "minio.chart" . }}
+    release: {{ .Release.Name }}
+    heritage: {{ .Release.Service }}
+    {{- if .Values.metrics.serviceMonitor.additionalLabels }}
+{{ toYaml .Values.metrics.serviceMonitor.additionalLabels | indent 4 }}
+    {{- end }}
+spec:
+  endpoints:
+    {{- if .Values.tls.enabled }}
+    - port: https
+    {{ else }}
+    - port: http
+    {{- end }}
+      path: /minio/v2/metrics/cluster
+      {{- if .Values.metrics.serviceMonitor.interval }}
+      interval: {{ .Values.metrics.serviceMonitor.interval }}
+      {{- end }}
+      {{- if .Values.metrics.serviceMonitor.scrapeTimeout }}
+      scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }}
+      {{- end }}
+      bearerTokenSecret:
+        name: {{ template "minio.fullname" . }}-prometheus
+        key: token
+  namespaceSelector:
+    matchNames:
+      - {{ .Release.Namespace | quote }}
+  selector:
+    matchLabels:
+      app: {{ include "minio.name" . }}
+      release: {{ .Release.Name }}
+{{- end }}

helm/minio/values.yaml

@@ -14,7 +14,7 @@ clusterDomain: cluster.local
 ##
 image:
   repository: quay.io/minio/minio
-  tag: RELEASE.2021-08-31T05-46-54Z
+  tag: RELEASE.2021-09-09T21-37-07Z
   pullPolicy: IfNotPresent
 
 imagePullSecrets: []
@@ -25,7 +25,7 @@ imagePullSecrets: []
 ##
 mcImage:
   repository: quay.io/minio/mc
-  tag: RELEASE.2021-07-27T06-46-19Z
+  tag: RELEASE.2021-09-02T09-21-27Z
   pullPolicy: IfNotPresent
 
 ## minio mode, i.e. standalone or distributed or gateway (nas)
@@ -318,3 +318,19 @@ networkPolicy:
 podDisruptionBudget:
   enabled: false
   maxUnavailable: 1
+
+## Specify the service account to use for the Minio pods. If 'create' is set to 'false'
+## and 'name' is left unspecified, the account 'default' will be used.
+serviceAccount:
+  create: true
+  ## The name of the service account to use. If 'create' is 'true', a service account with that name
+  ## will be created.
+  name: "minio-sa"
+
+metrics:
+  serviceMonitor:
+    enabled: false
+    additionalLabels: {}
+    # namespace: monitoring
+    # interval: 30s
+    # scrapeTimeout: 10s

index.yaml

@@ -1,9 +1,53 @@
 apiVersion: v1
 entries:
   minio:
+  - apiVersion: v1
+    appVersion: RELEASE.2021-09-09T21-37-07Z
+    created: "2021-09-13T09:42:51.343914425-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.1.1.tgz
+    version: 3.1.1
+  - apiVersion: v1
+    appVersion: RELEASE.2021-09-09T21-37-07Z
+    created: "2021-09-13T09:42:51.342547425-07:00"
+    description: Multi-Cloud Object Storage
+    digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a
+    home: https://min.io
+    icon: https://min.io/resources/img/logo/MINIO_wordmark.png
+    keywords:
+    - minio
+    - storage
+    - object-storage
+    - s3
+    - cluster
+    maintainers:
+    - email: dev@minio.io
+      name: MinIO, Inc
+    name: minio
+    sources:
+    - https://github.com/minio/minio
+    urls:
+    - https://charts.min.io/helm-releases/minio-3.1.0.tgz
+    version: 3.1.0
   - apiVersion: v1
     appVersion: RELEASE.2021-09-03T03-56-13Z
-    created: "2021-09-03T01:09:29.388403323-07:00"
+    created: "2021-09-13T09:42:51.341072255-07:00"
     description: Multi-Cloud Object Storage
     digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e
     home: https://min.io
@@ -25,7 +69,7 @@ entries:
     version: 3.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2021-09-03T01:09:29.387118116-07:00"
+    created: "2021-09-13T09:42:51.339631282-07:00"
     description: Multi-Cloud Object Storage
     digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906
     home: https://min.io
@@ -47,7 +91,7 @@ entries:
     version: 3.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2021-09-03T01:09:29.385781377-07:00"
+    created: "2021-09-13T09:42:51.338267944-07:00"
     description: Multi-Cloud Object Storage
     digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab
     home: https://min.io
@@ -69,7 +113,7 @@ entries:
     version: 3.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-31T05-46-54Z
-    created: "2021-09-03T01:09:29.384488334-07:00"
+    created: "2021-09-13T09:42:51.336838328-07:00"
     description: Multi-Cloud Object Storage
     digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b
     home: https://min.io
@@ -89,7 +133,7 @@ entries:
     version: 2.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2021-09-03T01:09:29.383253644-07:00"
+    created: "2021-09-13T09:42:51.335138412-07:00"
     description: Multi-Cloud Object Storage
     digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc
     home: https://min.io
@@ -109,7 +153,7 @@ entries:
     version: 2.0.0
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2021-09-03T01:09:29.381940948-07:00"
+    created: "2021-09-13T09:42:51.33351821-07:00"
     description: Multi-Cloud Object Storage
     digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f
     home: https://min.io
@@ -129,7 +173,7 @@ entries:
     version: 1.0.5
   - apiVersion: v1
     appVersion: RELEASE.2021-08-25T00-41-18Z
-    created: "2021-09-03T01:09:29.380699554-07:00"
+    created: "2021-09-13T09:42:51.332047714-07:00"
     description: Multi-Cloud Object Storage
     digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed
     home: https://min.io
@@ -149,7 +193,7 @@ entries:
     version: 1.0.4
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2021-09-03T01:09:29.379449808-07:00"
+    created: "2021-09-13T09:42:51.330248279-07:00"
     description: Multi-Cloud Object Storage
     digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d
     home: https://min.io
@@ -169,7 +213,7 @@ entries:
     version: 1.0.3
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2021-09-03T01:09:29.378188621-07:00"
+    created: "2021-09-13T09:42:51.328799101-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388
     home: https://min.io
@@ -189,7 +233,7 @@ entries:
     version: 1.0.2
   - apiVersion: v1
     appVersion: RELEASE.2021-08-20T18-32-01Z
-    created: "2021-09-03T01:09:29.376910712-07:00"
+    created: "2021-09-13T09:42:51.327376321-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c
     home: https://min.io
@@ -209,7 +253,7 @@ entries:
     version: 1.0.1
   - apiVersion: v1
     appVersion: RELEASE.2021-08-17T20-53-08Z
-    created: "2021-09-03T01:09:29.375397146-07:00"
+    created: "2021-09-13T09:42:51.32590408-07:00"
     description: High Performance, Kubernetes Native Object Storage
     digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188
     home: https://min.io
@@ -227,4 +271,4 @@ entries:
     urls:
     - https://charts.min.io/helm-releases/minio-1.0.0.tgz
     version: 1.0.0
-generated: "2021-09-03T01:09:28.813520489-07:00"
+generated: "2021-09-13T09:42:50.856325614-07:00"

internal/config/config.go

@@ -58,6 +58,7 @@ const (
 	RegionName = "name"
 	AccessKey  = "access_key"
 	SecretKey  = "secret_key"
+	License    = "license"
 )
 
 // Top level config constants.
@@ -79,6 +80,7 @@ const (
 	HealSubSys           = "heal"
 	ScannerSubSys        = "scanner"
 	CrawlerSubSys        = "crawler"
+	SubnetSubSys         = "subnet"
 
 	// Add new constants here if you add new fields to config.
 )
@@ -127,6 +129,7 @@ var SubSystems = set.CreateStringSet(
 	NotifyPostgresSubSys,
 	NotifyRedisSubSys,
 	NotifyWebhookSubSys,
+	SubnetSubSys,
 )
 
 // SubSystemsDynamic - all sub-systems that have dynamic config.
@@ -135,6 +138,7 @@ var SubSystemsDynamic = set.CreateStringSet(
 	CompressionSubSys,
 	ScannerSubSys,
 	HealSubSys,
+	SubnetSubSys,
 )
 
 // SubSystemsSingleTargets - subsystems which only support single target.

internal/config/identity/openid/help.go

@@ -44,15 +44,21 @@ var (
 			Optional:    true,
 			Type:        "string",
 		},
+		config.HelpKV{
+			Key:         ClaimUserinfo,
+			Description: `Enable fetching claims from UserInfo Endpoint for authenticated user`,
+			Optional:    true,
+			Type:        "on|off",
+		},
 		config.HelpKV{
 			Key:         ClaimPrefix,
-			Description: `JWT claim namespace prefix e.g. "customer1/"`,
+			Description: `[DEPRECATED use 'claim_name'] JWT claim namespace prefix e.g. "customer1/"`,
 			Optional:    true,
 			Type:        "string",
 		},
 		config.HelpKV{
 			Key:         RedirectURI,
-			Description: `Configure custom redirect_uri for OpenID login flow callback`,
+			Description: `[DEPRECATED use env 'MINIO_BROWSER_REDIRECT_URL'] Configure custom redirect_uri for OpenID login flow callback`,
 			Optional:    true,
 			Type:        "string",
 		},

internal/config/identity/openid/jwt.go

@@ -47,13 +47,14 @@ type Config struct {
 	JWKS    struct {
 		URL *xnet.URL `json:"url"`
 	} `json:"jwks"`
-	URL          *xnet.URL `json:"url,omitempty"`
-	ClaimPrefix  string    `json:"claimPrefix,omitempty"`
-	ClaimName    string    `json:"claimName,omitempty"`
-	RedirectURI  string    `json:"redirectURI,omitempty"`
-	DiscoveryDoc DiscoveryDoc
-	ClientID     string
-	ClientSecret string
+	URL           *xnet.URL `json:"url,omitempty"`
+	ClaimPrefix   string    `json:"claimPrefix,omitempty"`
+	ClaimName     string    `json:"claimName,omitempty"`
+	ClaimUserinfo bool      `json:"claimUserInfo,omitempty"`
+	RedirectURI   string    `json:"redirectURI,omitempty"`
+	DiscoveryDoc  DiscoveryDoc
+	ClientID      string
+	ClientSecret  string
 
 	provider    provider.Provider
 	publicKeys  map[string]crypto.PublicKey
@@ -61,6 +62,63 @@ type Config struct {
 	closeRespFn func(io.ReadCloser)
 }
 
+// UserInfo returns claims for authenticated user from userInfo endpoint.
+//
+// Some OIDC implementations such as GitLab do not support
+// claims as part of the normal oauth2 flow, instead rely
+// on service providers making calls to IDP to fetch additional
+// claims available from the UserInfo endpoint
+func (r *Config) UserInfo(accessToken string) (map[string]interface{}, error) {
+	if r.JWKS.URL == nil || r.JWKS.URL.String() == "" {
+		return nil, errors.New("openid not configured")
+	}
+	transport := http.DefaultTransport
+	if r.transport != nil {
+		transport = r.transport
+	}
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	req, err := http.NewRequest(http.MethodPost, r.DiscoveryDoc.UserInfoEndpoint, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if accessToken != "" {
+		req.Header.Set("Authorization", "Bearer "+accessToken)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	defer r.closeRespFn(resp.Body)
+	if resp.StatusCode != http.StatusOK {
+		// uncomment this for debugging when needed.
+		// reqBytes, _ := httputil.DumpRequest(req, false)
+		// fmt.Println(string(reqBytes))
+		// respBytes, _ := httputil.DumpResponse(resp, true)
+		// fmt.Println(string(respBytes))
+		return nil, errors.New(resp.Status)
+	}
+
+	dec := json.NewDecoder(resp.Body)
+	var claims = map[string]interface{}{}
+
+	if err = dec.Decode(&claims); err != nil {
+		// uncomment this for debugging when needed.
+		// reqBytes, _ := httputil.DumpRequest(req, false)
+		// fmt.Println(string(reqBytes))
+		// respBytes, _ := httputil.DumpResponse(resp, true)
+		// fmt.Println(string(respBytes))
+		return nil, err
+	}
+
+	return claims, nil
+}
+
 // LookupUser lookup userid for the provider
 func (r Config) LookupUser(userid string) (provider.User, error) {
 	if r.provider != nil {
@@ -122,9 +180,6 @@ func (r *Config) InitializeKeycloakProvider(adminURL, realm string) error {
 
 // PopulatePublicKey - populates a new publickey from the JWKS URL.
 func (r *Config) PopulatePublicKey() error {
-	r.Lock()
-	defer r.Unlock()
-
 	if r.JWKS.URL == nil || r.JWKS.URL.String() == "" {
 		return nil
 	}
@@ -135,6 +190,10 @@ func (r *Config) PopulatePublicKey() error {
 	client := &http.Client{
 		Transport: transport,
 	}
+
+	r.Lock()
+	defer r.Unlock()
+
 	resp, err := client.Get(r.JWKS.URL.String())
 	if err != nil {
 		return err
@@ -234,7 +293,7 @@ func updateClaimsExpiry(dsecs string, claims map[string]interface{}) error {
 }
 
 // Validate - validates the id_token.
-func (r *Config) Validate(token, dsecs string) (map[string]interface{}, error) {
+func (r *Config) Validate(token, accessToken, dsecs string) (map[string]interface{}, error) {
 	jp := new(jwtgo.Parser)
 	jp.ValidMethods = []string{
 		"RS256", "RS384", "RS512", "ES256", "ES384", "ES512",
@@ -273,6 +332,23 @@ func (r *Config) Validate(token, dsecs string) (map[string]interface{}, error) {
 		return nil, err
 	}
 
+	// If claim user info is enabled, get claims from userInfo
+	// and overwrite them with the claims from JWT.
+	if r.ClaimUserinfo {
+		if accessToken == "" {
+			return nil, errors.New("access_token is mandatory if user_info claim is enabled")
+		}
+		uclaims, err := r.UserInfo(accessToken)
+		if err != nil {
+			return nil, err
+		}
+		for k, v := range uclaims {
+			if _, ok := claims[k]; !ok { // only add to claims not update it.
+				claims[k] = v
+			}
+		}
+	}
+
 	return claims, nil
 }
 
@@ -283,29 +359,32 @@ func (Config) ID() ID {
 
 // OpenID keys and envs.
 const (
-	JwksURL      = "jwks_url"
-	ConfigURL    = "config_url"
-	ClaimName    = "claim_name"
-	ClaimPrefix  = "claim_prefix"
-	ClientID     = "client_id"
-	ClientSecret = "client_secret"
-	Vendor       = "vendor"
-	Scopes       = "scopes"
-	RedirectURI  = "redirect_uri"
+	JwksURL       = "jwks_url"
+	ConfigURL     = "config_url"
+	ClaimName     = "claim_name"
+	ClaimUserinfo = "claim_userinfo"
+	ClaimPrefix   = "claim_prefix"
+	ClientID      = "client_id"
+	ClientSecret  = "client_secret"
+
+	Vendor      = "vendor"
+	Scopes      = "scopes"
+	RedirectURI = "redirect_uri"
 
 	// Vendor specific ENV only enabled if the Vendor matches == "vendor"
 	KeyCloakRealm    = "keycloak_realm"
 	KeyCloakAdminURL = "keycloak_admin_url"
 
-	EnvIdentityOpenIDVendor       = "MINIO_IDENTITY_OPENID_VENDOR"
-	EnvIdentityOpenIDClientID     = "MINIO_IDENTITY_OPENID_CLIENT_ID"
-	EnvIdentityOpenIDClientSecret = "MINIO_IDENTITY_OPENID_CLIENT_SECRET"
-	EnvIdentityOpenIDJWKSURL      = "MINIO_IDENTITY_OPENID_JWKS_URL"
-	EnvIdentityOpenIDURL          = "MINIO_IDENTITY_OPENID_CONFIG_URL"
-	EnvIdentityOpenIDClaimName    = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
-	EnvIdentityOpenIDClaimPrefix  = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
-	EnvIdentityOpenIDRedirectURI  = "MINIO_IDENTITY_OPENID_REDIRECT_URI"
-	EnvIdentityOpenIDScopes       = "MINIO_IDENTITY_OPENID_SCOPES"
+	EnvIdentityOpenIDVendor        = "MINIO_IDENTITY_OPENID_VENDOR"
+	EnvIdentityOpenIDClientID      = "MINIO_IDENTITY_OPENID_CLIENT_ID"
+	EnvIdentityOpenIDClientSecret  = "MINIO_IDENTITY_OPENID_CLIENT_SECRET"
+	EnvIdentityOpenIDJWKSURL       = "MINIO_IDENTITY_OPENID_JWKS_URL"
+	EnvIdentityOpenIDURL           = "MINIO_IDENTITY_OPENID_CONFIG_URL"
+	EnvIdentityOpenIDClaimName     = "MINIO_IDENTITY_OPENID_CLAIM_NAME"
+	EnvIdentityOpenIDClaimUserInfo = "MINIO_IDENTITY_OPENID_CLAIM_USERINFO"
+	EnvIdentityOpenIDClaimPrefix   = "MINIO_IDENTITY_OPENID_CLAIM_PREFIX"
+	EnvIdentityOpenIDRedirectURI   = "MINIO_IDENTITY_OPENID_REDIRECT_URI"
+	EnvIdentityOpenIDScopes        = "MINIO_IDENTITY_OPENID_SCOPES"
 
 	// Vendor specific ENVs only enabled if the Vendor matches == "vendor"
 	EnvIdentityOpenIDKeyCloakRealm    = "MINIO_IDENTITY_OPENID_KEYCLOAK_REALM"
@@ -374,6 +453,10 @@ var (
 			Key:   ClaimName,
 			Value: iampolicy.PolicyName,
 		},
+		config.KV{
+			Key:   ClaimUserinfo,
+			Value: "",
+		},
 		config.KV{
 			Key:   ClaimPrefix,
 			Value: "",
@@ -410,15 +493,16 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 	}
 
 	c = Config{
-		RWMutex:      &sync.RWMutex{},
-		ClaimName:    env.Get(EnvIdentityOpenIDClaimName, kvs.Get(ClaimName)),
-		ClaimPrefix:  env.Get(EnvIdentityOpenIDClaimPrefix, kvs.Get(ClaimPrefix)),
-		RedirectURI:  env.Get(EnvIdentityOpenIDRedirectURI, kvs.Get(RedirectURI)),
-		publicKeys:   make(map[string]crypto.PublicKey),
-		ClientID:     env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
-		ClientSecret: env.Get(EnvIdentityOpenIDClientSecret, kvs.Get(ClientSecret)),
-		transport:    transport,
-		closeRespFn:  closeRespFn,
+		RWMutex:       &sync.RWMutex{},
+		ClaimName:     env.Get(EnvIdentityOpenIDClaimName, kvs.Get(ClaimName)),
+		ClaimUserinfo: env.Get(EnvIdentityOpenIDClaimUserInfo, kvs.Get(ClaimUserinfo)) == config.EnableOn,
+		ClaimPrefix:   env.Get(EnvIdentityOpenIDClaimPrefix, kvs.Get(ClaimPrefix)),
+		RedirectURI:   env.Get(EnvIdentityOpenIDRedirectURI, kvs.Get(RedirectURI)),
+		publicKeys:    make(map[string]crypto.PublicKey),
+		ClientID:      env.Get(EnvIdentityOpenIDClientID, kvs.Get(ClientID)),
+		ClientSecret:  env.Get(EnvIdentityOpenIDClientSecret, kvs.Get(ClientSecret)),
+		transport:     transport,
+		closeRespFn:   closeRespFn,
 	}
 
 	configURL := env.Get(EnvIdentityOpenIDURL, kvs.Get(ConfigURL))
@@ -433,6 +517,10 @@ func LookupConfig(kvs config.KVS, transport *http.Transport, closeRespFn func(io
 		}
 	}
 
+	if c.ClaimUserinfo && configURL == "" {
+		return c, errors.New("please specify config_url to enable fetching claims from UserInfo endpoint")
+	}
+
 	if scopeList := env.Get(EnvIdentityOpenIDScopes, kvs.Get(Scopes)); scopeList != "" {
 		var scopes []string
 		for _, scope := range strings.Split(scopeList, ",") {

internal/config/identity/openid/jwt_test.go

@@ -100,7 +100,7 @@ func TestJWTAzureFail(t *testing.T) {
 		t.Fatalf("Unexpected id %s for the validator", cfg.ID())
 	}
 
-	if _, err := cfg.Validate(jwtToken, ""); err == nil {
+	if _, err := cfg.Validate(jwtToken, "", ""); err == nil {
 		// Azure should fail due to non OIDC compliant JWT
 		// generated by Azure AD
 		t.Fatal(err)
@@ -154,7 +154,7 @@ func TestJWT(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err := cfg.Validate(u.Query().Get("Token"), ""); err == nil {
+	if _, err := cfg.Validate(u.Query().Get("Token"), "", ""); err == nil {
 		t.Fatal(err)
 	}
 }

internal/config/identity/openid/validators.go

@@ -31,7 +31,7 @@ type ID string
 type Validator interface {
 	// Validate is a custom validator function for this provider,
 	// each validation is authenticationType or provider specific.
-	Validate(token string, duration string) (map[string]interface{}, error)
+	Validate(idToken, accessToken, duration string) (map[string]interface{}, error)
 
 	// ID returns provider name of this provider.
 	ID() ID

internal/config/identity/openid/validators_test.go

@@ -27,7 +27,7 @@ import (
 
 type errorValidator struct{}
 
-func (e errorValidator) Validate(token, dsecs string) (map[string]interface{}, error) {
+func (e errorValidator) Validate(idToken, accessToken, dsecs string) (map[string]interface{}, error) {
 	return nil, ErrTokenExpired
 }
 
@@ -79,7 +79,7 @@ func TestValidators(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	if _, err = v.Validate("", ""); err != ErrTokenExpired {
+	if _, err = v.Validate("", "", ""); err != ErrTokenExpired {
 		t.Fatalf("Expected error %s, got %s", ErrTokenExpired, err)
 	}
 

internal/config/subnet/license.go

@@ -0,0 +1,71 @@
+// Copyright (c) 2015-2021 MinIO, Inc.
+//
+// This file is part of MinIO Object Storage stack
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package subnet
+
+import (
+	jwtgo "github.com/golang-jwt/jwt"
+	"github.com/minio/minio/internal/config"
+	"github.com/minio/pkg/env"
+)
+
+var (
+	// DefaultKVS - default KV config for subnet settings
+	DefaultKVS = config.KVS{
+		config.KV{
+			Key:   config.License,
+			Value: "",
+		},
+	}
+
+	// HelpLicense - provides help for license config
+	HelpLicense = config.HelpKVS{
+		config.HelpKV{
+			Key:         config.License,
+			Type:        "string",
+			Description: "Subnet license token for the cluster",
+			Optional:    true,
+		},
+	}
+)
+
+// Config represents the subnet related configuration
+type Config struct {
+	// The subnet license token
+	License string `json:"license"`
+}
+
+func validateLicenseFormat(lic string) error {
+	if len(lic) == 0 {
+		return nil
+	}
+
+	// Only verifying that the string is a parseable JWT token as of now
+	_, _, err := new(jwtgo.Parser).ParseUnverified(lic, jwtgo.MapClaims{})
+	return err
+}
+
+// LookupConfig - lookup config and override with valid environment settings if any.
+func LookupConfig(kvs config.KVS) (cfg Config, err error) {
+	if err = config.CheckValidKeys(config.SubnetSubSys, kvs, DefaultKVS); err != nil {
+		return cfg, err
+	}
+
+	cfg.License = env.Get(config.EnvMinIOSubnetLicense, kvs.Get(config.License))
+
+	return cfg, validateLicenseFormat(cfg.License)
+}

internal/crypto/sse-kms.go

@@ -210,8 +210,8 @@ func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []
 			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal KMS context is not base64-encoded")
 		}
 		var json = jsoniter.ConfigCompatibleWithStandardLibrary
-		if err = json.Unmarshal(b, ctx); err != nil {
-			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS context is invalid")
+		if err = json.Unmarshal(b, &ctx); err != nil {
+			return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS context is invalid %w", err)
 		}
 	}
 

internal/logger/target/http/http.go

@@ -98,7 +98,7 @@ func (h *Target) Init() error {
 	// Drain any response.
 	xhttp.DrainBody(resp.Body)
 
-	if resp.StatusCode != http.StatusOK {
+	if !acceptedResponseStatusCode(resp.StatusCode) {
 		switch resp.StatusCode {
 		case http.StatusForbidden:
 			return fmt.Errorf("%s returned '%s', please check if your auth token is correctly set",
@@ -112,6 +112,13 @@ func (h *Target) Init() error {
 	return nil
 }
 
+// Accepted HTTP Status Codes
+var acceptedStatusCodeMap = map[int]bool{http.StatusOK: true, http.StatusCreated: true, http.StatusAccepted: true, http.StatusNoContent: true}
+
+func acceptedResponseStatusCode(code int) bool {
+	return acceptedStatusCodeMap[code]
+}
+
 func (h *Target) startHTTPLogger() {
 	// Create a routine which sends json logs received
 	// from an internal channel.

internal/s3select/csv/record.go

@@ -22,6 +22,8 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"strconv"
+	"strings"
 
 	"github.com/bcicen/jstream"
 	csv "github.com/minio/csvparser"
@@ -43,6 +45,20 @@ type Record struct {
 func (r *Record) Get(name string) (*sql.Value, error) {
 	index, found := r.nameIndexMap[name]
 	if !found {
+		// Check if index.
+		if strings.HasPrefix(name, "_") {
+			idx, err := strconv.Atoi(strings.TrimPrefix(name, "_"))
+			if err != nil {
+				return nil, fmt.Errorf("column %v not found", name)
+			}
+			// The position count starts at 1.
+			idx--
+			if idx >= len(r.csvRecord) || idx < 0 {
+				// If field index > number of columns, return null
+				return sql.FromNull(), nil
+			}
+			return sql.FromBytes([]byte(r.csvRecord[idx])), nil
+		}
 		return nil, fmt.Errorf("column %v not found", name)
 	}
 

internal/s3select/select_test.go

@@ -675,6 +675,11 @@ func TestCSVQueries2(t *testing.T) {
 			query:      `SELECT num2 from s3object s WHERE num2 = 0.765111`,
 			wantResult: `{"num2":" 0.765111"}`,
 		},
+		{
+			name:       "select-non_exiting_values",
+			query:      `SELECT _1 as first, s._100 from s3object s LIMIT 1`,
+			wantResult: `{"first":"1","_100":null}`,
+		},
 	}
 
 	defRequest := `<?xml version="1.0" encoding="UTF-8"?>