use the correct endpoints for offline drives

This PR also preserves source ETag for replication

.github/workflows/go.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.13.x]
+        go-version: [1.14.x]
         os: [ubuntu-latest, windows-latest]
     steps:
       - uses: actions/checkout@v2

cmd/api-headers.go

@@ -128,6 +128,11 @@ func setObjectHeaders(w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSp
 			// values to client.
 			continue
 		}
+
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+			continue
+		}
 		var isSet bool
 		for _, userMetadataPrefix := range userMetadataKeyPrefixes {
 			if !strings.HasPrefix(k, userMetadataPrefix) {

cmd/api-response.go

@@ -564,6 +564,10 @@ func generateListObjectsV2Response(bucket, prefix, token, nextToken, startAfter,
 					// values to client.
 					continue
 				}
+				// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+				if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+					continue
+				}
 				content.UserMetadata[k] = v
 			}
 		}

cmd/bucket-replication.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"net/http"
+	"strings"
 	"time"
 
 	miniogo "github.com/minio/minio-go/v7"
@@ -75,8 +76,23 @@ func validateReplicationDestination(ctx context.Context, bucket string, rCfg *re
 	return false, BucketRemoteTargetNotFound{Bucket: bucket}
 }
 
+func mustReplicateWeb(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string, permErr APIErrorCode) bool {
+	if permErr != ErrNone {
+		return false
+	}
+	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+}
+
 // mustReplicate returns true if object meets replication criteria.
 func mustReplicate(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
+	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
+		return false
+	}
+	return mustReplicater(ctx, r, bucket, object, meta, replStatus)
+}
+
+// mustReplicater returns true if object meets replication criteria.
+func mustReplicater(ctx context.Context, r *http.Request, bucket, object string, meta map[string]string, replStatus string) bool {
 	if globalIsGateway {
 		return false
 	}
@@ -86,9 +102,6 @@ func mustReplicate(ctx context.Context, r *http.Request, bucket, object string,
 	if replication.StatusType(replStatus) == replication.Replica {
 		return false
 	}
-	if s3Err := isPutActionAllowed(getRequestAuthType(r), bucket, object, r, iampolicy.GetReplicationConfigurationAction); s3Err != ErrNone {
-		return false
-	}
 	cfg, err := getReplicationConfig(ctx, bucket)
 	if err != nil {
 		return false
@@ -110,9 +123,11 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 		if k == xhttp.AmzBucketReplicationStatus {
 			continue
 		}
+		if strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower) {
+			continue
+		}
 		meta[k] = v
 	}
-
 	tag, err := tags.ParseObjectTags(objInfo.UserTags)
 	if err != nil {
 		return
@@ -130,6 +145,7 @@ func putReplicationOpts(dest replication.Destination, objInfo ObjectInfo) (putOp
 		ReplicationVersionID: objInfo.VersionID,
 		ReplicationStatus:    miniogo.ReplicationStatusReplica,
 		ReplicationMTime:     objInfo.ModTime,
+		ReplicationETag:      objInfo.ETag,
 	}
 	if mode, ok := objInfo.UserDefined[xhttp.AmzObjectLockMode]; ok {
 		rmode := miniogo.RetentionMode(mode)
@@ -219,3 +235,26 @@ func replicateObject(ctx context.Context, bucket, object, versionID string, obje
 		logger.LogIf(ctx, err)
 	}
 }
+
+// filterReplicationStatusMetadata filters replication status metadata for COPY
+func filterReplicationStatusMetadata(metadata map[string]string) map[string]string {
+	// Copy on write
+	dst := metadata
+	var copied bool
+	delKey := func(key string) {
+		if _, ok := metadata[key]; !ok {
+			return
+		}
+		if !copied {
+			dst = make(map[string]string, len(metadata))
+			for k, v := range metadata {
+				dst[k] = v
+			}
+			copied = true
+		}
+		delete(dst, key)
+	}
+
+	delKey(xhttp.AmzBucketReplicationStatus)
+	return dst
+}

cmd/bucket-targets.go

@@ -90,11 +90,14 @@ func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *m
 			return BucketReplicationSourceNotVersioned{Bucket: bucket}
 		}
 		vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket)
-		if err != nil || vcfg.Status != string(versioning.Enabled) {
+		if err != nil {
 			if isErrBucketNotFound(err) {
 				return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket}
 			}
-			return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
+			if vcfg.Status != string(versioning.Enabled) {
+				return BucketReplicationTargetNotVersioned{Bucket: tgt.TargetBucket}
+			}
+			return err
 		}
 	}
 

cmd/config/certs.go

@@ -24,8 +24,6 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"io/ioutil"
-	"os"
-	"path"
 
 	"github.com/minio/minio/pkg/env"
 )
@@ -69,38 +67,6 @@ func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err er
 	return x509Certs, nil
 }
 
-// GetRootCAs - returns all the root CAs into certPool
-// at the input certsCADir
-func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
-	rootCAs, _ := x509.SystemCertPool()
-	if rootCAs == nil {
-		// In some systems (like Windows) system cert pool is
-		// not supported or no certificates are present on the
-		// system - so we create a new cert pool.
-		rootCAs = x509.NewCertPool()
-	}
-
-	fis, err := ioutil.ReadDir(certsCAsDir)
-	if err != nil {
-		if os.IsNotExist(err) || os.IsPermission(err) {
-			// Return success if CA's directory is missing or permission denied.
-			err = nil
-		}
-		return rootCAs, err
-	}
-
-	// Load all custom CA files.
-	for _, fi := range fis {
-		caCert, err := ioutil.ReadFile(path.Join(certsCAsDir, fi.Name()))
-		if err != nil {
-			// ignore files which are not readable.
-			continue
-		}
-		rootCAs.AppendCertsFromPEM(caCert)
-	}
-	return rootCAs, nil
-}
-
 // LoadX509KeyPair - load an X509 key pair (private key , certificate)
 // from the provided paths. The private key may be encrypted and is
 // decrypted using the ENV_VAR: MINIO_CERT_PASSWD.

cmd/config/certs_test.go

@@ -20,7 +20,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
-	"path/filepath"
 	"runtime"
 	"testing"
 )
@@ -194,60 +193,6 @@ M9ofSEt/bdRD
 	}
 }
 
-func TestGetRootCAs(t *testing.T) {
-	emptydir, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", emptydir)
-	}
-	defer os.RemoveAll(emptydir)
-
-	dir1, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", dir1)
-	}
-	defer os.RemoveAll(dir1)
-	if err = os.Mkdir(filepath.Join(dir1, "empty-dir"), 0755); err != nil {
-		t.Fatalf("Unable create empty dir. %v", err)
-	}
-
-	dir2, err := ioutil.TempDir("", "test-get-root-cas")
-	if err != nil {
-		t.Fatalf("Unable create temp directory. %v", dir2)
-	}
-	defer os.RemoveAll(dir2)
-	if err = ioutil.WriteFile(filepath.Join(dir2, "empty-file"), []byte{}, 0644); err != nil {
-		t.Fatalf("Unable create test file. %v", err)
-	}
-
-	testCases := []struct {
-		certCAsDir  string
-		expectedErr error
-	}{
-		// ignores non-existent directories.
-		{"nonexistent-dir", nil},
-		// Ignores directories.
-		{dir1, nil},
-		// Ignore empty directory.
-		{emptydir, nil},
-		// Loads the cert properly.
-		{dir2, nil},
-	}
-
-	for _, testCase := range testCases {
-		_, err := GetRootCAs(testCase.certCAsDir)
-
-		if testCase.expectedErr == nil {
-			if err != nil {
-				t.Fatalf("error: expected = <nil>, got = %v", err)
-			}
-		} else if err == nil {
-			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
-		} else if testCase.expectedErr.Error() != err.Error() {
-			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
-		}
-	}
-}
-
 func TestLoadX509KeyPair(t *testing.T) {
 	for i, testCase := range loadX509KeyPairTests {
 		privateKey, err := createTempFile("private.key", testCase.privateKey)

cmd/crypto/header.go

@@ -21,6 +21,8 @@ import (
 	"encoding/json"
 	"net/http"
 	"strings"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // SSEHeader is the general AWS SSE HTTP header key.
@@ -81,6 +83,8 @@ const (
 func RemoveSensitiveHeaders(h http.Header) {
 	h.Del(SSECKey)
 	h.Del(SSECopyKey)
+	h.Del(xhttp.AmzMetaUnencryptedContentLength)
+	h.Del(xhttp.AmzMetaUnencryptedContentMD5)
 }
 
 // IsRequested returns true if the HTTP headers indicates

cmd/crypto/header_test.go

@@ -457,6 +457,16 @@ var removeSensitiveHeadersTests = []struct {
 			"X-Amz-Meta-Test-1": []string{"Test-1"},
 		},
 	},
+	{ // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		Header: http.Header{
+			"X-Amz-Meta-X-Amz-Unencrypted-Content-Md5":    []string{"value"},
+			"X-Amz-Meta-X-Amz-Unencrypted-Content-Length": []string{"value"},
+			"X-Amz-Meta-Test-1":                           []string{"Test-1"},
+		},
+		ExpectedHeader: http.Header{
+			"X-Amz-Meta-Test-1": []string{"Test-1"},
+		},
+	},
 }
 
 func TestRemoveSensitiveHeaders(t *testing.T) {

cmd/crypto/metadata.go

@@ -19,6 +19,7 @@ import (
 	"encoding/base64"
 	"errors"
 
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -38,6 +39,8 @@ func IsMultiPart(metadata map[string]string) bool {
 func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons
 	delete(metadata, SSECKey)
 	delete(metadata, SSECopyKey)
+	delete(metadata, xhttp.AmzMetaUnencryptedContentLength)
+	delete(metadata, xhttp.AmzMetaUnencryptedContentMD5)
 }
 
 // RemoveSSEHeaders removes all crypto-specific SSE

cmd/erasure-multipart.go

@@ -702,6 +702,9 @@ func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket str
 
 	// Save successfully calculated md5sum.
 	fi.Metadata["etag"] = s3MD5
+	if opts.UserDefined["etag"] != "" { // preserve ETag if set
+		fi.Metadata["etag"] = opts.UserDefined["etag"]
+	}
 
 	// Save the consolidated actual size.
 	fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10)

cmd/erasure-object.go

@@ -710,8 +710,9 @@ func (er erasureObjects) putObject(ctx context.Context, bucket string, object st
 			Hash:       bitrotWriterSum(w),
 		})
 	}
-
-	opts.UserDefined["etag"] = r.MD5CurrentHexString()
+	if opts.UserDefined["etag"] == "" {
+		opts.UserDefined["etag"] = r.MD5CurrentHexString()
+	}
 
 	// Guess content-type from the extension if possible.
 	if opts.UserDefined["content-type"] == "" {

cmd/erasure-zones.go

@@ -2088,6 +2088,14 @@ func (z *erasureZones) Health(ctx context.Context, opts HealthOptions) HealthRes
 		}
 	}
 
+	// when maintenance is not specified we don't have
+	// to look at the healing side of the code.
+	if !opts.Maintenance {
+		return HealthResult{
+			Healthy: true,
+		}
+	}
+
 	// check if local disks are being healed, if they are being healed
 	// we need to tell healthy status as 'false' so that this server
 	// is not taken down for maintenance

cmd/erasure.go

@@ -167,7 +167,7 @@ func getDisksInfo(disks []StorageAPI, endpoints []string) (disksInfo []madmin.Di
 	errs = g.Wait()
 	// Wait for the routines.
 	for i, diskInfoErr := range errs {
-		ep := endpoints[i]
+		ep := disksInfo[i].Endpoint
 		if diskInfoErr != nil {
 			offlineDisks[ep]++
 			continue

cmd/fs-v1.go

@@ -906,6 +906,56 @@ func (fs *FSObjects) defaultFsJSON(object string) fsMetaV1 {
 	return fsMeta
 }
 
+func (fs *FSObjects) getObjectInfoNoFSLock(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
+	fsMeta := fsMetaV1{}
+	if HasSuffix(object, SlashSeparator) {
+		fi, err := fsStatDir(ctx, pathJoin(fs.fsPath, bucket, object))
+		if err != nil {
+			return oi, err
+		}
+		return fsMeta.ToObjectInfo(bucket, object, fi), nil
+	}
+
+	fsMetaPath := pathJoin(fs.fsPath, minioMetaBucket, bucketMetaPrefix, bucket, object, fs.metaJSONFile)
+	// Read `fs.json` to perhaps contend with
+	// parallel Put() operations.
+
+	rc, _, err := fsOpenFile(ctx, fsMetaPath, 0)
+	if err == nil {
+		fsMetaBuf, rerr := ioutil.ReadAll(rc)
+		rc.Close()
+		if rerr == nil {
+			var json = jsoniter.ConfigCompatibleWithStandardLibrary
+			if rerr = json.Unmarshal(fsMetaBuf, &fsMeta); rerr != nil {
+				// For any error to read fsMeta, set default ETag and proceed.
+				fsMeta = fs.defaultFsJSON(object)
+			}
+		} else {
+			// For any error to read fsMeta, set default ETag and proceed.
+			fsMeta = fs.defaultFsJSON(object)
+		}
+	}
+
+	// Return a default etag and content-type based on the object's extension.
+	if err == errFileNotFound {
+		fsMeta = fs.defaultFsJSON(object)
+	}
+
+	// Ignore if `fs.json` is not available, this is true for pre-existing data.
+	if err != nil && err != errFileNotFound {
+		logger.LogIf(ctx, err)
+		return oi, err
+	}
+
+	// Stat the file to get file size.
+	fi, err := fsStatFile(ctx, pathJoin(fs.fsPath, bucket, object))
+	if err != nil {
+		return oi, err
+	}
+
+	return fsMeta.ToObjectInfo(bucket, object, fi), nil
+}
+
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (fs *FSObjects) getObjectInfo(ctx context.Context, bucket, object string) (oi ObjectInfo, e error) {
 	fsMeta := fsMetaV1{}
@@ -1391,14 +1441,13 @@ func (fs *FSObjects) ListObjectVersions(ctx context.Context, bucket, prefix, mar
 // ListObjects - list all objects at prefix upto maxKeys., optionally delimited by '/'. Maintains the list pool
 // state for future re-entrant list requests.
 func (fs *FSObjects) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (loi ListObjectsInfo, e error) {
-
 	atomic.AddInt64(&fs.activeIOCount, 1)
 	defer func() {
 		atomic.AddInt64(&fs.activeIOCount, -1)
 	}()
 
 	return listObjects(ctx, fs, bucket, prefix, marker, delimiter, maxKeys, fs.listPool,
-		fs.listDirFactory(), fs.getObjectInfo, fs.getObjectInfo)
+		fs.listDirFactory(), fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock)
 }
 
 // GetObjectTags - get object tags from an existing object
@@ -1495,7 +1544,7 @@ func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remo
 // error walker returns error. Optionally if context.Done() is received
 // then Walk() stops the walker.
 func (fs *FSObjects) Walk(ctx context.Context, bucket, prefix string, results chan<- ObjectInfo, opts ObjectOptions) error {
-	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), results, fs.getObjectInfo, fs.getObjectInfo)
+	return fsWalk(ctx, fs, bucket, prefix, fs.listDirFactory(), results, fs.getObjectInfoNoFSLock, fs.getObjectInfoNoFSLock)
 }
 
 // HealObjects - no-op for fs. Valid only for Erasure.

cmd/gateway-main.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/minio/cli"
-	"github.com/minio/minio/cmd/config"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/certs"
@@ -178,7 +177,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	logger.FatalIf(err, "Invalid TLS certificate file")
 
 	// Check and load Root CAs.
-	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
+	globalRootCAs, err = certs.GetRootCAs(globalCertsCADir.Get())
 	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
 
 	// Register root CAs for remote ENVs

cmd/handler-utils.go

@@ -131,6 +131,13 @@ func extractMetadata(ctx context.Context, r *http.Request) (metadata map[string]
 		metadata[strings.ToLower(xhttp.ContentType)] = "application/octet-stream"
 	}
 
+	// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+	for k := range metadata {
+		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+			delete(metadata, k)
+		}
+	}
+
 	if contentEncoding, ok := metadata[strings.ToLower(xhttp.ContentEncoding)]; ok {
 		contentEncoding = trimAwsChunkedContentEncoding(contentEncoding)
 		if contentEncoding != "" {

cmd/http-tracer.go

@@ -129,6 +129,7 @@ func Trace(f http.HandlerFunc, logBody bool, w http.ResponseWriter, r *http.Requ
 
 	rq := trace.RequestInfo{
 		Time:     time.Now().UTC(),
+		Proto:    r.Proto,
 		Method:   r.Method,
 		Path:     r.URL.Path,
 		RawQuery: r.URL.RawQuery,

cmd/http/headers.go

@@ -102,6 +102,9 @@ const (
 	AmzSecurityToken        = "X-Amz-Security-Token"
 	AmzDecodedContentLength = "X-Amz-Decoded-Content-Length"
 
+	AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length"
+	AmzMetaUnencryptedContentMD5    = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5"
+
 	// Signature v2 related constants
 	AmzSignatureV2 = "Signature"
 	AmzAccessKeyID = "AWSAccessKeyId"
@@ -120,6 +123,9 @@ const (
 
 	// Header indicates if the mtime should be preserved by client
 	MinIOSourceMTime = "x-minio-source-mtime"
+
+	// Header indicates if the etag should be preserved by client
+	MinIOSourceETag = "x-minio-source-etag"
 )
 
 // Common http query params S3 API

cmd/http/server.go

@@ -191,14 +191,7 @@ func NewServer(addrs []string, handler http.Handler, getCert certs.GetCertificat
 			// TLS hardening
 			PreferServerCipherSuites: true,
 			MinVersion:               tls.VersionTLS12,
-			// Do not edit the next line, protos priority is kept
-			// on purpose in this manner for HTTP 2.0, we would
-			// still like HTTP 2.0 clients to negotiate connection
-			// to server if needed but by default HTTP 1.1 is
-			// expected. We need to change this in future
-			// when we wish to go back to HTTP 2.0 as default
-			// priority for HTTP protocol negotiation.
-			NextProtos: []string{"http/1.1", "h2"},
+			NextProtos:               []string{"h2", "http/1.1"},
 		}
 		tlsConfig.GetCertificate = getCert
 	}

cmd/lock-rest-client.go

@@ -149,7 +149,6 @@ func newlockRESTClient(endpoint Endpoint) *lockRESTClient {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 

cmd/object-api-options.go

@@ -164,7 +164,13 @@ func putOpts(ctx context.Context, r *http.Request, bucket, object string, metada
 	} else {
 		opts.MTime = UTCNow()
 	}
-
+	etag := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceETag))
+	if etag != "" {
+		if metadata == nil {
+			metadata = make(map[string]string)
+		}
+		metadata["etag"] = etag
+	}
 	// In the case of multipart custom format, the metadata needs to be checked in addition to header to see if it
 	// is SSE-S3 encrypted, primarily because S3 protocol does not require SSE-S3 headers in PutObjectPart calls
 	if GlobalGatewaySSE.SSES3() && (crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(metadata)) {

cmd/object-handlers.go

@@ -1155,6 +1155,8 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	if objTags != "" {
 		srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags
 	}
+	srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined)
+
 	srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true)
 	retPerms := isPutActionAllowed(getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectRetentionAction)
 	holdPerms := isPutActionAllowed(getRequestAuthType(r), dstBucket, dstObject, r, iampolicy.PutObjectLegalHoldAction)
@@ -1259,7 +1261,7 @@ func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Re
 	response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime)
 	encodedSuccessResponse := encodeResponse(response)
 	if mustReplicate(ctx, r, dstBucket, dstObject, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
-		defer replicateObject(ctx, dstBucket, dstObject, objInfo.VersionID, objectAPI, &eventArgs{
+		defer replicateObject(context.Background(), dstBucket, dstObject, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedCopy,
 			BucketName:   dstBucket,
 			Object:       objInfo,
@@ -1575,7 +1577,7 @@ func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Req
 		}
 	}
 	if mustReplicate(ctx, r, bucket, object, metadata, "") {
-		defer replicateObject(ctx, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
+		defer replicateObject(context.Background(), bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedPut,
 			BucketName:   bucket,
 			Object:       objInfo,
@@ -2650,7 +2652,7 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 
 	setPutObjHeaders(w, objInfo, false)
 	if mustReplicate(ctx, r, bucket, object, objInfo.UserDefined, objInfo.ReplicationStatus.String()) {
-		defer replicateObject(ctx, bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
+		defer replicateObject(context.Background(), bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
 			EventName:    event.ObjectCreatedCompleteMultipartUpload,
 			BucketName:   bucket,
 			Object:       objInfo,

cmd/prepare-storage.go

@@ -185,7 +185,6 @@ func IsServerResolvable(endpoint Endpoint) error {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 

cmd/server-main.go

@@ -119,7 +119,7 @@ func serverHandleCmdArgs(ctx *cli.Context) {
 	logger.FatalIf(err, "Unable to load the TLS configuration")
 
 	// Check and load Root CAs.
-	globalRootCAs, err = config.GetRootCAs(globalCertsCADir.Get())
+	globalRootCAs, err = certs.GetRootCAs(globalCertsCADir.Get())
 	logger.FatalIf(err, "Failed to read root CAs (%v)", err)
 
 	// Register root CAs for remote ENVs

cmd/signals.go

@@ -18,6 +18,8 @@ package cmd
 
 import (
 	"context"
+	"errors"
+	"net/http"
 	"os"
 	"strings"
 
@@ -73,7 +75,7 @@ func handleSignals() {
 			if objAPI := newObjectLayerWithoutSafeModeFn(); objAPI != nil {
 				objAPI.Shutdown(context.Background())
 			}
-			if err != nil {
+			if err != nil && !errors.Is(err, http.ErrServerClosed) {
 				logger.Fatal(err, "Unable to start MinIO server")
 			}
 			exit(true)

cmd/storage-rest-client.go

@@ -663,7 +663,6 @@ func newStorageRESTClient(endpoint Endpoint) *storageRESTClient {
 		tlsConfig = &tls.Config{
 			ServerName: endpoint.Hostname(),
 			RootCAs:    globalRootCAs,
-			NextProtos: []string{"http/1.1"}, // Force http1.1
 		}
 	}
 

cmd/utils.go

@@ -39,13 +39,13 @@ import (
 	"sync"
 	"time"
 
+	humanize "github.com/dustin/go-humanize"
+	"github.com/gorilla/mux"
 	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/madmin"
-
-	humanize "github.com/dustin/go-humanize"
-	"github.com/gorilla/mux"
+	"golang.org/x/net/http2"
 )
 
 const (
@@ -467,6 +467,11 @@ func newInternodeHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration)
 		// in raw stream.
 		DisableCompression: true,
 	}
+
+	if tlsConfig != nil {
+		http2.ConfigureTransport(tr)
+	}
+
 	return func() *http.Transport {
 		return tr
 	}
@@ -490,6 +495,11 @@ func newCustomHTTPTransport(tlsConfig *tls.Config, dialTimeout time.Duration) fu
 		// in raw stream.
 		DisableCompression: true,
 	}
+
+	if tlsConfig != nil {
+		http2.ConfigureTransport(tr)
+	}
+
 	return func() *http.Transport {
 		return tr
 	}

cmd/web-handlers.go

@@ -48,6 +48,7 @@ import (
 	"github.com/minio/minio/pkg/auth"
 	objectlock "github.com/minio/minio/pkg/bucket/object/lock"
 	"github.com/minio/minio/pkg/bucket/policy"
+	"github.com/minio/minio/pkg/bucket/replication"
 	"github.com/minio/minio/pkg/event"
 	"github.com/minio/minio/pkg/handlers"
 	"github.com/minio/minio/pkg/hash"
@@ -961,6 +962,7 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 
 	retPerms := ErrAccessDenied
 	holdPerms := ErrAccessDenied
+	replPerms := ErrAccessDenied
 	if authErr != nil {
 		if authErr == errNoAuthToken {
 			// Check if anonymous (non-owner) has access to upload objects.
@@ -1016,6 +1018,17 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		}) {
 			holdPerms = ErrNone
 		}
+		if globalIAMSys.IsAllowed(iampolicy.Args{
+			AccountName:     claims.AccessKey,
+			Action:          iampolicy.GetReplicationConfigurationAction,
+			BucketName:      bucket,
+			ConditionValues: getConditionValues(r, "", claims.AccessKey, claims.Map()),
+			IsOwner:         owner,
+			ObjectName:      object,
+			Claims:          claims.Map(),
+		}) {
+			replPerms = ErrNone
+		}
 	}
 
 	// Check if bucket is a reserved bucket name or invalid.
@@ -1082,6 +1095,10 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	mustReplicate := mustReplicateWeb(ctx, r, bucket, object, metadata, "", replPerms)
+	if mustReplicate {
+		metadata[xhttp.AmzBucketReplicationStatus] = string(replication.Pending)
+	}
 	pReader = NewPutObjReader(hashReader, nil, nil)
 	// get gateway encryption options
 	opts, err := putOpts(ctx, r, bucket, object, metadata)
@@ -1155,7 +1172,17 @@ func (web *webAPIHandlers) Upload(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 	}
-
+	if mustReplicate {
+		defer replicateObject(context.Background(), bucket, object, objInfo.VersionID, objectAPI, &eventArgs{
+			EventName:    event.ObjectCreatedPut,
+			BucketName:   bucket,
+			Object:       objInfo,
+			ReqParams:    extractReqParams(r),
+			RespElements: extractRespElements(w),
+			UserAgent:    r.UserAgent(),
+			Host:         handlers.GetSourceIP(r),
+		}, false)
+	}
 	// Notify object created event.
 	sendEvent(eventArgs{
 		EventName:    event.ObjectCreatedPut,

cmd/xl-storage-format-v2.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 )
 
@@ -397,6 +398,11 @@ func (j xlMetaV2Object) ToFileInfo(volume, path string) (FileInfo, error) {
 	}
 	fi.Metadata = make(map[string]string, len(j.MetaUser)+len(j.MetaSys))
 	for k, v := range j.MetaUser {
+		// https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w
+		if strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentLength) || strings.EqualFold(k, xhttp.AmzMetaUnencryptedContentMD5) {
+			continue
+		}
+
 		fi.Metadata[k] = v
 	}
 	for k, v := range j.MetaSys {

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '3.7'
 # 9001 through 9004.
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     volumes:
       - data1-1:/data1
       - data1-2:/data2
@@ -22,7 +22,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     volumes:
       - data2-1:/data1
       - data2-2:/data2
@@ -39,7 +39,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     volumes:
       - data3-1:/data1
       - data3-2:/data2
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     volumes:
       - data4-1:/data1
       - data4-2:/data2

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -29,7 +29,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -56,7 +56,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -83,7 +83,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3.7'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -33,7 +33,7 @@ services:
       retries: 3
 
   minio2:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -64,7 +64,7 @@ services:
       retries: 3
 
   minio3:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -95,7 +95,7 @@ services:
       retries: 3
 
   minio4:
-    image: minio/minio:RELEASE.2020-08-07T01-23-07Z
+    image: minio/minio:RELEASE.2020-08-08T04-50-06Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/zh_CN/bucket/lifecycle/README.md

@@ -0,0 +1,57 @@
+# 存储桶生命周期配置快速入门指南 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
+
+在存储桶上启用对象的生命周期配置，可以设置在指定天数或指定日期后自动删除对象。
+
+## 1. 前提条件
+- 安装MinIO - [MinIO快速入门指南](https://docs.min.io/cn/minio-quickstart-guide).
+- 安装`mc` - [mc快速入门指南](https://docs.minio.io/cn/minio-client-quickstart-guide.html)
+
+## 2. 启用存储桶生命周期配置
+
+- 创建一个存储桶的生命周期配置，该配置让前缀`old/`下的对象在`2020-01-01T00:00:00.000Z`过期，同时前缀`temp/`下的对象在7天后过期。
+- 使用`mc`启用存储桶的生命周期配置:
+
+```sh
+$ mc ilm import play/testbucket
+{
+    "Rules": [
+        {
+            "Expiration": {
+                "Date": "2020-01-01T00:00:00.000Z"
+            },
+            "ID": "OldPictures",
+            "Filter": {
+                "Prefix": "old/"
+            },
+            "Status": "Enabled"
+        },
+        {
+            "Expiration": {
+                "Days": 7
+            },
+            "ID": "TempUploads",
+            "Filter": {
+                "Prefix": "temp/"
+            },
+            "Status": "Enabled"
+        }
+    ]
+}
+
+Lifecycle configuration imported successfully to `play/testbucket`.
+```
+
+- 列出当前的设置
+```
+$ mc ilm list play/testbucket
+     ID     |  Prefix  |  Enabled   | Expiry |  Date/Days   |  Transition  |    Date/Days     |  Storage-Class   |       Tags
+------------|----------|------------|--------|--------------|--------------|------------------|------------------|------------------
+OldPictures |   old/   |    ✓       |  ✓     |  1 Jan 2020  |     ✗        |                  |                  |
+------------|----------|------------|--------|--------------|--------------|------------------|------------------|------------------
+TempUploads |  temp/   |    ✓       |  ✓     |   7 day(s)   |     ✗        |                  |                  |
+------------|----------|------------|--------|--------------|--------------|------------------|------------------|------------------
+```
+
+## 进一步探索
+- [MinIO | Golang Client API文档](https://docs.min.io/cn/golang-client-api-reference.html#SetBucketLifecycle)
+- [对象生命周期管理](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html)

docs/zh_CN/bucket/quota/README.md

@@ -0,0 +1,40 @@
+# 存储桶配额配置快速入门指南 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
+
+![quota](https://raw.githubusercontent.com/minio/minio/master/docs/zh_CN/bucket/quota/bucketquota.png)
+
+存储桶有两种配额类型可供选择，分别是FIFO和Hard。
+
+- `Hard` 表示达到配置的配额限制后，禁止向存储桶写入数据。
+- `FIFO` 会自动删除最旧的内容，直到存储桶的空间使用在限制范围内，同时允许写入。
+
+> 注意：网关或独立单磁盘模式下不支持存储桶配额。
+
+## 前置条件
+- 安装MinIO - [MinIO快速入门指南](https://docs.min.io/cn/minio-quickstart-guide).
+- [`mc`和MinIO Server一起使用](https://docs.min.io/cn/minio-client-quickstart-guide)
+
+## 设置存储桶配额
+
+### 在MinIO对象存储上，设置存储桶`mybucket`的额度为1GB，配额类型为hard:
+
+```sh
+$ mc admin bucket quota myminio/mybucket --hard 1gb
+```
+
+### 将MinIO上的存储桶"mybucket"的额度设置为5GB，配额类型为FIFO，这样就会自动删除较旧的内容，以确保存储桶的空间使用保持在5GB以内
+
+```sh
+$ mc admin bucket quota myminio/mybucket --fifo 5gb
+```
+
+### 验证MinIO上的存储桶`mybucket`的配额设置
+
+```sh
+$ mc admin bucket quota myminio/mybucket
+```
+
+### 清除MinIO上的存储桶`mybucket`的配额设置
+
+```sh
+$ mc admin bucket quota myminio/mybucket --clear
+```

docs/zh_CN/bucket/replication/README.md

@@ -0,0 +1,84 @@
+# 存储桶复制指南 [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/)
+
+Bucket replication is designed to replicate selected objects in a bucket to a destination bucket.
+存储桶复制功能可以把存储桶中选中的对象复制到目标存储桶。
+
+要想复制一个存储桶上的对象,到同一集群或者不同集群的目标站点上的目标存储桶中，首先要为源存储桶和目标存储桶启用[版本控制功能](https://docs.minio.io/docs/minio-bucket-versioning-guide.html)。最后，需要在源MinIO服务器上配置目标站点和目标存储桶。
+
+##  强调
+- 和AWS S3不同，MinIO的源存储桶和目标存储桶名字可以一样，可处理各种情况，例如*Splunk*，*Veeam*站点到站点*DR*。
+- 与AWS S3不同，MinIO纯天然支持跨源桶和目标桶的对象锁定保留。
+- 比[AWS S3 Bucket Replication Config](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html)实现更简单，像IAM Role, AccessControlTranslation, Metrics and SourceSelectionCriteria这些功能在MinIO中不需要.
+
+## 如何使用?
+如下所示在源群集上创建复制目标：
+
+```
+mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1
+Role ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket'
+```
+
+请注意，admin需要具有源集群上的*s3:GetReplicationConfigurationAction*权限。在目标上使用的凭据需要具有*s3:ReplicateObject*权限. 成功创建并授权后，将生成复制目标ARN。下面的命令列出了所有当前授权的复制目标：:
+
+```
+mc admin bucket remote ls myminio/srcbucket --service "replication"
+Role ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket'
+```
+
+现在可以使用带有复制配置JSON文件，把复制配置添加到源存储桶。上面的角色ARN作为配置中的json元素传入。
+
+```json
+{
+  "Role" :"arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket",
+  "Rules": [
+    {
+      "Status": "Enabled",
+      "Priority": 1,
+      "DeleteMarkerReplication": { "Status": "Disabled" },
+      "Filter" : {
+        "And": {
+            "Prefix": "Tax",
+            "Tags": [
+                {
+                "Key": "Year",
+                "Value": "2019"
+                },
+                {
+                "Key": "Company",
+                "Value": "AcmeCorp"
+                }
+            ]
+        }
+      },
+      "Destination": {
+        "Bucket": "arn:aws:s3:::destbucket",
+        "StorageClass": "STANDARD"
+      }
+    }
+  ]
+}
+```
+
+```
+mc replicate add myminio/srcbucket/Tax --priority 1 --arn "arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD"
+Replication configuration applied successfully to myminio/srcbucket.
+```
+
+复制配置遵循[AWS S3规范](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). 现在，上传到源存储桶中符合复制条件的任何对象都将被MinIO服务器自动复制到远程目标存储桶中。通过禁用配置中的特定规则或者删除复制配置，都可以随时禁用复制。
+
+
+按照S3规范，当从源存储桶中删除一个对象后，副本不会被删除。
+
+![delete](https://raw.githubusercontent.com/minio/minio/master/docs/zh_CN/bucket/replication/DELETE_bucket_replication.png)
+
+当对象锁定与复制结合使用时，源桶和目标桶都需要启用对象锁定。同理，如果目标也支持加密，则服务器端将复制加密的对象。
+
+复制状态可以在源和目标对象的元数据中看到。在源端，根据复制的结果是成功还是失败，`X-Amz-Replication-Status`会从`PENDING`变更为`COMPLETE`或者 `FAILED`状态。 在目标端，对象成功复制，`X-Amz-Replication-Status`会被设置为`REPLICA`状态。在定期的磁盘扫描周期中，任何复制失败都将自动重新尝试。
+
+![put](https://raw.githubusercontent.com/minio/minio/master/docs/zh_CN/bucket/replication/PUT_bucket_replication.png)
+
+![head](https://raw.githubusercontent.com/minio/minio/master/docs/zh_CN/bucket/replication/HEAD_bucket_replication.png)
+
+## 进一步探索
+- [MinIO存储桶版本控制实现](https://docs.minio.io/docs/minio-bucket-versioning-guide.html)
+- [MinIO客户端快速入门指南](https://docs.minio.io/cn/minio-client-quickstart-guide.html)

go.mod

@@ -7,7 +7,7 @@ require (
 	git.apache.org/thrift.git v0.13.0
 	github.com/Azure/azure-pipeline-go v0.2.1
 	github.com/Azure/azure-storage-blob-go v0.8.0
-	github.com/Azure/go-autorest/autorest/adal v0.9.0 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.9.1 // indirect
 	github.com/Shopify/sarama v1.24.1
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/alecthomas/participle v0.2.1
@@ -47,13 +47,13 @@ require (
 	github.com/miekg/dns v1.1.8
 	github.com/minio/cli v1.22.0
 	github.com/minio/highwayhash v1.0.0
-	github.com/minio/minio-go/v7 v7.0.3
+	github.com/minio/minio-go/v7 v7.0.5-0.20200811211821-14ed05478889
 	github.com/minio/selfupdate v0.3.1
 	github.com/minio/sha256-simd v0.1.1
-	github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37
+	github.com/minio/simdjson-go v0.1.5
 	github.com/minio/sio v0.2.0
 	github.com/mitchellh/go-homedir v1.1.0
-	github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061 // indirect
+	github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 // indirect
 	github.com/montanaflynn/stats v0.5.0
 	github.com/nats-io/nats-server/v2 v2.1.7
 	github.com/nats-io/nats-streaming-server v0.18.0 // indirect
@@ -76,14 +76,13 @@ require (
 	github.com/tidwall/sjson v1.0.4
 	github.com/tinylib/msgp v1.1.2
 	github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a
-	github.com/willf/bitset v1.1.10 // indirect
+	github.com/willf/bitset v1.1.11 // indirect
 	github.com/willf/bloom v2.0.3+incompatible
 	github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c
 	go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200707003333-58bb8ae09f8e
 	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
 	golang.org/x/net v0.0.0-20200707034311-ab3426394381
 	golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae
-	golang.org/x/tools v0.0.0-20200724172932-b5fc9d354d99 // indirect
 	google.golang.org/api v0.5.0
 	gopkg.in/jcmturner/gokrb5.v7 v7.3.0
 	gopkg.in/ldap.v3 v3.0.3

go.sum

@@ -10,8 +10,8 @@ github.com/Azure/azure-storage-blob-go v0.8.0 h1:53qhf0Oxa0nOjgbDeeYPUeyiNmafAFE
 github.com/Azure/azure-storage-blob-go v0.8.0/go.mod h1:lPI3aLPpuLTeUwh1sViKXFxwl2B6teiRqI0deQUvsw0=
 github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
 github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
-github.com/Azure/go-autorest/autorest/adal v0.9.0 h1:SigMbuFNuKgc1xcGhaeapbh+8fgsu+GxgDRFyg7f5lM=
-github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
+github.com/Azure/go-autorest/autorest/adal v0.9.1 h1:xjPqigMQe2+0DAJ5A6MLUPp5D2r2Io8qHCuCMMI/yJU=
+github.com/Azure/go-autorest/autorest/adal v0.9.1/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg=
 github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw=
 github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74=
 github.com/Azure/go-autorest/autorest/mocks v0.4.0 h1:z20OWOSG5aCye0HEkDp6TPmP17ZcfeMxPi6HnSALa8c=
@@ -291,14 +291,14 @@ github.com/minio/highwayhash v1.0.0 h1:iMSDhgUILCr0TNm8LWlSjF8N0ZIj2qbO8WHp6Q/J2
 github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc=
 github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
 github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
-github.com/minio/minio-go/v7 v7.0.3 h1:a2VHaXDlKBcB3J5XJhKVfWBRi1+ZmMWFXABQ8TLlWbA=
-github.com/minio/minio-go/v7 v7.0.3/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw=
+github.com/minio/minio-go/v7 v7.0.5-0.20200811211821-14ed05478889 h1:uO6mz/7ywYA4U/Xl/FzJ/FePE4HIJT76/UFdXFoOqUY=
+github.com/minio/minio-go/v7 v7.0.5-0.20200811211821-14ed05478889/go.mod h1:CSt2ETZNs+bIIhWTse0mcZKZWMGrFU7Er7RR0TmkDYk=
 github.com/minio/selfupdate v0.3.1 h1:BWEFSNnrZVMUWXbXIgLDNDjbejkmpAmZvy/nCz1HlEs=
 github.com/minio/selfupdate v0.3.1/go.mod h1:b8ThJzzH7u2MkF6PcIra7KaXO9Khf6alWPvMSyTDCFM=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
-github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37 h1:pDeao6M5AEd8hwTtGmE0pVKomlL56JFRa5SiXDZAuJE=
-github.com/minio/simdjson-go v0.1.5-0.20200303142138-b17fe061ea37/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
+github.com/minio/simdjson-go v0.1.5 h1:6T5mHh7r3kUvgwhmFWQAjoPV5Yt5oD/VPjAI9ViH1kM=
+github.com/minio/simdjson-go v0.1.5/go.mod h1:oKURrZZEBtqObgJrSjN1Ln2n9MJj2icuBTkeJzZnvSI=
 github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA=
 github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -311,8 +311,8 @@ github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUb
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
-github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061 h1:UCU8+cLbbvyxi0sQ9fSeoEhZgvrrD9HKMtX6Gmc1vk8=
-github.com/mmcloughlin/avo v0.0.0-20200523190732-4439b6b2c061/go.mod h1:wqKykBG2QzQDJEzvRkcS8x6MiSJkF52hXZsXcjaB3ls=
+github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104 h1:ULR/QWMgcgRiZLUjSSJMU+fW+RDMstRdmnDWj9Q+AsA=
+github.com/mmcloughlin/avo v0.0.0-20200803215136-443f81d77104/go.mod h1:wqKykBG2QzQDJEzvRkcS8x6MiSJkF52hXZsXcjaB3ls=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421 h1:ZqeYNhU3OHLH3mGKHDcjJRFFRrJa6eAM5H+CtDdOsPc=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
@@ -436,8 +436,8 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a h1:0R4NLDRDZX6JcmhJgXi5E4b8Wg84ihbmUKp/GvSPEzc=
 github.com/valyala/tcplisten v0.0.0-20161114210144-ceec8f93295a/go.mod h1:v3UYOV9WzVtRmSR+PDvWpU/qWl4Wa5LApYYX4ZtKbio=
-github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc=
-github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE=
+github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI=
 github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA=
 github.com/willf/bloom v2.0.3+incompatible/go.mod h1:MmAltL9pDMNTrvUkxdg0k0q5I0suxmuwp3KbyrZLOZ8=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
@@ -447,7 +447,6 @@ github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
-github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 go.etcd.io/bbolt v1.3.5 h1:XAzx9gjCb0Rxj7EoqcClPD1d5ZBxZJk0jbuoPHenBt0=
 go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
 go.etcd.io/etcd/v3 v3.3.0-rc.0.0.20200707003333-58bb8ae09f8e h1:HZQLoe71Q24wVyDrGBRcVuogx32U+cPlcm/WoSLUI6c=
@@ -488,8 +487,6 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl
 golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
 golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
-golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
-golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -503,7 +500,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -516,8 +512,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA=
-golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -543,7 +537,6 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db h1:6/JqlYfC1CCaLnGceQTI+sDGhC9UBSPAsBqI0Gun6kU=
 golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
-golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
@@ -567,8 +560,6 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20200425043458-8463f397d07c h1:iHhCR0b26amDCiiO+kBguKZom9aMF+NrFxh9zeKR/XU=
 golang.org/x/tools v0.0.0-20200425043458-8463f397d07c/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
-golang.org/x/tools v0.0.0-20200724172932-b5fc9d354d99 h1:OHn441rq5CeM5r1xJ0OmY7lfdTvnedi6k+vQiI7G9b8=
-golang.org/x/tools v0.0.0-20200724172932-b5fc9d354d99/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=

mint/preinstall.sh

@@ -19,7 +19,7 @@ export APT="apt --quiet --yes"
 export WGET="wget --quiet --no-check-certificate"
 
 # install nodejs source list
-if ! $WGET --output-document=- https://deb.nodesource.com/setup_13.x | bash -; then
+if ! $WGET --output-document=- https://deb.nodesource.com/setup_14.x | bash -; then
     echo "unable to set nodejs repository"
     exit 1
 fi

mint/run/core/s3select/csv.py

@@ -15,37 +15,41 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import os
 import io
+import os
 
 from minio import Minio
-from minio.select.options import (SelectObjectOptions, CSVInput,
-                                  RequestProgress, InputSerialization,
-                                  OutputSerialization, CSVOutput, JsonOutput)
-
+from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
+                                  JSONOutput, OutputSerialization,
+                                  RequestProgress, SelectObjectOptions)
 from utils import *
 
+
 def test_sql_api(test_name, client, bucket_name, input_data, sql_opts, expected_output):
     """ Test if the passed SQL request has the output equal to the passed execpted one"""
     object_name = generate_object_name()
     got_output = b''
     try:
         bytes_content = io.BytesIO(input_data)
-        client.put_object(bucket_name, object_name, io.BytesIO(input_data), len(input_data))
+        client.put_object(bucket_name, object_name,
+                          io.BytesIO(input_data), len(input_data))
         data = client.select_object_content(bucket_name, object_name, sql_opts)
         # Get the records
         records = io.BytesIO()
         for d in data.stream(10*1024):
             records.write(d.encode('utf-8'))
-        got_output = records.getvalue()
+            got_output = records.getvalue()
     except Exception as select_err:
         if not isinstance(expected_output, Exception):
-            raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
+            raise ValueError(
+                'Test {} unexpectedly failed with: {}'.format(test_name, select_err))
     else:
         if isinstance(expected_output, Exception):
-            raise ValueError('Test {}: expected an exception, got {}'.format(test_name, got_output))
+            raise ValueError(
+                'Test {}: expected an exception, got {}'.format(test_name, got_output))
         if got_output != expected_output:
-            raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(test_name, expected_output, got_output))
+            raise ValueError('Test {}: data mismatch. Expected : {}, Received {}'.format(
+                test_name, expected_output, got_output))
     finally:
         client.remove_object(bucket_name, object_name)
 
@@ -55,28 +59,34 @@ def test_csv_input_custom_quote_char(client, log_output):
     log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
 
     tests = [
-            # Invalid quote character, should fail
-            ('""', '"', b'col1,col2,col3\n', Exception()),
-            # UTF-8 quote character
-            ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(), b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
-            # Only one field is quoted
-            ('"', '"', b'"col1",col2,col3\n', b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
-            ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'),
-            ('\'', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
-            ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
-            ('', '"', b'"col1",col2,col3\n', b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
-            ('', '"', b'"col1","col2","col3"\n', b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
-            ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'),
-            ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'),
-            ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'),
-            ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'),
-            ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
-            ('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'),
-            ('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'),
-            ('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
-            ('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'),
-            ('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'),
-            ('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'),
+        # Invalid quote character, should fail
+        ('""', '"', b'col1,col2,col3\n', Exception()),
+        # UTF-8 quote character
+        ('ع', '"', 'عcol1ع,عcol2ع,عcol3ع\n'.encode(),
+         b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
+        # Only one field is quoted
+        ('"', '"', b'"col1",col2,col3\n',
+         b'{"_1":"col1","_2":"col2","_3":"col3"}\n'),
+        ('"', '"', b'"col1,col2,col3"\n', b'{"_1":"col1,col2,col3"}\n'),
+        ('\'', '"', b'"col1",col2,col3\n',
+         b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
+        ('', '"', b'"col1",col2,col3\n',
+         b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
+        ('', '"', b'"col1",col2,col3\n',
+         b'{"_1":"\\"col1\\"","_2":"col2","_3":"col3"}\n'),
+        ('', '"', b'"col1","col2","col3"\n',
+         b'{"_1":"\\"col1\\"","_2":"\\"col2\\"","_3":"\\"col3\\""}\n'),
+        ('"', '"', b'""""""\n', b'{"_1":"\\"\\""}\n'),
+        ('"', '"', b'A",B\n', b'{"_1":"A\\"","_2":"B"}\n'),
+        ('"', '"', b'A"",B\n', b'{"_1":"A\\"\\"","_2":"B"}\n'),
+        ('"', '\\', b'A\\B,C\n', b'{"_1":"A\\\\B","_2":"C"}\n'),
+        ('"', '"', b'"A""B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
+        ('"', '\\', b'"A\\B","CD"\n', b'{"_1":"AB","_2":"CD"}\n'),
+        ('"', '\\', b'"A\\,","CD"\n', b'{"_1":"A,","_2":"CD"}\n'),
+        ('"', '\\', b'"A\\"B","CD"\n', b'{"_1":"A\\"B","_2":"CD"}\n'),
+        ('"', '\\', b'"A\\""\n', b'{"_1":"A\\""}\n'),
+        ('"', '\\', b'"A\\"\\"B"\n', b'{"_1":"A\\"\\"B"}\n'),
+        ('"', '\\', b'"A\\"","\\"B"\n', b'{"_1":"A\\"","_2":"\\"B"}\n'),
     ]
 
     client.make_bucket(bucket_name)
@@ -84,54 +94,56 @@ def test_csv_input_custom_quote_char(client, log_output):
     try:
         for idx, (quote_char, escape_char, data, expected_output) in enumerate(tests):
             sql_opts = SelectObjectOptions(
-                    expression="select * from s3object",
-                    input_serialization=InputSerialization(
-                        compression_type="NONE",
-                        csv=CSVInput(FileHeaderInfo="NONE",
-                            RecordDelimiter="\n",
-                            FieldDelimiter=",",
-                            QuoteCharacter=quote_char,
-                            QuoteEscapeCharacter=escape_char,
-                            Comments="#",
-                            AllowQuotedRecordDelimiter="FALSE",),
-                        ),
-                    output_serialization=OutputSerialization(
-                        json = JsonOutput(
-                           RecordDelimiter="\n",
-                           )
-                        ),
-                    request_progress=RequestProgress(
-                        enabled="False"
-                        )
+                expression="select * from s3object",
+                input_serialization=InputSerialization(
+                    compression_type="NONE",
+                    csv=CSVInput(file_header_info="NONE",
+                                 record_delimiter="\n",
+                                 field_delimiter=",",
+                                 quote_character=quote_char,
+                                 quote_escape_character=escape_char,
+                                 comments="#",
+                                 allow_quoted_record_delimiter="FALSE",),
+                ),
+                output_serialization=OutputSerialization(
+                    json=JSONOutput(
+                        record_delimiter="\n",
                     )
+                ),
+                request_progress=RequestProgress(
+                    enabled="False"
+                )
+            )
 
-            test_sql_api(f'test_{idx}', client, bucket_name, data, sql_opts, expected_output)
+            test_sql_api(f'test_{idx}', client, bucket_name,
+                         data, sql_opts, expected_output)
     finally:
         client.remove_bucket(bucket_name)
 
     # Test passes
     print(log_output.json_report())
 
+
 def test_csv_output_custom_quote_char(client, log_output):
     # Get a unique bucket_name and object_name
     log_output.args['bucket_name'] = bucket_name = generate_bucket_name()
 
     tests = [
-            # UTF-8 quote character
-            ("''", "''", b'col1,col2,col3\n', Exception()),
-            ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
-            ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'),
-            ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'),
-            ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'),
-            ('"', '"', b'""""\n', b'""""\n'),
-            ('"', '"', b'\n', b''),
-            ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
-            ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"),
-            ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"),
-            ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"),
-            ("'", "\\", b'col\'\n', b"'col\\''\n"),
-            # Two consecutive escaped quotes
-            ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"),
+        # UTF-8 quote character
+        ("''", "''", b'col1,col2,col3\n', Exception()),
+        ("'", "'", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
+        ("", '"', b'col1,col2,col3\n', b'\x00col1\x00,\x00col2\x00,\x00col3\x00\n'),
+        ('"', '"', b'col1,col2,col3\n', b'"col1","col2","col3"\n'),
+        ('"', '"', b'col"1,col2,col3\n', b'"col""1","col2","col3"\n'),
+        ('"', '"', b'""""\n', b'""""\n'),
+        ('"', '"', b'\n', b''),
+        ("'", "\\", b'col1,col2,col3\n', b"'col1','col2','col3'\n"),
+        ("'", "\\", b'col""1,col2,col3\n', b"'col\"\"1','col2','col3'\n"),
+        ("'", "\\", b'col\'1,col2,col3\n', b"'col\\'1','col2','col3'\n"),
+        ("'", "\\", b'"col\'1","col2","col3"\n', b"'col\\'1','col2','col3'\n"),
+        ("'", "\\", b'col\'\n', b"'col\\''\n"),
+        # Two consecutive escaped quotes
+        ("'", "\\", b'"a"""""\n', b"'a\"\"'\n"),
     ]
 
     client.make_bucket(bucket_name)
@@ -139,34 +151,35 @@ def test_csv_output_custom_quote_char(client, log_output):
     try:
         for idx, (quote_char, escape_char, input_data, expected_output) in enumerate(tests):
             sql_opts = SelectObjectOptions(
-                    expression="select * from s3object",
-                    input_serialization=InputSerialization(
-                        compression_type="NONE",
-                        csv=CSVInput(FileHeaderInfo="NONE",
-                            RecordDelimiter="\n",
-                            FieldDelimiter=",",
-                            QuoteCharacter='"',
-                            QuoteEscapeCharacter='"',
-                            Comments="#",
-                            AllowQuotedRecordDelimiter="FALSE",),
-                        ),
-                    output_serialization=OutputSerialization(
-                        csv=CSVOutput(QuoteFields="ALWAYS",
-                            RecordDelimiter="\n",
-                            FieldDelimiter=",",
-                            QuoteCharacter=quote_char,
-                            QuoteEscapeCharacter=escape_char,)
-                        ),
-                    request_progress=RequestProgress(
-                        enabled="False"
-                        )
-                    )
+                expression="select * from s3object",
+                input_serialization=InputSerialization(
+                    compression_type="NONE",
+                    csv=CSVInput(file_header_info="NONE",
+                                 record_delimiter="\n",
+                                 field_delimiter=",",
+                                 quote_character='"',
+                                 quote_escape_character='"',
+                                 comments="#",
+                                 allow_quoted_record_delimiter="FALSE",
+                                 ),
+                ),
+                output_serialization=OutputSerialization(
+                    csv=CSVOutput(quote_fields="ALWAYS",
+                                  record_delimiter="\n",
+                                  field_delimiter=",",
+                                  quote_character=quote_char,
+                                  quote_escape_character=escape_char,
+                                  )
+                ),
+                request_progress=RequestProgress(
+                    enabled="False"
+                )
+            )
 
-            test_sql_api(f'test_{idx}', client, bucket_name, input_data, sql_opts, expected_output)
+            test_sql_api(f'test_{idx}', client, bucket_name,
+                         input_data, sql_opts, expected_output)
     finally:
         client.remove_bucket(bucket_name)
 
     # Test passes
     print(log_output.json_report())
-
-

mint/run/core/s3select/sql_ops.py

@@ -18,14 +18,14 @@
 import io
 from datetime import datetime
 
-from minio import Minio
-from minio.select.options import (SelectObjectOptions, CSVInput, JSONInput,
-                                  RequestProgress, InputSerialization,
-                                  OutputSerialization, CSVOutput, JsonOutput)
+from minio.select.options import (CSVInput, CSVOutput, InputSerialization,
+                                  JSONInput, JSONOutput, OutputSerialization,
+                                  RequestProgress, SelectObjectOptions)
+from utils import generate_bucket_name, generate_object_name
 
-from utils import *
 
-def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql_output, tests, log_output):
+def test_sql_expressions_custom_input_output(client, input_bytes, sql_input,
+                                             sql_output, tests, log_output):
     bucket_name = generate_bucket_name()
     object_name = generate_object_name()
 
@@ -48,10 +48,11 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
                     output_serialization=sql_output,
                     request_progress=RequestProgress(
                         enabled="False"
-                        )
                     )
+                )
 
-                data = client.select_object_content(bucket_name, object_name, options)
+                data = client.select_object_content(
+                    bucket_name, object_name, options)
 
                 # Get the records
                 records = io.BytesIO()
@@ -62,13 +63,15 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
                 if got_output != expected_output:
                     if type(expected_output) == datetime:
                         # Attempt to parse the date which will throw an exception for any issue
-                        datetime.strptime(got_output.decode("utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
+                        datetime.strptime(got_output.decode(
+                            "utf-8").strip(), '%Y-%m-%dT%H:%M:%S.%f%z')
                     else:
-                        raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(idx+1, expected_output, got_output))
+                        raise ValueError('Test {}: data mismatch. Expected : {}. Received: {}.'.format(
+                            idx+1, expected_output, got_output))
 
                 log_output.args['total_success'] += 1
             except Exception as err:
-                continue ## TODO, raise instead
+                continue  # TODO, raise instead
                 # raise Exception(err)
     finally:
         client.remove_object(bucket_name, object_name)
@@ -77,16 +80,16 @@ def test_sql_expressions_custom_input_output(client, input_bytes, sql_input, sql
 
 def test_sql_expressions(client, input_json_bytes, tests, log_output):
     input_serialization = InputSerialization(
-            compression_type="NONE",
-            json=JSONInput(Type="DOCUMENT"),
-            )
+        compression_type="NONE",
+        json=JSONInput(json_type="DOCUMENT"),
+    )
 
-    output_serialization=OutputSerialization(
-            csv=CSVOutput(QuoteFields="ASNEEDED")
-            )
+    output_serialization = OutputSerialization(
+        csv=CSVOutput(quote_fields="ASNEEDED")
+    )
 
     test_sql_expressions_custom_input_output(client, input_json_bytes,
-            input_serialization, output_serialization, tests, log_output)
+                                             input_serialization, output_serialization, tests, log_output)
 
 
 def test_sql_operators(client, log_output):
@@ -98,31 +101,38 @@ def test_sql_operators(client, log_output):
 """
 
     tests = [
-       # Logical operators
-       ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'),
-       ("NOT", "select * from S3Object s where NOT s.id = 1", b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
-       ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3", b'1,John,3\n3,Yves,5\n'),
-       # Comparison Operators
-       ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'),
-       (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'),
-       ("<=", "select * from S3Object s where s.age <= 4", b'1,John,3\n2,Elliot,4\n4,,0\n'),
-       (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'),
-       ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'),
-       ("<>", "select * from S3Object s where s.age <> 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
-       ("!=", "select * from S3Object s where s.age != 4", b'1,John,3\n3,Yves,5\n4,,0\n'),
-       ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5", b'2,Elliot,4\n3,Yves,5\n'),
-       ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'),
-       # Pattern Matching Operators
-       ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'),
-       ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'),
-       # Unitary Operators
-       ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'),
-       ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL", b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
-       # Math Operators
-       ("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'),
-       ("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'),
-       ("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'),
-       ("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'),
+        # Logical operators
+        ("AND", "select * from S3Object s where s.id = 1 AND s.name = 'John'", b'1,John,3\n'),
+        ("NOT", "select * from S3Object s where NOT s.id = 1",
+            b'2,Elliot,4\n3,Yves,5\n4,,0\n'),
+        ("OR", "select * from S3Object s where s.id = 1 OR s.id = 3",
+            b'1,John,3\n3,Yves,5\n'),
+        # Comparison Operators
+        ("<", "select * from S3Object s where s.age < 4", b'1,John,3\n4,,0\n'),
+        (">", "select * from S3Object s where s.age > 4", b'3,Yves,5\n'),
+        ("<=", "select * from S3Object s where s.age <= 4",
+            b'1,John,3\n2,Elliot,4\n4,,0\n'),
+        (">=", "select * from S3Object s where s.age >= 4", b'2,Elliot,4\n3,Yves,5\n'),
+        ("=", "select * from S3Object s where s.age = 4", b'2,Elliot,4\n'),
+        ("<>", "select * from S3Object s where s.age <> 4",
+            b'1,John,3\n3,Yves,5\n4,,0\n'),
+        ("!=", "select * from S3Object s where s.age != 4",
+            b'1,John,3\n3,Yves,5\n4,,0\n'),
+        ("BETWEEN", "select * from S3Object s where s.age BETWEEN 4 AND 5",
+            b'2,Elliot,4\n3,Yves,5\n'),
+        ("IN", "select * from S3Object s where s.age IN (3,5)", b'1,John,3\n3,Yves,5\n'),
+        # Pattern Matching Operators
+        ("LIKE_", "select * from S3Object s where s.name LIKE '_ves'", b'3,Yves,5\n'),
+        ("LIKE%", "select * from S3Object s where s.name LIKE 'Ell%t'", b'2,Elliot,4\n'),
+        # Unitary Operators
+        ("NULL", "select * from S3Object s where s.name IS NULL", b'4,,0\n'),
+        ("NOT_NULL", "select * from S3Object s where s.age IS NOT NULL",
+            b'1,John,3\n2,Elliot,4\n3,Yves,5\n4,,0\n'),
+        # Math Operators
+        ("+", "select * from S3Object s where s.age = 1+3 ", b'2,Elliot,4\n'),
+        ("-", "select * from S3Object s where s.age = 5-1 ", b'2,Elliot,4\n'),
+        ("*", "select * from S3Object s where s.age = 2*2 ", b'2,Elliot,4\n'),
+        ("%", "select * from S3Object s where s.age = 10%6 ", b'2,Elliot,4\n'),
     ]
 
     try:
@@ -141,20 +151,20 @@ def test_sql_operators_precedence(client, log_output):
     json_testfile = """{"id": 1, "name": "Eric"}"""
 
     tests = [
-       ("-_1", "select -3*3 from S3Object", b'-9\n'),
-       ("*", "select 10-3*2 from S3Object", b'4\n'),
-       ("/", "select 13-10/5 from S3Object", b'11\n'),
-       ("%", "select 13-10%5 from S3Object", b'13\n'),
-       ("+", "select 1+1*3 from S3Object", b'4\n'),
-       ("-_2", "select 1-1*3 from S3Object", b'-2\n'),
-       ("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'),
-       ("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'),
-       ("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
-       ("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'),
-       ("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
-       ("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'),
-       ("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'),
-       ("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'),
+        ("-_1", "select -3*3 from S3Object", b'-9\n'),
+        ("*", "select 10-3*2 from S3Object", b'4\n'),
+        ("/", "select 13-10/5 from S3Object", b'11\n'),
+        ("%", "select 13-10%5 from S3Object", b'13\n'),
+        ("+", "select 1+1*3 from S3Object", b'4\n'),
+        ("-_2", "select 1-1*3 from S3Object", b'-2\n'),
+        ("=", "select * from S3Object as s where s.id = 13-12", b'1,Eric\n'),
+        ("<>", "select * from S3Object as s where s.id <> 1-1", b'1,Eric\n'),
+        ("NOT", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
+        ("AND", "select * from S3Object where true AND true OR false ", b'1,Eric\n'),
+        ("OR", "select * from S3Object where false OR NOT false", b'1,Eric\n'),
+        ("IN", "select * from S3Object as s where s.id <> -1 AND s.id IN (1,2,3)", b'1,Eric\n'),
+        ("BETWEEN", "select * from S3Object as s where s.id <> -1 AND s.id BETWEEN -1 AND 3", b'1,Eric\n'),
+        ("LIKE", "select * from S3Object as s where s.id <> -1 AND s.name LIKE 'E%'", b'1,Eric\n'),
     ]
 
     try:
@@ -168,7 +178,6 @@ def test_sql_operators_precedence(client, log_output):
     print(log_output.json_report())
 
 
-
 def test_sql_functions_agg_cond_conv(client, log_output):
 
     json_testfile = """{"id": 1, "name": "John", "age": 3}
@@ -178,17 +187,18 @@ def test_sql_functions_agg_cond_conv(client, log_output):
 {"id": 5, "name": "Eric", "age": 0}
 """
     tests = [
-       # Aggregate functions
-       ("COUNT", "select count(*) from S3Object s", b'5\n'),
-       ("AVG", "select avg(s.age) from S3Object s", b'3\n'),
-       ("MAX", "select max(s.age) from S3Object s", b'5\n'),
-       ("MIN", "select min(s.age) from S3Object s", b'0\n'),
-       ("SUM", "select sum(s.age) from S3Object s", b'12\n'),
-       # Conditional functions
-       ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'),
-       ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'),
-       ## Conversion functions
-       ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s", b'3.0\n4.0\n5.0\n\n0.0\n'),
+        # Aggregate functions
+        ("COUNT", "select count(*) from S3Object s", b'5\n'),
+        ("AVG", "select avg(s.age) from S3Object s", b'3\n'),
+        ("MAX", "select max(s.age) from S3Object s", b'5\n'),
+        ("MIN", "select min(s.age) from S3Object s", b'0\n'),
+        ("SUM", "select sum(s.age) from S3Object s", b'12\n'),
+        # Conditional functions
+        ("COALESCE", "SELECT COALESCE(s.age, 99) FROM S3Object s", b'3\n4\n5\n99\n0\n'),
+        ("NULLIF", "SELECT NULLIF(s.age, 0) FROM S3Object s", b'3\n4\n5\n\n\n'),
+        # Conversion functions
+        ("CAST", "SELECT CAST(s.age AS FLOAT) FROM S3Object s",
+            b'3.0\n4.0\n5.0\n\n0.0\n'),
 
     ]
 
@@ -210,36 +220,49 @@ def test_sql_functions_date(client, log_output):
 """
 
     tests = [
-       # DATE_ADD
-       ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2022-01-02T03:04:05.006+07:30\n'),
-       ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-02-02T03:04:05.006+07:30\n'),
-       ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-01T03:04:05.006+07:30\n'),
-       ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T04:04:05.006+07:30\n'),
-       ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:09:05.006+07:30\n'),
-       ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017-01-02T03:04:10.006+07:30\n'),
-       # DATE_DIFF
-       ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'),
-       ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'),
-       ("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'),
-       # EXTRACT
-       ("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'),
-       ("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'),
-       ("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'),
-       ("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'),
-       ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'),
-       ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'),
-       # TO_STRING
-       ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s", b'"January 2, 2017"\n'),
-       ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'),
-       ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'),
-       ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'),
-       ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s", b'"January 2, 2017 3:4 AM"\n'),
-       ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
-       ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
-       ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s", b'2017-01-02T3:4:05+0730\n'),
-       ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s", b'2017-01-02T3:4:05+07:30\n'),
-       ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s", b'2017-01-02T03:04:05.006+07:30\n'),
-       ("UTCNOW", "select UTCNOW() from S3Object", datetime(1,1,1)),
+        # DATE_ADD
+        ("DATE_ADD_1", "select DATE_ADD(year, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
+         b'2022-01-02T03:04:05.006+07:30\n'),
+        ("DATE_ADD_2", "select DATE_ADD(month, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
+            b'2017-02-02T03:04:05.006+07:30\n'),
+        ("DATE_ADD_3", "select DATE_ADD(day, -1, TO_TIMESTAMP(s.datez)) from S3Object as s",
+            b'2017-01-01T03:04:05.006+07:30\n'),
+        ("DATE_ADD_4", "select DATE_ADD(hour, 1, TO_TIMESTAMP(s.datez)) from S3Object as s",
+            b'2017-01-02T04:04:05.006+07:30\n'),
+        ("DATE_ADD_5", "select DATE_ADD(minute, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
+            b'2017-01-02T03:09:05.006+07:30\n'),
+        ("DATE_ADD_6", "select DATE_ADD(second, 5, TO_TIMESTAMP(s.datez)) from S3Object as s",
+            b'2017-01-02T03:04:10.006+07:30\n'),
+        # DATE_DIFF
+        ("DATE_DIFF_1", "select DATE_DIFF(year, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011-01-01T')) from S3Object as s", b'-6\n'),
+        ("DATE_DIFF_2", "select DATE_DIFF(month, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2011T')) from S3Object as s", b'-72\n'),
+        ("DATE_DIFF_3", "select DATE_DIFF(day, TO_TIMESTAMP(s.datez), TO_TIMESTAMP('2010-01-02T')) from S3Object as s", b'-2556\n'),
+        # EXTRACT
+        ("EXTRACT_1", "select EXTRACT(year FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'2017\n'),
+        ("EXTRACT_2", "select EXTRACT(month FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'1\n'),
+        ("EXTRACT_3", "select EXTRACT(hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'3\n'),
+        ("EXTRACT_4", "select EXTRACT(minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'4\n'),
+        ("EXTRACT_5", "select EXTRACT(timezone_hour FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'7\n'),
+        ("EXTRACT_6", "select EXTRACT(timezone_minute FROM TO_TIMESTAMP(s.datez)) from S3Object as s", b'30\n'),
+        # TO_STRING
+        ("TO_STRING_1", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y') from S3Object as s",
+            b'"January 2, 2017"\n'),
+        ("TO_STRING_2", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMM d, yyyy') from S3Object as s", b'"Jan 2, 2017"\n'),
+        ("TO_STRING_3", "select TO_STRING(TO_TIMESTAMP(s.datez), 'M-d-yy') from S3Object as s", b'1-2-17\n'),
+        ("TO_STRING_4", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MM-d-y') from S3Object as s", b'01-2-2017\n'),
+        ("TO_STRING_5", "select TO_STRING(TO_TIMESTAMP(s.datez), 'MMMM d, y h:m a') from S3Object as s",
+            b'"January 2, 2017 3:4 AM"\n'),
+        ("TO_STRING_6", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
+            b'2017-01-02T3:4:05+0730\n'),
+        ("TO_STRING_7", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssX') from S3Object as s",
+            b'2017-01-02T3:4:05+0730\n'),
+        ("TO_STRING_8", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXX') from S3Object as s",
+            b'2017-01-02T3:4:05+0730\n'),
+        ("TO_STRING_9", "select TO_STRING(TO_TIMESTAMP(s.datez), 'y-MM-dd''T''H:m:ssXXXXX') from S3Object as s",
+            b'2017-01-02T3:4:05+07:30\n'),
+        ("TO_TIMESTAMP", "select TO_TIMESTAMP(s.datez) from S3Object as s",
+            b'2017-01-02T03:04:05.006+07:30\n'),
+        ("UTCNOW", "select UTCNOW() from S3Object", datetime(1, 1, 1)),
 
     ]
 
@@ -253,6 +276,7 @@ def test_sql_functions_date(client, log_output):
     # Test passes
     print(log_output.json_report())
 
+
 def test_sql_functions_string(client, log_output):
 
     json_testfile = """
@@ -262,23 +286,26 @@ def test_sql_functions_string(client, log_output):
 """
 
     tests = [
-       # CHAR_LENGTH
-       ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
-       ("CHARACTER_LENGTH", "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
-       # LOWER
-       ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'),
-       # SUBSTRING
-       ("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'),
-       ("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'),
-       ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'),
-       # TRIM
-       ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
-       ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t         \n'),
-       ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2", b'       \tfoobar\t\n'),
-       ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
-       ("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'),
-       # UPPER
-       ("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'),
+        # CHAR_LENGTH
+        ("CHAR_LENGTH", "select CHAR_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
+        ("CHARACTER_LENGTH",
+            "select CHARACTER_LENGTH(s.name) from S3Object as s", b'4\n24\n21\n'),
+        # LOWER
+        ("LOWER", "select LOWER(s.name) from S3Object as s where s.id= 1", b'john\n'),
+        # SUBSTRING
+        ("SUBSTRING_1", "select SUBSTRING(s.name FROM 2) from S3Object as s where s.id = 1", b'ohn\n'),
+        ("SUBSTRING_2", "select SUBSTRING(s.name FROM 2 FOR 2) from S3Object as s where s.id = 1", b'oh\n'),
+        ("SUBSTRING_3", "select SUBSTRING(s.name FROM -1 FOR 2) from S3Object as s where s.id = 1", b'\n'),
+        # TRIM
+        ("TRIM_1", "select TRIM(s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
+        ("TRIM_2", "select TRIM(LEADING FROM s.name) from S3Object as s where s.id = 2",
+            b'\tfoobar\t         \n'),
+        ("TRIM_3", "select TRIM(TRAILING FROM s.name) from S3Object as s where s.id = 2",
+            b'       \tfoobar\t\n'),
+        ("TRIM_4", "select TRIM(BOTH FROM s.name) from S3Object as s where s.id = 2", b'\tfoobar\t\n'),
+        ("TRIM_5", "select TRIM(BOTH '12' FROM s.name) from S3Object as s where s.id = 3", b'foobar\n'),
+        # UPPER
+        ("UPPER", "select UPPER(s.name) from S3Object as s where s.id= 1", b'JOHN\n'),
     ]
 
     try:
@@ -297,14 +324,15 @@ def test_sql_datatypes(client, log_output):
 {"name": "John"}
 """
     tests = [
-       ("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'),
-       ("int", "select CAST('13' AS INT) from S3Object", b'13\n'),
-       ("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'),
-       ("string", "select CAST(true AS STRING) from S3Object", b'true\n'),
-       ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'),
-       ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
-       ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
-       ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object", b'2007-04-05T14:30Z\n'),
+        ("bool", "select CAST('true' AS BOOL) from S3Object", b'true\n'),
+        ("int", "select CAST('13' AS INT) from S3Object", b'13\n'),
+        ("integer", "select CAST('13' AS INTEGER) from S3Object", b'13\n'),
+        ("string", "select CAST(true AS STRING) from S3Object", b'true\n'),
+        ("float", "select CAST('13.3' AS FLOAT) from S3Object", b'13.3\n'),
+        ("decimal", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
+        ("numeric", "select CAST('14.3' AS FLOAT) from S3Object", b'14.3\n'),
+        ("timestamp", "select CAST('2007-04-05T14:30Z' AS TIMESTAMP) from S3Object",
+            b'2007-04-05T14:30Z\n'),
     ]
 
     try:
@@ -323,14 +351,17 @@ def test_sql_select(client, log_output):
     json_testfile = """{"id": 1, "created": "June 27", "modified": "July 6" }
 {"id": 2, "Created": "June 28", "Modified": "July 7", "Cast": "Random Date" }"""
     tests = [
-       ("select_1", "select * from S3Object", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
-       ("select_2", "select * from S3Object s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
-       ("select_3", "select * from S3Object as s", b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
-       ("select_4", "select s.line from S3Object as s", b'\n\n'),
-       ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'),
-       ("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'),
-       ("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'),
-       ("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'),
+        ("select_1", "select * from S3Object",
+         b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
+        ("select_2", "select * from S3Object s",
+            b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
+        ("select_3", "select * from S3Object as s",
+            b'1,June 27,July 6\n2,June 28,July 7,Random Date\n'),
+        ("select_4", "select s.line from S3Object as s", b'\n\n'),
+        ("select_5", 'select s."Created" from S3Object as s', b'\nJune 28\n'),
+        ("select_5", 'select s."Cast" from S3Object as s', b'\nRandom Date\n'),
+        ("where", 'select s.created from S3Object as s', b'June 27\nJune 28\n'),
+        ("limit", 'select * from S3Object as s LIMIT 1', b'1,June 27,July 6\n'),
     ]
 
     try:
@@ -343,23 +374,29 @@ def test_sql_select(client, log_output):
     # Test passes
     print(log_output.json_report())
 
+
 def test_sql_select_json(client, log_output):
     json_testcontent = """{ "Rules": [ {"id": "1"}, {"expr": "y > x"}, {"id": "2", "expr": "z = DEBUG"} ]}
 { "created": "June 27", "modified": "July 6" }
 """
     tests = [
-       ("select_1", "SELECT id FROM S3Object[*].Rules[*].id", b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
-       ("select_2", "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
-       ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
-       ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]", b'{}\n{"created":"June 27","modified":"July 6"}\n'),
-       ("select_5", "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
+        ("select_1", "SELECT id FROM S3Object[*].Rules[*].id",
+         b'{"id":"1"}\n{}\n{"id":"2"}\n{}\n'),
+        ("select_2",
+            "SELECT id FROM S3Object[*].Rules[*].id WHERE id IS NOT MISSING", b'{"id":"1"}\n{"id":"2"}\n'),
+        ("select_3", "SELECT d.created, d.modified FROM S3Object[*] d",
+            b'{}\n{"created":"June 27","modified":"July 6"}\n'),
+        ("select_4", "SELECT _1.created, _1.modified FROM S3Object[*]",
+            b'{}\n{"created":"June 27","modified":"July 6"}\n'),
+        ("select_5",
+            "Select s.rules[1].expr from S3Object s", b'{"expr":"y > x"}\n{}\n'),
     ]
 
-    input_serialization = InputSerialization(json=JSONInput(Type="DOCUMENT"))
-    output_serialization = OutputSerialization(json=JsonOutput())
+    input_serialization = InputSerialization(json=JSONInput(json_type="DOCUMENT"))
+    output_serialization = OutputSerialization(json=JSONOutput())
     try:
         test_sql_expressions_custom_input_output(client, json_testcontent,
-                input_serialization, output_serialization, tests, log_output)
+                                                 input_serialization, output_serialization, tests, log_output)
     except Exception as select_err:
         raise select_err
         # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
@@ -374,20 +411,20 @@ def test_sql_select_csv_no_header(client, log_output):
 val4,val5,val6
 """
     tests = [
-       ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
+        ("select_1", "SELECT s._2 FROM S3Object as s", b'val2\nval5\n'),
     ]
 
-    input_serialization=InputSerialization(
-            csv=CSVInput(
-                FileHeaderInfo="NONE",
-                AllowQuotedRecordDelimiter="FALSE",
-                ),
-            )
+    input_serialization = InputSerialization(
+        csv=CSVInput(
+            file_header_info="NONE",
+            allow_quoted_record_delimiter="FALSE",
+        ),
+    )
 
-    output_serialization=OutputSerialization(csv=CSVOutput())
+    output_serialization = OutputSerialization(csv=CSVOutput())
     try:
         test_sql_expressions_custom_input_output(client, json_testcontent,
-                input_serialization, output_serialization, tests, log_output)
+                                                 input_serialization, output_serialization, tests, log_output)
     except Exception as select_err:
         raise select_err
         # raise ValueError('Test {} unexpectedly failed with: {}'.format(test_name, select_err))
@@ -395,5 +432,3 @@ val4,val5,val6
 
     # Test passes
     print(log_output.json_report())
-
-

mint/run/core/s3select/tests.py

@@ -16,12 +16,18 @@
 # limitations under the License.
 
 import os
-from sys import exit
-from minio import Minio
+import sys
+from csv import (test_csv_input_custom_quote_char,
+                 test_csv_output_custom_quote_char)
 
+from minio import Minio
+from sql_ops import (test_sql_datatypes, test_sql_functions_agg_cond_conv,
+                     test_sql_functions_date, test_sql_functions_string,
+                     test_sql_operators, test_sql_operators_precedence,
+                     test_sql_select, test_sql_select_csv_no_header,
+                     test_sql_select_json)
 from utils import LogOutput
-from sql_ops import *
-from csv import *
+
 
 def main():
     """
@@ -39,48 +45,56 @@ def main():
             secret_key = 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG'
             secure = True
 
-        client = Minio(server_endpoint, access_key, secret_key, secure=False)
+        client = Minio(server_endpoint, access_key, secret_key, secure=secure)
 
-        log_output = LogOutput(client.select_object_content, 'test_csv_input_quote_char')
+        log_output = LogOutput(client.select_object_content,
+                               'test_csv_input_quote_char')
         test_csv_input_custom_quote_char(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_csv_output_quote_char')
+        log_output = LogOutput(client.select_object_content,
+                               'test_csv_output_quote_char')
         test_csv_output_custom_quote_char(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_operators')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_operators')
         test_sql_operators(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_operators_precedence')
+        log_output = LogOutput(client.select_object_content,
+                               'test_sql_operators_precedence')
         test_sql_operators_precedence(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_functions_agg_cond_conv')
+        log_output = LogOutput(client.select_object_content,
+                               'test_sql_functions_agg_cond_conv')
         test_sql_functions_agg_cond_conv(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_functions_date')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_functions_date')
         test_sql_functions_date(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_functions_string')
+        log_output = LogOutput(client.select_object_content,
+                               'test_sql_functions_string')
         test_sql_functions_string(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_datatypes')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_datatypes')
         test_sql_datatypes(client, log_output)
 
         log_output = LogOutput(client.select_object_content, 'test_sql_select')
         test_sql_select(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_select_json')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_select_json')
         test_sql_select_json(client, log_output)
 
-        log_output = LogOutput(client.select_object_content, 'test_sql_select_csv')
+        log_output = LogOutput(
+            client.select_object_content, 'test_sql_select_csv')
         test_sql_select_csv_no_header(client, log_output)
 
     except Exception as err:
         print(log_output.json_report(err))
-        exit(1)
+        sys.exit(1)
+
 
 if __name__ == "__main__":
     # Execute only if run as a script
     main()
-
-
-

mint/run/core/s3select/utils.py

@@ -15,11 +15,12 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import uuid
 import inspect
 import json
 import time
 import traceback
+import uuid
+
 
 class LogOutput(object):
     """
@@ -100,7 +101,6 @@ class LogOutput(object):
 def generate_bucket_name():
     return "s3select-test-" + str(uuid.uuid4())
 
+
 def generate_object_name():
     return str(uuid.uuid4())
-
-

pkg/certs/ca-certs.go

@@ -0,0 +1,57 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package certs
+
+import (
+	"crypto/x509"
+	"io/ioutil"
+	"os"
+	"path"
+)
+
+// GetRootCAs - returns all the root CAs into certPool
+// at the input certsCADir
+func GetRootCAs(certsCAsDir string) (*x509.CertPool, error) {
+	rootCAs, _ := x509.SystemCertPool()
+	if rootCAs == nil {
+		// In some systems (like Windows) system cert pool is
+		// not supported or no certificates are present on the
+		// system - so we create a new cert pool.
+		rootCAs = x509.NewCertPool()
+	}
+
+	fis, err := ioutil.ReadDir(certsCAsDir)
+	if err != nil {
+		if os.IsNotExist(err) || os.IsPermission(err) {
+			// Return success if CA's directory is missing or permission denied.
+			err = nil
+		}
+		return rootCAs, err
+	}
+
+	// Load all custom CA files.
+	for _, fi := range fis {
+		caCert, err := ioutil.ReadFile(path.Join(certsCAsDir, fi.Name()))
+		if err != nil {
+			// ignore files which are not readable.
+			continue
+		}
+		rootCAs.AppendCertsFromPEM(caCert)
+	}
+
+	return rootCAs, nil
+}

pkg/certs/ca-certs_test.go

@@ -0,0 +1,78 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package certs
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestGetRootCAs(t *testing.T) {
+	emptydir, err := ioutil.TempDir("", "test-get-root-cas")
+	if err != nil {
+		t.Fatalf("Unable create temp directory. %v", emptydir)
+	}
+	defer os.RemoveAll(emptydir)
+
+	dir1, err := ioutil.TempDir("", "test-get-root-cas")
+	if err != nil {
+		t.Fatalf("Unable create temp directory. %v", dir1)
+	}
+	defer os.RemoveAll(dir1)
+	if err = os.Mkdir(filepath.Join(dir1, "empty-dir"), 0755); err != nil {
+		t.Fatalf("Unable create empty dir. %v", err)
+	}
+
+	dir2, err := ioutil.TempDir("", "test-get-root-cas")
+	if err != nil {
+		t.Fatalf("Unable create temp directory. %v", dir2)
+	}
+	defer os.RemoveAll(dir2)
+	if err = ioutil.WriteFile(filepath.Join(dir2, "empty-file"), []byte{}, 0644); err != nil {
+		t.Fatalf("Unable create test file. %v", err)
+	}
+
+	testCases := []struct {
+		certCAsDir  string
+		expectedErr error
+	}{
+		// ignores non-existent directories.
+		{"nonexistent-dir", nil},
+		// Ignores directories.
+		{dir1, nil},
+		// Ignore empty directory.
+		{emptydir, nil},
+		// Loads the cert properly.
+		{dir2, nil},
+	}
+
+	for _, testCase := range testCases {
+		_, err := GetRootCAs(testCase.certCAsDir)
+
+		if testCase.expectedErr == nil {
+			if err != nil {
+				t.Fatalf("error: expected = <nil>, got = %v", err)
+			}
+		} else if err == nil {
+			t.Fatalf("error: expected = %v, got = <nil>", testCase.expectedErr)
+		} else if testCase.expectedErr.Error() != err.Error() {
+			t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err)
+		}
+	}
+}

pkg/env/web_env.go

@@ -29,7 +29,10 @@ import (
 	"net/url"
 	"os"
 	"regexp"
+	"strings"
 	"time"
+
+	"github.com/dgrijalva/jwt-go"
 )
 
 const (
@@ -46,40 +49,16 @@ func RegisterGlobalCAs(CAs *x509.CertPool) {
 	globalRootCAs = CAs
 }
 
-func isValidEnvScheme(scheme string) bool {
-	switch scheme {
-	case webEnvScheme:
-		fallthrough
-	case webEnvSchemeSecure:
-		return true
-	}
-	return false
-}
-
 var (
 	hostKeys = regexp.MustCompile("^(https?://)(.*?):(.*?)@(.*?)$")
 )
 
-func fetchEnvHTTP(envKey string, u *url.URL) (string, error) {
-	switch u.Scheme {
-	case webEnvScheme:
-		u.Scheme = "http"
-	case webEnvSchemeSecure:
-		u.Scheme = "https"
-	}
-
-	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
-	defer cancel()
-
-	var (
-		username, password string
-	)
-
-	envURL := u.String()
+func fetchHTTPConstituentParts(u *url.URL) (username string, password string, envURL string, err error) {
+	envURL = u.String()
 	if hostKeys.MatchString(envURL) {
 		parts := hostKeys.FindStringSubmatch(envURL)
 		if len(parts) != 5 {
-			return "", errors.New("invalid arguments")
+			return "", "", "", errors.New("invalid arguments")
 		}
 		username = parts[2]
 		password = parts[3]
@@ -90,16 +69,51 @@ func fetchEnvHTTP(envKey string, u *url.URL) (string, error) {
 		username = u.User.Username()
 		password, _ = u.User.Password()
 	}
+	return username, password, envURL, nil
+}
+
+func getEnvValueFromHTTP(urlStr, envKey string) (string, error) {
+	u, err := url.Parse(urlStr)
+	if err != nil {
+		return "", err
+	}
+
+	switch u.Scheme {
+	case webEnvScheme:
+		u.Scheme = "http"
+	case webEnvSchemeSecure:
+		u.Scheme = "https"
+	default:
+		return "", errors.New("invalid arguments")
+	}
+
+	username, password, envURL, err := fetchHTTPConstituentParts(u)
+	if err != nil {
+		return "", err
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
+	defer cancel()
 
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, envURL+"?key="+envKey, nil)
 	if err != nil {
 		return "", err
 	}
 
-	if username != "" && password != "" {
-		req.SetBasicAuth(username, password)
+	claims := &jwt.StandardClaims{
+		ExpiresAt: int64(15 * time.Minute),
+		Issuer:    username,
+		Subject:   envKey,
 	}
 
+	token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims)
+	ss, err := token.SignedString([]byte(password))
+	if err != nil {
+		return "", err
+	}
+
+	req.Header.Set("Authorization", "Bearer "+ss)
+
 	clnt := &http.Client{
 		Transport: &http.Transport{
 			Proxy: http.ProxyFromEnvironment,
@@ -149,19 +163,21 @@ func Environ() []string {
 // to fetch ENV values for the env value from a remote server.
 func LookupEnv(key string) (string, bool) {
 	v, ok := os.LookupEnv(key)
-	if ok {
-		u, err := url.Parse(v)
+	if ok && strings.HasPrefix(v, webEnvScheme) {
+		// If env value starts with `env*://`
+		// continue to parse and fetch from remote
+		var err error
+		v, err = getEnvValueFromHTTP(strings.TrimSpace(v), key)
 		if err != nil {
-			return v, true
-		}
-		if !isValidEnvScheme(u.Scheme) {
-			return v, true
-		}
-		v, err = fetchEnvHTTP(key, u)
-		if err != nil {
-			return "", false
+			// fallback to cached value if-any.
+			return os.LookupEnv("_" + key)
 		}
+		// Set the ENV value to _env value,
+		// this value is a fallback in-case of
+		// server restarts when webhook server
+		// is down.
+		os.Setenv("_"+key, v)
 		return v, true
 	}
-	return "", false
+	return v, ok
 }

pkg/env/web_env_test.go

@@ -0,0 +1,81 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package env
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"github.com/gorilla/mux"
+)
+
+func GetenvHandler(w http.ResponseWriter, r *http.Request) {
+	vars := mux.Vars(r)
+	if vars["namespace"] != "default" {
+		http.Error(w, "namespace not found", http.StatusNotFound)
+		return
+	}
+	if vars["name"] != "minio" {
+		http.Error(w, "tenant not found", http.StatusNotFound)
+		return
+	}
+	if vars["key"] != "MINIO_ARGS" {
+		http.Error(w, "key not found", http.StatusNotFound)
+		return
+	}
+	w.Write([]byte("http://127.0.0.{1..4}:9000/data{1...4}"))
+	w.(http.Flusher).Flush()
+}
+
+func startTestServer(t *testing.T) *httptest.Server {
+	router := mux.NewRouter().SkipClean(true).UseEncodedPath()
+	router.Methods(http.MethodGet).
+		Path("/webhook/v1/getenv/{namespace}/{name}").
+		HandlerFunc(GetenvHandler).Queries("key", "{key:.*}")
+
+	ts := httptest.NewServer(router)
+	t.Cleanup(func() {
+		ts.Close()
+	})
+
+	return ts
+}
+
+func TestWebEnv(t *testing.T) {
+	ts := startTestServer(t)
+
+	u, err := url.Parse(ts.URL)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	v, err := getEnvValueFromHTTP(
+		fmt.Sprintf("env://minio:minio123@%s/webhook/v1/getenv/default/minio",
+			u.Host),
+		"MINIO_ARGS")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if v != "http://127.0.0.{1..4}:9000/data{1...4}" {
+		t.Fatalf("Unexpected value %s", v)
+	}
+}

pkg/licverifier/verifier.go

@@ -0,0 +1,72 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// Package licverifier implements a simple library to verify MinIO Subnet license keys.
+package licverifier
+
+import (
+	"crypto/ecdsa"
+	"errors"
+	"fmt"
+
+	"github.com/dgrijalva/jwt-go"
+)
+
+// LicenseVerifier needs an ECDSA public key in PEM format for initialization.
+type LicenseVerifier struct {
+	ecPubKey *ecdsa.PublicKey
+}
+
+// LicenseInfo holds customer metadata present in the license key.
+type LicenseInfo struct {
+	Email           string `json:"sub"`         // Email of the license key requestor
+	TeamName        string `json:"teamName"`    // Subnet team name
+	AccountID       int64  `json:"accountId"`   // Subnet account id
+	StorageCapacity int64  `json:"capacity"`    // Storage capacity used in TB
+	ServiceType     string `json:"serviceType"` // Subnet service type
+}
+
+// Valid checks if customer metadata from the license key is valid.
+func (li *LicenseInfo) Valid() error {
+	if li.AccountID <= 0 {
+		return errors.New("Invalid accountId in claims")
+	}
+	return nil
+}
+
+// NewLicenseVerifier returns an initialized license verifier with the given
+// ECDSA public key in PEM format.
+func NewLicenseVerifier(pemBytes []byte) (*LicenseVerifier, error) {
+	pbKey, err := jwt.ParseECPublicKeyFromPEM(pemBytes)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to parse public key: %s", err)
+	}
+	return &LicenseVerifier{
+		ecPubKey: pbKey,
+	}, nil
+}
+
+// Verify verifies the license key and validates the claims present in it.
+func (lv *LicenseVerifier) Verify(license string) (LicenseInfo, error) {
+	var licInfo LicenseInfo
+	_, err := jwt.ParseWithClaims(license, &licInfo, func(token *jwt.Token) (interface{}, error) {
+		return lv.ecPubKey, nil
+	})
+	if err != nil {
+		return LicenseInfo{}, fmt.Errorf("Failed to verify license: %s", err)
+	}
+	return licInfo, nil
+}

pkg/licverifier/verifier_test.go

@@ -0,0 +1,88 @@
+/*
+ * MinIO Cloud Storage, (C) 2020 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package licverifier
+
+import (
+	"fmt"
+	"testing"
+)
+
+// TestLicenseVerify tests the license key verification process with a valid and
+// an invalid key.
+func TestLicenseVerify(t *testing.T) {
+	pemBytes := []byte(`-----BEGIN PUBLIC KEY-----
+MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEbo+e1wpBY4tBq9AONKww3Kq7m6QP/TBQ
+mr/cKCUyBL7rcAvg0zNq1vcSrUSGlAmY3SEDCu3GOKnjG/U4E7+p957ocWSV+mQU
+9NKlTdQFGF3+aO6jbQ4hX/S5qPyF+a3z
+-----END PUBLIC KEY-----`)
+	lv, err := NewLicenseVerifier(pemBytes)
+	if err != nil {
+		t.Fatalf("Failed to create license verifier: %s", err)
+	}
+	testCases := []struct {
+		lic             string
+		expectedLicInfo LicenseInfo
+		shouldPass      bool
+	}{{"", LicenseInfo{}, false},
+		{"eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJrcCtjMUBtaW5pby5pbyIsInRlYW1OYW1lIjoiR3JpbmdvdHRzIEluYy4iLCJleHAiOjEuNjI4MjAxODYyNjgwNzM3Nzc1ZTksImNhcGFjaXR5Ijo1MCwiaWF0IjoxLjU5NjY2NTg2MjY4MDczNzc3NWU5LCJhY2NvdW50SWQiOjEsInNlcnZpY2VUeXBlIjoiU1RBTkRBUkQifQ._2EgZpjVGo3hRacO2MNavDqZoaP-hwDQ745Z-t-N6lKDwhHOzwhENb9UhiubOQ_yTJ9Ia5EqMhQrC1QCrk8-ThiftmjFGKTyYw5j7gvox_5L-R8HIegACynVlmBlF6IV", LicenseInfo{
+			Email:           "kp+c1@minio.io",
+			TeamName:        "Gringotts Inc.",
+			AccountID:       1,
+			StorageCapacity: 50,
+			ServiceType:     "STANDARD",
+		}, true},
+	}
+
+	for i, tc := range testCases {
+		licInfo, err := lv.Verify(tc.lic)
+		if err != nil && tc.shouldPass {
+			t.Fatalf("%d: Expected license to pass verification but failed with %s", i+1, err)
+		}
+		if err == nil {
+			if !tc.shouldPass {
+				t.Fatalf("%d: Expected license to fail verification but passed", i+1)
+			}
+			if tc.expectedLicInfo != licInfo {
+				t.Fatalf("%d: Expected license info %v but got %v", i+1, tc.expectedLicInfo, licInfo)
+			}
+		}
+	}
+}
+
+// Example creates a LicenseVerifier using the ECDSA public key in pemBytes. It
+// uses the Verify method of the LicenseVerifier to verify and extract the
+// claims present in the license key.
+func Example() {
+	pemBytes := []byte(`-----BEGIN PUBLIC KEY-----
+MHYwEAYHKoZIzj0CAQYFK4EEACIDYgAEbo+e1wpBY4tBq9AONKww3Kq7m6QP/TBQ
+mr/cKCUyBL7rcAvg0zNq1vcSrUSGlAmY3SEDCu3GOKnjG/U4E7+p957ocWSV+mQU
+9NKlTdQFGF3+aO6jbQ4hX/S5qPyF+a3z
+-----END PUBLIC KEY-----`)
+
+	lv, err := NewLicenseVerifier(pemBytes)
+	if err != nil {
+		fmt.Println("Failed to create license verifier", err)
+	}
+
+	licenseKey := "eyJhbGciOiJFUzM4NCIsInR5cCI6IkpXVCJ9.eyJzdWIiOiJrcCtjMUBtaW5pby5pbyIsInRlYW1OYW1lIjoiR3JpbmdvdHRzIEluYy4iLCJleHAiOjEuNjI4MjAxODYyNjgwNzM3Nzc1ZTksImNhcGFjaXR5Ijo1MCwiaWF0IjoxLjU5NjY2NTg2MjY4MDczNzc3NWU5LCJhY2NvdW50SWQiOjEsInNlcnZpY2VUeXBlIjoiU1RBTkRBUkQifQ._2EgZpjVGo3hRacO2MNavDqZoaP-hwDQ745Z-t-N6lKDwhHOzwhENb9UhiubOQ_yTJ9Ia5EqMhQrC1QCrk8-ThiftmjFGKTyYw5j7gvox_5L-R8HIegACynVlmBlF6IV"
+	licInfo, err := lv.Verify(licenseKey)
+	if err != nil {
+		fmt.Println("Failed to verify license key", err)
+	}
+
+	fmt.Println("License metadata", licInfo)
+}

pkg/trace/trace.go

@@ -42,6 +42,7 @@ type CallStats struct {
 // RequestInfo represents trace of http request
 type RequestInfo struct {
 	Time     time.Time   `json:"time"`
+	Proto    string      `json:"proto"`
 	Method   string      `json:"method"`
 	Path     string      `json:"path,omitempty"`
 	RawQuery string      `json:"rawquery,omitempty"`