Make sure to drain body upon an error (#7197)

Also cleanup redundant code and use it at a common place

Dockerfile.simpleci

@@ -0,0 +1,49 @@
+#-------------------------------------------------------------
+# Stage 1: Build and Unit tests
+#-------------------------------------------------------------
+FROM golang:1.11.4
+
+COPY . /go/src/github.com/minio/minio
+WORKDIR /go/src/github.com/minio/minio
+
+RUN apt-get update && apt-get install -y jq
+RUN make
+RUN bash -c 'diff -au <(gofmt -s -d cmd) <(printf "")'
+RUN bash -c 'diff -au <(gofmt -s -d pkg) <(printf "")'
+RUN for d in $(go list ./... | grep -v browser); do go test -v -race --timeout 15m "$d"; done
+RUN make verify
+RUN make coverage || true
+
+#-------------------------------------------------------------
+# Stage 2: Test Frontend
+#-------------------------------------------------------------
+FROM node:10.15-stretch-slim
+
+COPY browser /minio/browser
+WORKDIR /minio/browser
+
+RUN yarn
+RUN yarn test
+
+#-------------------------------------------------------------
+# Stage 3: Run Gateway Tests
+#-------------------------------------------------------------
+
+FROM ubuntu:16.04
+
+COPY --from=0 /go/src/github.com/minio/minio/minio ./minio
+COPY buildscripts/gateway-tests.sh ./gateway-tests.sh
+RUN apt-get update && apt-get install -y git wget jq curl dnsmasq
+
+RUN wget https://dl.google.com/go/go1.11.4.linux-amd64.tar.gz && \
+	tar -C /usr/local -xzf go1.11.4.linux-amd64.tar.gz
+
+ENV DEBIAN_FRONTEND noninteractive
+ENV LANG C.UTF-8
+ENV GOROOT /usr/local/go
+
+RUN mkdir -p /go
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH
+
+RUN ./gateway-tests.sh

buildscripts/gateway-tests.sh

@@ -0,0 +1,60 @@
+#!/bin/bash
+#
+# Minio Cloud Storage, (C) 2019 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+set -E
+set -o pipefail
+
+export SERVER_ENDPOINT=127.0.0.1:24240
+export ENABLE_HTTPS=0
+export ACCESS_KEY=minio
+export SECRET_KEY=minio123
+export MINIO_ACCESS_KEY=minio
+export MINIO_SECRET_KEY=minio123
+export AWS_ACCESS_KEY_ID=minio
+export AWS_SECRET_ACCESS_KEY=minio123
+
+trap "cat server.log;cat gateway.log" SIGHUP SIGINT SIGTERM
+
+./minio --quiet --json server data --address 127.0.0.1:24242 > server.log &
+sleep 3
+./minio --quiet --json gateway s3 http://127.0.0.1:24242 --address 127.0.0.1:24240 > gateway.log &
+sleep 3
+
+mkdir -p /mint
+git clone https://github.com/minio/mint /mint
+cd /mint
+
+export MINT_ROOT_DIR=${MINT_ROOT_DIR:-/mint}
+export MINT_RUN_CORE_DIR="$MINT_ROOT_DIR/run/core"
+export MINT_RUN_SECURITY_DIR="$MINT_ROOT_DIR/run/security"
+export WGET="wget --quiet --no-check-certificate"
+
+go get github.com/go-ini/ini
+
+./create-data-files.sh
+./preinstall.sh
+
+# install mint app packages
+for pkg in "build"/*/install.sh; do
+    echo "Running $pkg"
+    $pkg
+done
+
+./postinstall.sh
+
+/mint/entrypoint.sh || cat server.log gateway.log fail

cmd/admin-handlers.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016, 2017, 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,6 +25,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"sort"
 	"strconv"
 	"strings"
 	"time"
@@ -299,7 +300,7 @@ func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Reque
 	writeSuccessResponseJSON(w, jsonBytes)
 }
 
-// ServerDrivesPerfInfo holds informantion about address, performance
+// ServerDrivesPerfInfo holds information about address, performance
 // of all drives on one server. It also reports any errors if encountered
 // while trying to reach this server.
 type ServerDrivesPerfInfo struct {
@@ -312,18 +313,20 @@ type ServerDrivesPerfInfo struct {
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.
 type ServerCPULoadInfo struct {
-	Addr  string     `json:"addr"`
-	Error string     `json:"error,omitempty"`
-	Load  []cpu.Load `json:"load"`
+	Addr         string     `json:"addr"`
+	Error        string     `json:"error,omitempty"`
+	Load         []cpu.Load `json:"load"`
+	HistoricLoad []cpu.Load `json:"historicLoad"`
 }
 
 // ServerMemUsageInfo holds informantion about memory utilization
 // of one minio node. It also reports any errors if encountered
 // while trying to reach this server.
 type ServerMemUsageInfo struct {
-	Addr  string      `json:"addr"`
-	Error string      `json:"error,omitempty"`
-	Usage []mem.Usage `json:"usage"`
+	Addr          string      `json:"addr"`
+	Error         string      `json:"error,omitempty"`
+	Usage         []mem.Usage `json:"usage"`
+	HistoricUsage []mem.Usage `json:"historicUsage"`
 }
 
 // PerfInfoHandler - GET /minio/admin/v1/performance?perfType={perfType}
@@ -415,6 +418,103 @@ func (a adminAPIHandlers) PerfInfoHandler(w http.ResponseWriter, r *http.Request
 	return
 }
 
+func newLockEntry(l lockRequesterInfo, resource, server string) *madmin.LockEntry {
+	entry := &madmin.LockEntry{Timestamp: l.Timestamp, Resource: resource, ServerList: []string{server}, Owner: l.Node, Source: l.Source, ID: l.UID}
+	if l.Writer {
+		entry.Type = "Write"
+	} else {
+		entry.Type = "Read"
+	}
+	return entry
+}
+
+func topLockEntries(peerLocks []*PeerLocks) madmin.LockEntries {
+	const listCount int = 10
+	entryMap := make(map[string]*madmin.LockEntry)
+	for _, peerLock := range peerLocks {
+		if peerLock == nil {
+			continue
+		}
+		for k, v := range peerLock.Locks {
+			for _, lockReqInfo := range v {
+				if val, ok := entryMap[lockReqInfo.UID]; ok {
+					val.ServerList = append(val.ServerList, peerLock.Addr)
+				} else {
+					entryMap[lockReqInfo.UID] = newLockEntry(lockReqInfo, k, peerLock.Addr)
+				}
+			}
+		}
+	}
+	var lockEntries = make(madmin.LockEntries, 0)
+	for _, v := range entryMap {
+		lockEntries = append(lockEntries, *v)
+	}
+	sort.Sort(lockEntries)
+	if len(lockEntries) > listCount {
+		lockEntries = lockEntries[:listCount]
+	}
+	return lockEntries
+}
+
+// PeerLocks holds server information result of one node
+type PeerLocks struct {
+	Addr  string
+	Locks GetLocksResp
+}
+
+// TopLocksHandler Get list of locks in use
+func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "TopLocks")
+
+	objectAPI := newObjectLayerFn()
+	if objectAPI == nil || globalNotificationSys == nil {
+		writeErrorResponseJSON(w, ErrServerNotInitialized, r.URL)
+		return
+	}
+
+	// Method only allowed in Distributed XL mode.
+	if globalIsDistXL == false {
+		writeErrorResponseJSON(w, ErrMethodNotAllowed, r.URL)
+		return
+	}
+
+	// Authenticate request
+	// Setting the region as empty so as the mc server info command is irrespective to the region.
+	adminAPIErr := checkAdminRequestAuthType(ctx, r, "")
+	if adminAPIErr != ErrNone {
+		writeErrorResponseJSON(w, adminAPIErr, r.URL)
+		return
+	}
+
+	thisAddr, err := xnet.ParseHost(GetLocalPeer(globalEndpoints))
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+		return
+	}
+
+	peerLocks := globalNotificationSys.GetLocks(ctx)
+	// Once we have received all the locks currently used from peers
+	// add the local peer locks list as well.
+	localLocks := globalLockServer.ll.DupLockMap()
+	peerLocks = append(peerLocks, &PeerLocks{
+		Addr:  thisAddr.String(),
+		Locks: localLocks,
+	})
+
+	topLocks := topLockEntries(peerLocks)
+
+	// Marshal API response
+	jsonBytes, err := json.Marshal(topLocks)
+	if err != nil {
+		writeErrorResponseJSON(w, toAdminAPIErrCode(ctx, err), r.URL)
+		return
+	}
+
+	// Reply with storage information (across nodes in a
+	// distributed setup) as json.
+	writeSuccessResponseJSON(w, jsonBytes)
+}
+
 // StartProfilingResult contains the status of the starting
 // profiling action in a given server
 type StartProfilingResult struct {

cmd/admin-handlers_test.go

@@ -119,7 +119,6 @@ var (
         "broker": "",
         "topic": "",
         "qos": 0,
-        "clientId": "",
         "username": "",
         "password": "",
         "reconnectInterval": 0,
@@ -961,81 +960,3 @@ func collectHealResults(t *testing.T, adminTestBed *adminXLTestBed, bucket,
 
 	return res
 }
-
-func TestHealStartNStatusHandler(t *testing.T) {
-	adminTestBed, err := prepareAdminXLTestBed()
-	if err != nil {
-		t.Fatal("Failed to initialize a single node XL backend for admin handler tests.")
-	}
-	defer adminTestBed.TearDown()
-
-	// gen. test data
-	adminTestBed.GenerateHealTestData(t)
-	defer adminTestBed.CleanupHealTestData(t)
-
-	// Prepare heal-start request to send to the server.
-	healOpts := madmin.HealOpts{
-		Recursive: true,
-		DryRun:    false,
-	}
-	bucketName, objName := "mybucket", "myobject-0"
-	var hss madmin.HealStartSuccess
-
-	{
-		req := mkHealStartReq(t, bucketName, objName, healOpts)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if http.StatusOK != rec.Code {
-			t.Errorf("Unexpected status code - got %d but expected %d",
-				rec.Code, http.StatusOK)
-		}
-
-		err = json.Unmarshal(rec.Body.Bytes(), &hss)
-		if err != nil {
-			t.Fatal("unable to unmarshal response")
-		}
-
-		if hss.ClientToken == "" {
-			t.Errorf("unexpected result")
-		}
-	}
-
-	{
-		// test with an invalid client token
-		req := mkHealStatusReq(t, bucketName, objName, hss.ClientToken+hss.ClientToken)
-		rec := httptest.NewRecorder()
-		adminTestBed.router.ServeHTTP(rec, req)
-		if rec.Code != http.StatusBadRequest {
-			t.Errorf("Unexpected status code")
-		}
-	}
-
-	{
-		// fetch heal status
-		results := collectHealResults(t, adminTestBed, bucketName,
-			objName, hss.ClientToken, 5)
-
-		// check if we got back an expected record
-		foundIt := false
-		for _, item := range results.Items {
-			if item.Type == madmin.HealItemObject &&
-				item.Bucket == bucketName && item.Object == objName {
-				foundIt = true
-			}
-		}
-		if !foundIt {
-			t.Error("did not find expected heal record in heal results")
-		}
-
-		// check that the heal settings in the results is the
-		// same as what we started the heal seq. with.
-		if results.HealSettings != healOpts {
-			t.Errorf("unexpected heal settings: %v",
-				results.HealSettings)
-		}
-
-		if results.Summary == healStoppedStatus {
-			t.Errorf("heal sequence stopped unexpectedly")
-		}
-	}
-}

cmd/admin-heal-ops.go

@@ -596,7 +596,7 @@ func (h *healSequence) healConfig() error {
 				return errHealStopSignalled
 			}
 			o := objectInfos.Objects[index]
-			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun)
+			res, herr := objectAPI.HealObject(h.ctx, o.Bucket, o.Name, h.settings.DryRun, h.settings.Remove)
 			// Object might have been deleted, by the time heal
 			// was attempted we ignore this file an move on.
 			if isErrObjectNotFound(herr) {
@@ -692,7 +692,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		return errServerNotInitialized
 	}
 
-	results, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun)
+	results, err := objectAPI.HealBucket(h.ctx, bucket, h.settings.DryRun, h.settings.Remove)
 	// push any available results before checking for error
 	for _, result := range results {
 		if perr := h.pushHealResultItem(result); perr != nil {
@@ -719,7 +719,7 @@ func (h *healSequence) healBucket(bucket string) error {
 		return nil
 	}
 
-	entries := runtime.NumCPU() * globalEndpoints.Nodes()
+	entries := runtime.NumCPU()
 
 	marker := ""
 	isTruncated := true
@@ -775,7 +775,7 @@ func (h *healSequence) healObject(bucket, object string) error {
 		return errServerNotInitialized
 	}
 
-	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun)
+	hri, err := objectAPI.HealObject(h.ctx, bucket, object, h.settings.DryRun, h.settings.Remove)
 	if isErrObjectNotFound(err) {
 		return nil
 	}

cmd/admin-router.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -114,6 +114,10 @@ func registerAdminRouter(router *mux.Router, enableConfigOps, enableIAMOps bool)
 		adminV1Router.Methods(http.MethodGet).Path("/list-canned-policies").HandlerFunc(httpTraceHdrs(adminAPI.ListCannedPolicies))
 	}
 
+	// -- Top APIs --
+	// Top locks
+	adminV1Router.Methods(http.MethodGet).Path("/top/locks").HandlerFunc(httpTraceHdrs(adminAPI.TopLocksHandler))
+
 	// If none of the routes match, return error.
 	adminV1Router.NotFoundHandler = http.HandlerFunc(httpTraceHdrs(notFoundHandlerJSON))
 }

cmd/config-common.go

@@ -103,16 +103,14 @@ func readConfigEtcd(ctx context.Context, client *etcd.Client, configFile string)
 	return nil, errConfigNotFound
 }
 
-// watchConfig - watches for changes on `configFile` on etcd and loads them.
-func watchConfig(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
-	if globalEtcdClient != nil {
-		ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
-		defer cancel()
-		for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
-			for _, event := range watchResp.Events {
-				if event.IsModify() || event.IsCreate() {
-					loadCfgFn(objAPI)
-				}
+// watchConfigEtcd - watches for changes on `configFile` on etcd and loads them.
+func watchConfigEtcd(objAPI ObjectLayer, configFile string, loadCfgFn func(ObjectLayer) error) {
+	ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout)
+	defer cancel()
+	for watchResp := range globalEtcdClient.Watch(ctx, configFile) {
+		for _, event := range watchResp.Events {
+			if event.IsModify() || event.IsCreate() {
+				loadCfgFn(objAPI)
 			}
 		}
 	}

cmd/config-current.go

@@ -25,6 +25,7 @@ import (
 	"sync"
 
 	"github.com/minio/minio/cmd/crypto"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/event"
@@ -282,7 +283,7 @@ func (s *serverConfig) loadFromEnvs() {
 	if jwksURL, ok := os.LookupEnv("MINIO_IAM_JWKS_URL"); ok {
 		if u, err := xnet.ParseURL(jwksURL); err == nil {
 			s.OpenID.JWKS.URL = u
-			s.OpenID.JWKS.PopulatePublicKey()
+			logger.FatalIf(s.OpenID.JWKS.PopulatePublicKey(), "Unable to populate public key from JWKS URL")
 		}
 	}
 
@@ -545,19 +546,15 @@ func (s *serverConfig) loadToCachedConfigs() {
 		globalIsCompressionEnabled = compressionConf.Enabled
 	}
 
-	if globalIAMValidators == nil {
-		globalIAMValidators = getAuthValidators(s)
-	}
+	globalIAMValidators = getAuthValidators(s)
 
-	if globalPolicyOPA == nil {
-		if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
-			globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
-				URL:         s.Policy.OPA.URL,
-				AuthToken:   s.Policy.OPA.AuthToken,
-				Transport:   NewCustomHTTPTransport(),
-				CloseRespFn: CloseResponse,
-			})
-		}
+	if s.Policy.OPA.URL != nil && s.Policy.OPA.URL.String() != "" {
+		globalPolicyOPA = iampolicy.NewOpa(iampolicy.OpaArgs{
+			URL:         s.Policy.OPA.URL,
+			AuthToken:   s.Policy.OPA.AuthToken,
+			Transport:   NewCustomHTTPTransport(),
+			CloseRespFn: xhttp.DrainBody,
+		})
 	}
 }
 

cmd/config-current_test.go

@@ -233,7 +233,7 @@ func TestValidateConfig(t *testing.T) {
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "redis": { "1": { "enable": true, "format": "namespace", "address": "example.com:80", "password": "xxx", "key": "key1" } }}}`, true},
 
 		// Test 27 - Test MQTT
-		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "clientId": "", "username": "", "password": "", "queueDir": ""}}}}`, false},
+		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "mqtt": { "1": { "enable": true, "broker": "",  "topic": "", "qos": 0, "username": "", "password": "", "queueDir": ""}}}}`, false},
 
 		// Test 28 - Test NSQ
 		{`{"version": "` + v + `", "credential": { "accessKey": "minio", "secretKey": "minio123" }, "region": "us-east-1", "browser": "on", "notify": { "nsq": { "1": { "enable": true, "nsqdAddress": "", "topic": ""} }}}`, false},

cmd/config-versions.go

@@ -893,7 +893,7 @@ type serverConfigV32 struct {
 	} `json:"policy"`
 }
 
-// serverConfigV33 is just like version '32', removes clientID from NATS and adds queueDir with MQTT.
+// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir with MQTT.
 type serverConfigV33 struct {
 	quick.Config `json:"-"` // ignore interfaces
 

cmd/config.go

@@ -157,6 +157,8 @@ func initConfig(objAPI ObjectLayer) error {
 		return errServerNotInitialized
 	}
 
+	configFile := path.Join(minioConfigPrefix, minioConfigFile)
+
 	if globalEtcdClient != nil {
 		if err := checkConfigEtcd(context.Background(), globalEtcdClient, getConfigFile()); err != nil {
 			if err == errConfigNotFound {
@@ -172,6 +174,10 @@ func initConfig(objAPI ObjectLayer) error {
 				return err
 			}
 		}
+
+		// Watch config for changes and reloads them.
+		go watchConfigEtcd(objAPI, configFile, loadConfig)
+
 	} else {
 		if isFile(getConfigFile()) {
 			if err := migrateConfig(); err != nil {
@@ -184,15 +190,11 @@ func initConfig(objAPI ObjectLayer) error {
 		if err := migrateConfigToMinioSys(objAPI); err != nil {
 			return err
 		}
-	}
 
-	configFile := path.Join(minioConfigPrefix, minioConfigFile)
-
-	// Watch config for changes and reloads them in-memory.
-	go watchConfig(objAPI, configFile, loadConfig)
-
-	if err := migrateMinioSysConfig(objAPI); err != nil {
-		return err
+		// Migrates backend '<export_path>/.minio.sys/config/config.json' to latest version.
+		if err := migrateMinioSysConfig(objAPI); err != nil {
+			return err
+		}
 	}
 
 	return loadConfig(objAPI)

cmd/crypto/vault.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	vault "github.com/hashicorp/vault/api"
+	"github.com/minio/minio/cmd/logger"
 )
 
 var (
@@ -63,6 +64,7 @@ type vaultService struct {
 	config        *VaultConfig
 	client        *vault.Client
 	leaseDuration time.Duration
+	tokenRenewer  *vault.Renewer
 }
 
 var _ KMS = (*vaultService)(nil) // compiler check that *vaultService implements KMS
@@ -120,42 +122,80 @@ func NewVault(config VaultConfig) (KMS, error) {
 	if config.Namespace != "" {
 		client.SetNamespace(config.Namespace)
 	}
+	v := &vaultService{client: client, config: &config}
 
-	payload := map[string]interface{}{
-		"role_id":   config.Auth.AppRole.ID,
-		"secret_id": config.Auth.AppRole.Secret,
-	}
-	resp, err := client.Logical().Write("auth/approle/login", payload)
-	if err != nil {
+	if err := v.authenticate(); err != nil {
 		return nil, err
 	}
-	if resp.Auth == nil {
-		return nil, ErrKMSAuthLogin
-	}
-
-	client.SetToken(resp.Auth.ClientToken)
-	v := &vaultService{client: client, config: &config, leaseDuration: time.Duration(resp.Auth.LeaseDuration)}
-	v.renewToken()
 	return v, nil
 }
 
-// renewToken starts a new go-routine which renews
-// the vault authentication token periodically.
-func (v *vaultService) renewToken() {
+// reauthenticate() tries to login in 1 minute
+// intervals until successful.
+func (v *vaultService) reauthenticate() {
 	retryDelay := 1 * time.Minute
 	go func() {
 		for {
-			s, err := v.client.Auth().Token().RenewSelf(int(v.leaseDuration))
-			if err != nil {
+			if err := v.authenticate(); err != nil {
 				time.Sleep(retryDelay)
 				continue
 			}
-			nextRenew := s.Auth.LeaseDuration / 2
-			time.Sleep(time.Duration(nextRenew) * time.Second)
+			return
 		}
 	}()
 }
 
+// renewer calls vault client's renewer that automatically
+// renews secret periodically
+func (v *vaultService) renewer(secret *vault.Secret) {
+	renewer, err := v.client.NewRenewer(&vault.RenewerInput{
+		Secret: secret,
+	})
+	if err != nil {
+		logger.FatalIf(err, "crypto: hashicorp vault token renewer could not be started")
+	}
+	v.tokenRenewer = renewer
+	go renewer.Renew()
+	defer renewer.Stop()
+
+	for {
+		select {
+		case err := <-renewer.DoneCh():
+			if err != nil {
+				v.reauthenticate()
+				renewer.Stop()
+				return
+			}
+
+			// Renewal is now over
+		case renewal := <-renewer.RenewCh():
+			v.leaseDuration = time.Duration(renewal.Secret.Auth.LeaseDuration)
+		}
+	}
+}
+
+// authenticate logs the app to vault, and starts the auto renewer
+// before secret expires
+func (v *vaultService) authenticate() (err error) {
+	payload := map[string]interface{}{
+		"role_id":   v.config.Auth.AppRole.ID,
+		"secret_id": v.config.Auth.AppRole.Secret,
+	}
+	var secret *vault.Secret
+	secret, err = v.client.Logical().Write("auth/approle/login", payload)
+	if err != nil {
+		return
+	}
+	if secret.Auth == nil {
+		err = ErrKMSAuthLogin
+		return
+	}
+	v.client.SetToken(secret.Auth.ClientToken)
+	v.leaseDuration = time.Duration(secret.Auth.LeaseDuration)
+	go v.renewer(secret)
+	return
+}
+
 // GenerateKey returns a new plaintext key, generated by the KMS,
 // and a sealed version of this plaintext key encrypted using the
 // named key referenced by keyID. It also binds the generated key

cmd/dummy-object-layer_test.go

@@ -119,11 +119,11 @@ func (api *DummyObjectLayer) HealFormat(ctx context.Context, dryRun bool) (item
 	return
 }
 
-func (api *DummyObjectLayer) HealBucket(ctx context.Context, bucket string, dryRun bool) (items []madmin.HealResultItem, err error) {
+func (api *DummyObjectLayer) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (items []madmin.HealResultItem, err error) {
 	return
 }
 
-func (api *DummyObjectLayer) HealObject(ctx context.Context, bucket, object string, dryRun bool) (item madmin.HealResultItem, err error) {
+func (api *DummyObjectLayer) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (item madmin.HealResultItem, err error) {
 	return
 }
 

cmd/endpoint.go

@@ -204,6 +204,7 @@ func (endpoints EndpointList) GetString(i int) string {
 // local endpoints from given list of endpoints
 func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
 	var memUsages []mem.Usage
+	var historicUsages []mem.Usage
 	var addr string
 	scratchSpace := map[string]bool{}
 	for _, endpoint := range endpoints {
@@ -215,12 +216,15 @@ func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
 			addr = GetLocalPeer(endpoints)
 			memUsage := mem.GetUsage()
 			memUsages = append(memUsages, memUsage)
+			historicUsage := mem.GetHistoricUsage()
+			historicUsages = append(historicUsages, historicUsage)
 			scratchSpace[endpoint.Host] = true
 		}
 	}
 	return ServerMemUsageInfo{
-		Addr:  addr,
-		Usage: memUsages,
+		Addr:          addr,
+		Usage:         memUsages,
+		HistoricUsage: historicUsages,
 	}
 }
 
@@ -228,6 +232,7 @@ func localEndpointsMemUsage(endpoints EndpointList) ServerMemUsageInfo {
 // local endpoints from given list of endpoints
 func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
 	var cpuLoads []cpu.Load
+	var historicLoads []cpu.Load
 	var addr string
 	scratchSpace := map[string]bool{}
 	for _, endpoint := range endpoints {
@@ -239,12 +244,15 @@ func localEndpointsCPULoad(endpoints EndpointList) ServerCPULoadInfo {
 			addr = GetLocalPeer(endpoints)
 			cpuLoad := cpu.GetLoad()
 			cpuLoads = append(cpuLoads, cpuLoad)
+			historicLoad := cpu.GetHistoricLoad()
+			historicLoads = append(historicLoads, historicLoad)
 			scratchSpace[endpoint.Host] = true
 		}
 	}
 	return ServerCPULoadInfo{
-		Addr: addr,
-		Load: cpuLoads,
+		Addr:         addr,
+		Load:         cpuLoads,
+		HistoricLoad: historicLoads,
 	}
 }
 

cmd/fs-v1.go

@@ -1245,14 +1245,14 @@ func (fs *FSObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRe
 }
 
 // HealObject - no-op for fs. Valid only for XL.
-func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool) (
+func (fs *FSObjects) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (
 	res madmin.HealResultItem, err error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return res, NotImplemented{}
 }
 
 // HealBucket - no-op for fs, Valid only for XL.
-func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem,
+func (fs *FSObjects) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) ([]madmin.HealResultItem,
 	error) {
 	logger.LogIf(ctx, NotImplemented{})
 	return nil, NotImplemented{}

cmd/fs-v1_test.go

@@ -390,7 +390,7 @@ func TestFSHealObject(t *testing.T) {
 	defer os.RemoveAll(disk)
 
 	obj := initFSObjects(disk, t)
-	_, err := obj.HealObject(context.Background(), "bucket", "object", false)
+	_, err := obj.HealObject(context.Background(), "bucket", "object", false, false)
 	if err == nil || !isSameType(err, NotImplemented{}) {
 		t.Fatalf("Heal Object should return NotImplemented error ")
 	}

cmd/gateway-main.go

@@ -203,6 +203,9 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		// Override any values from ENVs.
 		srvCfg.loadFromEnvs()
 
+		// Load values to cached global values.
+		srvCfg.loadToCachedConfigs()
+
 		// hold the mutex lock before a new config is assigned.
 		globalServerConfigMu.Lock()
 		globalServerConfig = srvCfg
@@ -223,7 +226,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 		globalConfigSys = NewConfigSys()
 
 		// Load globalServerConfig from etcd
-		_ = globalConfigSys.Init(newObject)
+		logger.LogIf(context.Background(), globalConfigSys.Init(newObject))
 	}
 
 	// Load logger subsystem
@@ -247,7 +250,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	globalIAMSys = NewIAMSys()
 	if enableIAMOps {
 		// Initialize IAM sys.
-		_ = globalIAMSys.Init(newObject)
+		logger.LogIf(context.Background(), globalIAMSys.Init(newObject))
 	}
 
 	// Create new policy system.
@@ -259,7 +262,7 @@ func StartGateway(ctx *cli.Context, gw Gateway) {
 	// Create new notification system.
 	globalNotificationSys = NewNotificationSys(globalServerConfig, globalEndpoints)
 	if globalEtcdClient != nil && newObject.IsNotificationSupported() {
-		_ = globalNotificationSys.Init(newObject)
+		logger.LogIf(context.Background(), globalNotificationSys.Init(newObject))
 	}
 
 	// Encryption support checks in gateway mode.

cmd/gateway-unsupported.go

@@ -92,7 +92,7 @@ func (a GatewayUnsupported) HealFormat(ctx context.Context, dryRun bool) (madmin
 }
 
 // HealBucket - Not implemented stub
-func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error) {
+func (a GatewayUnsupported) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) ([]madmin.HealResultItem, error) {
 	return nil, NotImplemented{}
 }
 
@@ -102,7 +102,7 @@ func (a GatewayUnsupported) ListBucketsHeal(ctx context.Context) (buckets []Buck
 }
 
 // HealObject - Not implemented stub
-func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun bool) (h madmin.HealResultItem, e error) {
+func (a GatewayUnsupported) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (h madmin.HealResultItem, e error) {
 	return h, NotImplemented{}
 }
 

cmd/gateway/nas/gateway-nas.go

@@ -17,6 +17,8 @@
 package nas
 
 import (
+	"context"
+
 	"github.com/minio/cli"
 	minio "github.com/minio/minio/cmd"
 	"github.com/minio/minio/pkg/auth"
@@ -121,6 +123,12 @@ func (n *nasObjects) IsListenBucketSupported() bool {
 	return false
 }
 
+func (n *nasObjects) StorageInfo(ctx context.Context) minio.StorageInfo {
+	sinfo := n.ObjectLayer.StorageInfo(ctx)
+	sinfo.Backend.Type = minio.Unknown
+	return sinfo
+}
+
 // nasObjects implements gateway for Minio and S3 compatible object storage servers.
 type nasObjects struct {
 	minio.ObjectLayer

cmd/gateway/oss/gateway-oss.go

@@ -31,6 +31,7 @@ import (
 	"github.com/minio/cli"
 	miniogopolicy "github.com/minio/minio-go/pkg/policy"
 	minio "github.com/minio/minio/cmd"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/auth"
 	"github.com/minio/minio/pkg/hash"
@@ -844,7 +845,7 @@ func ossListObjectParts(client *oss.Client, bucket, object, uploadID string, par
 	}
 
 	// always drain output (response body)
-	defer minio.CloseResponse(resp.Body)
+	defer xhttp.DrainBody(resp.Body)
 
 	err = xml.NewDecoder(resp.Body).Decode(&lupr)
 	if err != nil {

cmd/gateway/s3/gateway-s3.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2017, 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -177,8 +177,13 @@ func isAmazonS3Endpoint(urlStr string) bool {
 	return s3utils.IsAmazonEndpoint(*u)
 }
 
-// - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
-var defaultMinioProviders = []credentials.Provider{
+// Chains all credential types, in the following order:
+//  - AWS env vars (i.e. AWS_ACCESS_KEY_ID)
+//  - AWS creds file (i.e. AWS_SHARED_CREDENTIALS_FILE or ~/.aws/credentials)
+//  - Static credentials provided by user (i.e. MINIO_ACCESS_KEY)
+var defaultProviders = []credentials.Provider{
+	&credentials.EnvAWS{},
+	&credentials.FileAWSCredentials{},
 	&credentials.EnvMinio{},
 }
 
@@ -196,6 +201,7 @@ var defaultAWSCredProviders = []credentials.Provider{
 			Transport: minio.NewCustomHTTPTransport(),
 		},
 	},
+	&credentials.EnvMinio{},
 }
 
 // newS3 - Initializes a new client by auto probing S3 server signature.
@@ -213,9 +219,10 @@ func newS3(urlStr string) (*miniogo.Core, error) {
 	var creds *credentials.Credentials
 	if isAmazonS3Endpoint(urlStr) {
 		// If we see an Amazon S3 endpoint, then we use more ways to fetch backend credentials.
-		creds = credentials.NewChainCredentials(append(defaultAWSCredProviders, defaultMinioProviders...))
+		// Specifically IAM style rotating credentials are only supported with AWS S3 endpoint.
+		creds = credentials.NewChainCredentials(defaultAWSCredProviders)
 	} else {
-		creds = credentials.NewChainCredentials(defaultMinioProviders)
+		creds = credentials.NewChainCredentials(defaultProviders)
 	}
 
 	clnt, err := miniogo.NewWithCredentials(endpoint, creds, secure, "")

cmd/generic-handlers.go

@@ -381,37 +381,9 @@ type resourceHandler struct {
 	handler http.Handler
 }
 
-// List of default allowable HTTP methods.
-var defaultAllowableHTTPMethods = []string{
-	http.MethodGet,
-	http.MethodPut,
-	http.MethodHead,
-	http.MethodPost,
-	http.MethodDelete,
-	http.MethodOptions,
-}
-
 // setCorsHandler handler for CORS (Cross Origin Resource Sharing)
 func setCorsHandler(h http.Handler) http.Handler {
-	commonS3Headers := []string{
-		"Date",
-		"ETag",
-		"Server",
-		"Connection",
-		"Accept-Ranges",
-		"Content-Range",
-		"Content-Encoding",
-		"Content-Length",
-		"Content-Type",
-		"x-amz-request-id",
-	}
-	c := cors.New(cors.Options{
-		AllowedOrigins:   []string{"*"},
-		AllowedMethods:   defaultAllowableHTTPMethods,
-		AllowedHeaders:   []string{"*"},
-		ExposedHeaders:   commonS3Headers,
-		AllowCredentials: true,
-	})
+	c := cors.AllowAll()
 	return c.Handler(h)
 }
 

cmd/handler-utils.go

@@ -40,7 +40,7 @@ func parseLocationConstraint(r *http.Request) (location string, s3Error APIError
 	// be created at default region.
 	locationConstraint := createBucketLocationConfiguration{}
 	err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength)
-	if err != nil && err != io.EOF {
+	if err != nil && r.ContentLength != 0 {
 		logger.LogIf(context.Background(), err)
 		// Treat all other failures as XML parsing errors.
 		return "", ErrMalformedXML

cmd/handler-utils_test.go

@@ -39,48 +39,50 @@ func TestIsValidLocationContraint(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	// Test with corrupted XML
+	// Corrupted XML
 	malformedReq := &http.Request{
 		Body:          ioutil.NopCloser(bytes.NewBuffer([]byte("<>"))),
 		ContentLength: int64(len("<>")),
 	}
-	if _, err := parseLocationConstraint(malformedReq); err != ErrMalformedXML {
-		t.Fatal("Unexpected error: ", err)
+
+	// Not an XML
+	badRequest := &http.Request{
+		Body:          ioutil.NopCloser(bytes.NewReader([]byte("garbage"))),
+		ContentLength: int64(len("garbage")),
 	}
 
 	// generates the input request with XML bucket configuration set to the request body.
-	createExpectedRequest := func(req *http.Request, location string) (*http.Request, error) {
+	createExpectedRequest := func(req *http.Request, location string) *http.Request {
 		createBucketConfig := createBucketLocationConfiguration{}
 		createBucketConfig.Location = location
-		var createBucketConfigBytes []byte
-		createBucketConfigBytes, e := xml.Marshal(createBucketConfig)
-		if e != nil {
-			return nil, e
-		}
+		createBucketConfigBytes, _ := xml.Marshal(createBucketConfig)
 		createBucketConfigBuffer := bytes.NewBuffer(createBucketConfigBytes)
 		req.Body = ioutil.NopCloser(createBucketConfigBuffer)
 		req.ContentLength = int64(createBucketConfigBuffer.Len())
-		return req, nil
+		return req
 	}
 
 	testCases := []struct {
-		locationForInputRequest string
-		serverConfigRegion      string
-		expectedCode            APIErrorCode
+		request            *http.Request
+		serverConfigRegion string
+		expectedCode       APIErrorCode
 	}{
 		// Test case - 1.
-		{globalMinioDefaultRegion, globalMinioDefaultRegion, ErrNone},
+		{createExpectedRequest(&http.Request{}, "eu-central-1"), globalMinioDefaultRegion, ErrNone},
 		// Test case - 2.
 		// In case of empty request body ErrNone is returned.
-		{"", globalMinioDefaultRegion, ErrNone},
+		{createExpectedRequest(&http.Request{}, ""), globalMinioDefaultRegion, ErrNone},
+		// Test case - 3
+		// In case of garbage request body ErrMalformedXML is returned.
+		{badRequest, globalMinioDefaultRegion, ErrMalformedXML},
+		// Test case - 4
+		// In case of invalid XML request body ErrMalformedXML is returned.
+		{malformedReq, globalMinioDefaultRegion, ErrMalformedXML},
 	}
+
 	for i, testCase := range testCases {
-		inputRequest, e := createExpectedRequest(&http.Request{}, testCase.locationForInputRequest)
-		if e != nil {
-			t.Fatalf("Test %d: Failed to Marshal bucket configuration", i+1)
-		}
 		globalServerConfig.SetRegion(testCase.serverConfigRegion)
-		_, actualCode := parseLocationConstraint(inputRequest)
+		_, actualCode := parseLocationConstraint(testCase.request)
 		if testCase.expectedCode != actualCode {
 			t.Errorf("Test %d: Expected the APIErrCode to be %d, but instead found %d", i+1, testCase.expectedCode, actualCode)
 		}

cmd/http/close.go

@@ -0,0 +1,52 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package http
+
+import (
+	"io"
+	"io/ioutil"
+	"sync"
+)
+
+var b512pool = sync.Pool{
+	New: func() interface{} {
+		buf := make([]byte, 512)
+		return &buf
+	},
+}
+
+// DrainBody close non nil response with any response Body.
+// convenient wrapper to drain any remaining data on response body.
+//
+// Subsequently this allows golang http RoundTripper
+// to re-use the same connection for future requests.
+func DrainBody(respBody io.ReadCloser) {
+	// Callers should close resp.Body when done reading from it.
+	// If resp.Body is not closed, the Client's underlying RoundTripper
+	// (typically Transport) may not be able to re-use a persistent TCP
+	// connection to the server for a subsequent "keep-alive" request.
+	if respBody != nil {
+		// Drain any remaining Body and then close the connection.
+		// Without this closing connection would disallow re-using
+		// the same connection for future uses.
+		//  - http://stackoverflow.com/a/17961593/4465767
+		bufp := b512pool.Get().(*[]byte)
+		defer b512pool.Put(bufp)
+		io.CopyBuffer(ioutil.Discard, respBody, *bufp)
+		respBody.Close()
+	}
+}

cmd/local-locker.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -26,17 +26,18 @@ import (
 
 // lockRequesterInfo stores various info from the client for each lock that is requested.
 type lockRequesterInfo struct {
-	writer          bool      // Bool whether write or read lock.
-	node            string    // Network address of client claiming lock.
-	serviceEndpoint string    // RPC path of client claiming lock.
-	uid             string    // UID to uniquely identify request of client.
-	timestamp       time.Time // Timestamp set at the time of initialization.
-	timeLastCheck   time.Time // Timestamp for last check of validity of lock.
+	Writer          bool      // Bool whether write or read lock.
+	Node            string    // Network address of client claiming lock.
+	ServiceEndpoint string    // RPC path of client claiming lock.
+	UID             string    // UID to uniquely identify request of client.
+	Timestamp       time.Time // Timestamp set at the time of initialization.
+	TimeLastCheck   time.Time // Timestamp for last check of validity of lock.
+	Source          string    // Contains line, function and filename reqesting the lock.
 }
 
 // isWriteLock returns whether the lock is a write or read lock.
 func isWriteLock(lri []lockRequesterInfo) bool {
-	return len(lri) == 1 && lri[0].writer
+	return len(lri) == 1 && lri[0].Writer
 }
 
 // localLocker implements Dsync.NetLocker
@@ -62,12 +63,13 @@ func (l *localLocker) Lock(args dsync.LockArgs) (reply bool, err error) {
 	if !isLockTaken { // No locks held on the given name, so claim write lock
 		l.lockMap[args.Resource] = []lockRequesterInfo{
 			{
-				writer:          true,
-				node:            args.ServerAddr,
-				serviceEndpoint: args.ServiceEndpoint,
-				uid:             args.UID,
-				timestamp:       UTCNow(),
-				timeLastCheck:   UTCNow(),
+				Writer:          true,
+				Node:            args.ServerAddr,
+				ServiceEndpoint: args.ServiceEndpoint,
+				Source:          args.Source,
+				UID:             args.UID,
+				Timestamp:       UTCNow(),
+				TimeLastCheck:   UTCNow(),
 			},
 		}
 	}
@@ -98,12 +100,13 @@ func (l *localLocker) RLock(args dsync.LockArgs) (reply bool, err error) {
 	l.mutex.Lock()
 	defer l.mutex.Unlock()
 	lrInfo := lockRequesterInfo{
-		writer:          false,
-		node:            args.ServerAddr,
-		serviceEndpoint: args.ServiceEndpoint,
-		uid:             args.UID,
-		timestamp:       UTCNow(),
-		timeLastCheck:   UTCNow(),
+		Writer:          false,
+		Node:            args.ServerAddr,
+		ServiceEndpoint: args.ServiceEndpoint,
+		Source:          args.Source,
+		UID:             args.UID,
+		Timestamp:       UTCNow(),
+		TimeLastCheck:   UTCNow(),
 	}
 	if lri, ok := l.lockMap[args.Resource]; ok {
 		if reply = !isWriteLock(lri); reply {
@@ -149,3 +152,18 @@ func (l *localLocker) ForceUnlock(args dsync.LockArgs) (reply bool, err error) {
 	}
 	return true, nil
 }
+
+func (l *localLocker) DupLockMap() map[string][]lockRequesterInfo {
+	l.mutex.Lock()
+	defer l.mutex.Unlock()
+
+	lockCopy := make(map[string][]lockRequesterInfo)
+	for k, v := range l.lockMap {
+		var lockSlice []lockRequesterInfo
+		for _, lockInfo := range v {
+			lockSlice = append(lockSlice, lockInfo)
+		}
+		lockCopy[k] = lockSlice
+	}
+	return lockCopy
+}

cmd/lock-rpc-server-common.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -34,12 +34,12 @@ type nameLockRequesterInfoPair struct {
 func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
 	// Check if entry is still in map (could have been removed altogether by 'concurrent' (R)Unlock of last entry)
 	if lri, ok := l.lockMap[nlrip.name]; ok {
-		if !l.removeEntry(nlrip.name, nlrip.lri.uid, &lri) {
+		if !l.removeEntry(nlrip.name, nlrip.lri.UID, &lri) {
 			// Remove failed, in case it is a:
-			if nlrip.lri.writer {
+			if nlrip.lri.Writer {
 				// Writer: this should never happen as the whole (mapped) entry should have been deleted
 				reqInfo := (&logger.ReqInfo{}).AppendTags("name", nlrip.name)
-				reqInfo.AppendTags("uid", nlrip.lri.uid)
+				reqInfo.AppendTags("uid", nlrip.lri.UID)
 				ctx := logger.SetReqInfo(context.Background(), reqInfo)
 				logger.LogIf(ctx, errors.New("Lock maintenance failed to remove entry for write lock (should never happen)"))
 			} // Reader: this can happen if multiple read locks were active and
@@ -53,7 +53,7 @@ func (l *localLocker) removeEntryIfExists(nlrip nameLockRequesterInfoPair) {
 func (l *localLocker) removeEntry(name, uid string, lri *[]lockRequesterInfo) bool {
 	// Find correct entry to remove based on uid.
 	for index, entry := range *lri {
-		if entry.uid == uid {
+		if entry.UID == uid {
 			if len(*lri) == 1 {
 				// Remove the (last) lock.
 				delete(l.lockMap, name)
@@ -76,9 +76,9 @@ func getLongLivedLocks(m map[string][]lockRequesterInfo, interval time.Duration)
 	for name, lriArray := range m {
 		for idx := range lriArray {
 			// Check whether enough time has gone by since last check
-			if time.Since(lriArray[idx].timeLastCheck) >= interval {
+			if time.Since(lriArray[idx].TimeLastCheck) >= interval {
 				rslt = append(rslt, nameLockRequesterInfoPair{name: name, lri: lriArray[idx]})
-				lriArray[idx].timeLastCheck = UTCNow()
+				lriArray[idx].TimeLastCheck = UTCNow()
 			}
 		}
 	}

cmd/lock-rpc-server-common_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -29,12 +29,12 @@ func TestLockRpcServerRemoveEntryIfExists(t *testing.T) {
 	defer os.RemoveAll(testPath)
 
 	lri := lockRequesterInfo{
-		writer:          false,
-		node:            "host",
-		serviceEndpoint: "rpc-path",
-		uid:             "0123-4567",
-		timestamp:       UTCNow(),
-		timeLastCheck:   UTCNow(),
+		Writer:          false,
+		Node:            "host",
+		ServiceEndpoint: "rpc-path",
+		UID:             "0123-4567",
+		Timestamp:       UTCNow(),
+		TimeLastCheck:   UTCNow(),
 	}
 	nlrip := nameLockRequesterInfoPair{name: "name", lri: lri}
 
@@ -66,20 +66,20 @@ func TestLockRpcServerRemoveEntry(t *testing.T) {
 	defer os.RemoveAll(testPath)
 
 	lockRequesterInfo1 := lockRequesterInfo{
-		writer:          true,
-		node:            "host",
-		serviceEndpoint: "rpc-path",
-		uid:             "0123-4567",
-		timestamp:       UTCNow(),
-		timeLastCheck:   UTCNow(),
+		Writer:          true,
+		Node:            "host",
+		ServiceEndpoint: "rpc-path",
+		UID:             "0123-4567",
+		Timestamp:       UTCNow(),
+		TimeLastCheck:   UTCNow(),
 	}
 	lockRequesterInfo2 := lockRequesterInfo{
-		writer:          true,
-		node:            "host",
-		serviceEndpoint: "rpc-path",
-		uid:             "89ab-cdef",
-		timestamp:       UTCNow(),
-		timeLastCheck:   UTCNow(),
+		Writer:          true,
+		Node:            "host",
+		ServiceEndpoint: "rpc-path",
+		UID:             "89ab-cdef",
+		Timestamp:       UTCNow(),
+		TimeLastCheck:   UTCNow(),
 	}
 
 	locker.ll.lockMap["name"] = []lockRequesterInfo{
@@ -128,12 +128,12 @@ func TestLockRpcServerGetLongLivedLocks(t *testing.T) {
 		{
 			lockMap: map[string][]lockRequesterInfo{
 				"test": {{
-					writer:          true,
-					node:            "10.1.10.21",
-					serviceEndpoint: "/lock/mnt/disk1",
-					uid:             "10000112",
-					timestamp:       ut,
-					timeLastCheck:   ut,
+					Writer:          true,
+					Node:            "10.1.10.21",
+					ServiceEndpoint: "/lock/mnt/disk1",
+					UID:             "10000112",
+					Timestamp:       ut,
+					TimeLastCheck:   ut,
 				}},
 			},
 			lockInterval: 1 * time.Minute,
@@ -143,12 +143,12 @@ func TestLockRpcServerGetLongLivedLocks(t *testing.T) {
 		{
 			lockMap: map[string][]lockRequesterInfo{
 				"test": {{
-					writer:          true,
-					node:            "10.1.10.21",
-					serviceEndpoint: "/lock/mnt/disk1",
-					uid:             "10000112",
-					timestamp:       ut,
-					timeLastCheck:   ut.Add(-2 * time.Minute),
+					Writer:          true,
+					Node:            "10.1.10.21",
+					ServiceEndpoint: "/lock/mnt/disk1",
+					UID:             "10000112",
+					Timestamp:       ut,
+					TimeLastCheck:   ut.Add(-2 * time.Minute),
 				}},
 			},
 			lockInterval: 1 * time.Minute,
@@ -156,12 +156,12 @@ func TestLockRpcServerGetLongLivedLocks(t *testing.T) {
 				{
 					name: "test",
 					lri: lockRequesterInfo{
-						writer:          true,
-						node:            "10.1.10.21",
-						serviceEndpoint: "/lock/mnt/disk1",
-						uid:             "10000112",
-						timestamp:       ut,
-						timeLastCheck:   ut.Add(-2 * time.Minute),
+						Writer:          true,
+						Node:            "10.1.10.21",
+						ServiceEndpoint: "/lock/mnt/disk1",
+						UID:             "10000112",
+						Timestamp:       ut,
+						TimeLastCheck:   ut.Add(-2 * time.Minute),
 					},
 				},
 			},

cmd/lock-rpc-server.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -94,7 +94,7 @@ func (l *lockRPCReceiver) Expired(args *LockArgs, reply *bool) error {
 	if lri, ok := l.ll.lockMap[args.LockArgs.Resource]; ok {
 		// Check whether uid is still active
 		for _, entry := range lri {
-			if entry.uid == args.LockArgs.UID {
+			if entry.UID == args.LockArgs.UID {
 				*reply = false // When uid found, lock is still active so return not expired.
 				return nil     // When uid found *reply is set to true.
 			}
@@ -123,7 +123,7 @@ func (l *lockRPCReceiver) lockMaintenance(interval time.Duration) {
 	// Validate if long lived locks are indeed clean.
 	for _, nlrip := range nlripLongLived {
 		// Initialize client based on the long live locks.
-		host, err := xnet.ParseHost(nlrip.lri.node)
+		host, err := xnet.ParseHost(nlrip.lri.Node)
 		if err != nil {
 			logger.LogIf(context.Background(), err)
 			continue
@@ -136,7 +136,7 @@ func (l *lockRPCReceiver) lockMaintenance(interval time.Duration) {
 
 		// Call back to original server verify whether the lock is still active (based on name & uid)
 		expired, _ := c.Expired(dsync.LockArgs{
-			UID:      nlrip.lri.uid,
+			UID:      nlrip.lri.UID,
 			Resource: nlrip.name,
 		})
 

cmd/lock-rpc-server_test.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -32,10 +32,10 @@ func testLockEquality(lriLeft, lriRight []lockRequesterInfo) bool {
 	}
 
 	for i := 0; i < len(lriLeft); i++ {
-		if lriLeft[i].writer != lriRight[i].writer ||
-			lriLeft[i].node != lriRight[i].node ||
-			lriLeft[i].serviceEndpoint != lriRight[i].serviceEndpoint ||
-			lriLeft[i].uid != lriRight[i].uid {
+		if lriLeft[i].Writer != lriRight[i].Writer ||
+			lriLeft[i].Node != lriRight[i].Node ||
+			lriLeft[i].ServiceEndpoint != lriRight[i].ServiceEndpoint ||
+			lriLeft[i].UID != lriRight[i].UID {
 			return false
 		}
 	}
@@ -97,10 +97,10 @@ func TestLockRpcServerLock(t *testing.T) {
 			gotLri, _ := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
-					writer:          true,
-					node:            "node",
-					serviceEndpoint: "rpc-path",
-					uid:             "0123-4567",
+					Writer:          true,
+					Node:            "node",
+					ServiceEndpoint: "rpc-path",
+					UID:             "0123-4567",
 				},
 			}
 			if !testLockEquality(expectedLri, gotLri) {
@@ -213,10 +213,10 @@ func TestLockRpcServerRLock(t *testing.T) {
 			gotLri, _ := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
-					writer:          false,
-					node:            "node",
-					serviceEndpoint: "rpc-path",
-					uid:             "0123-4567",
+					Writer:          false,
+					Node:            "node",
+					ServiceEndpoint: "rpc-path",
+					UID:             "0123-4567",
 				},
 			}
 			if !testLockEquality(expectedLri, gotLri) {
@@ -315,10 +315,10 @@ func TestLockRpcServerRUnlock(t *testing.T) {
 			gotLri, _ := locker.ll.lockMap["name"]
 			expectedLri := []lockRequesterInfo{
 				{
-					writer:          false,
-					node:            "node",
-					serviceEndpoint: "rpc-path",
-					uid:             "89ab-cdef",
+					Writer:          false,
+					Node:            "node",
+					ServiceEndpoint: "rpc-path",
+					UID:             "89ab-cdef",
 				},
 			}
 			if !testLockEquality(expectedLri, gotLri) {

cmd/logger/target/http/http.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -20,9 +20,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"errors"
-	"io"
-	"io/ioutil"
 	gohttp "net/http"
+
+	xhttp "github.com/minio/minio/cmd/http"
 )
 
 // Target implements logger.Target and sends the json
@@ -56,11 +56,9 @@ func (h *Target) startHTTPLogger() {
 			if err != nil {
 				continue
 			}
-			if resp.Body != nil {
-				buf := make([]byte, 512)
-				io.CopyBuffer(ioutil.Discard, resp.Body, buf)
-				resp.Body.Close()
-			}
+
+			// Drain any response.
+			xhttp.DrainBody(resp.Body)
 		}
 	}()
 }

cmd/logger/utils.go

@@ -39,11 +39,11 @@ var (
 		}
 		return fmt.Sprint
 	}()
-	ColorFgRed = func() func(format string, a ...interface{}) string {
+	ColorFgRed = func() func(a ...interface{}) string {
 		if isTerminal() {
-			return color.New(color.FgRed).SprintfFunc()
+			return color.New(color.FgRed).SprintFunc()
 		}
-		return fmt.Sprintf
+		return fmt.Sprint
 	}()
 	ColorBgRed = func() func(format string, a ...interface{}) string {
 		if isTerminal() {

cmd/namespace-lock.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2016 Minio, Inc.
+ * Minio Cloud Storage, (C) 2016, 2017, 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -53,9 +53,9 @@ type RWLocker interface {
 
 // RWLockerSync - internal locker interface.
 type RWLockerSync interface {
-	GetLock(timeout time.Duration) bool
+	GetLock(id, source string, timeout time.Duration) bool
 	Unlock()
-	GetRLock(timeout time.Duration) bool
+	GetRLock(id, source string, timeout time.Duration) bool
 	RUnlock()
 }
 
@@ -162,9 +162,9 @@ func (n *nsLockMap) lock(volume, path string, lockSource, opsID string, readLock
 
 	// Locking here will block (until timeout).
 	if readLock {
-		locked = nsLk.GetRLock(timeout)
+		locked = nsLk.GetRLock(opsID, lockSource, timeout)
 	} else {
-		locked = nsLk.GetLock(timeout)
+		locked = nsLk.GetLock(opsID, lockSource, timeout)
 	}
 
 	if !locked { // We failed to get the lock

cmd/notification.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -340,6 +340,44 @@ func (sys *NotificationSys) ServerInfo(ctx context.Context) []ServerInfo {
 	return serverInfo
 }
 
+// GetLocks - makes GetLocks RPC call on all peers.
+func (sys *NotificationSys) GetLocks(ctx context.Context) []*PeerLocks {
+	var idx = 0
+	locksResp := make([]*PeerLocks, len(sys.peerRPCClientMap))
+	var wg sync.WaitGroup
+	for addr, client := range sys.peerRPCClientMap {
+		wg.Add(1)
+		go func(idx int, addr xnet.Host, client *PeerRPCClient) {
+			defer wg.Done()
+			// Try to fetch serverInfo remotely in three attempts.
+			for i := 0; i < 3; i++ {
+				serverLocksResp, err := client.GetLocks()
+				if err == nil {
+					locksResp[idx] = &PeerLocks{
+						Addr:  addr.String(),
+						Locks: serverLocksResp,
+					}
+					return
+				}
+
+				// Last iteration log the error.
+				if i == 2 {
+					reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String())
+					ctx := logger.SetReqInfo(ctx, reqInfo)
+					logger.LogOnceIf(ctx, err, addr.String())
+				}
+				// Wait for one second and no need wait after last attempt.
+				if i < 2 {
+					time.Sleep(1 * time.Second)
+				}
+			}
+		}(idx, addr, client)
+		idx++
+	}
+	wg.Wait()
+	return locksResp
+}
+
 // SetBucketPolicy - calls SetBucketPolicy RPC call on all peers.
 func (sys *NotificationSys) SetBucketPolicy(ctx context.Context, bucketName string, bucketPolicy *policy.Policy) {
 	go func() {

cmd/object-api-interface.go

@@ -82,8 +82,8 @@ type ObjectLayer interface {
 	// Healing operations.
 	ReloadFormat(ctx context.Context, dryRun bool) error
 	HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error)
-	HealBucket(ctx context.Context, bucket string, dryRun bool) ([]madmin.HealResultItem, error)
-	HealObject(ctx context.Context, bucket, object string, dryRun bool) (madmin.HealResultItem, error)
+	HealBucket(ctx context.Context, bucket string, dryRun, remove bool) ([]madmin.HealResultItem, error)
+	HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (madmin.HealResultItem, error)
 	ListBucketsHeal(ctx context.Context) (buckets []BucketInfo, err error)
 	ListObjectsHeal(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error)
 

cmd/object-handlers.go

@@ -31,6 +31,8 @@ import (
 	"strconv"
 	"strings"
 
+	"time"
+
 	snappy "github.com/golang/snappy"
 	"github.com/gorilla/mux"
 	miniogo "github.com/minio/minio-go"
@@ -185,6 +187,12 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 		return getObjectNInfo(ctx, bucket, object, rs, r.Header, readLock, ObjectOptions{})
 	}
 
+	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
+	if err != nil {
+		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
+		return
+	}
+
 	if err = s3Select.Open(getObject); err != nil {
 		if serr, ok := err.(s3select.SelectError); ok {
 			w.WriteHeader(serr.HTTPStatusCode())
@@ -198,12 +206,6 @@ func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r
 	s3Select.Evaluate(w)
 	s3Select.Close()
 
-	objInfo, err := getObjectInfo(ctx, bucket, object, opts)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		return
-	}
-
 	// Get host and port from Request.RemoteAddr.
 	host, port, err := net.SplitHostPort(handlers.GetSourceIP(r))
 	if err != nil {
@@ -1673,7 +1675,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 			return
 		}
 		if crypto.IsEncrypted(li.UserDefined) {
-			isEncrypted = true
 			if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(li.UserDefined) {
 				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL, guessIsBrowserReq(r))
 				return
@@ -1682,7 +1683,7 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 				writeErrorResponse(w, ErrSSEMultipartEncrypted, r.URL, guessIsBrowserReq(r))
 				return
 			}
-			isEncrypted = true // to detect SSE-S3 encryption
+			isEncrypted = true
 			var key []byte
 			if crypto.SSEC.IsRequested(r.Header) {
 				key, err = ParseSSECustomerRequest(r)
@@ -1727,9 +1728,6 @@ func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *htt
 		writeErrorResponse(w, toAPIErrorCode(ctx, err), r.URL, guessIsBrowserReq(r))
 		return
 	}
-	if isEncrypted {
-		partInfo.ETag = tryDecryptETag(objectEncryptionKey, partInfo.ETag, crypto.SSEC.IsRequested(r.Header))
-	}
 
 	if isEncrypted {
 		partInfo.ETag = tryDecryptETag(objectEncryptionKey, partInfo.ETag, crypto.SSEC.IsRequested(r.Header))
@@ -2132,6 +2130,36 @@ func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *ht
 	writeSuccessResponseXML(w, encodedSuccessResponse)
 }
 
+// Send white spaces every 10 seconds to the client till completeMultiPartUpload() is done so that the client does not time out.
+// Downside is we might send 200OK and then send error XML. But accoording to S3 spec
+// the client is supposed to check for error XML even if it received 200OK. But for erasure this is not a
+// problem as completeMultiPartUpload() is quick. Even For FS, it would not be an issue
+// as we do background append as and when the parts arrive and completeMultiPartUpload is quick.
+// Only in a rare case where parts would be out of order will FS' completeMultiPartUpload() take a longer time.
+func sendWhiteSpace(ctx context.Context, w http.ResponseWriter) <-chan struct{} {
+	doneCh := make(chan struct{})
+	go func() {
+		ticker := time.NewTicker(time.Second * 10)
+		for {
+			select {
+			case <-ticker.C:
+				// Write a white space char to the client to prevent client timeouts
+				// when the server takes a long time to completeMultiPartUpload()
+				if _, err := w.Write([]byte("\n\r")); err != nil {
+					logger.LogIf(ctx, err)
+					return
+				}
+				w.(http.Flusher).Flush()
+			case doneCh <- struct{}{}:
+				ticker.Stop()
+				return
+			}
+		}
+
+	}()
+	return doneCh
+}
+
 // CompleteMultipartUploadHandler - Complete multipart upload.
 func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := newContext(r, w, "CompleteMultipartUpload")
@@ -2197,8 +2225,6 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 			return
 		}
 		if crypto.IsEncrypted(li.UserDefined) {
-			isEncrypted = true
-			ssec = crypto.SSEC.IsEncrypted(li.UserDefined)
 			var key []byte
 			isEncrypted = true
 			ssec = crypto.SSEC.IsEncrypted(li.UserDefined)
@@ -2261,7 +2287,11 @@ func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWrite
 	if api.CacheAPI() != nil {
 		completeMultiPartUpload = api.CacheAPI().CompleteMultipartUpload
 	}
+	completeDoneCh := sendWhiteSpace(ctx, w)
 	objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, completeParts, opts)
+	// Stop writing white spaces to the client. Note that close(doneCh) style is not used as it
+	// can cause white space to be written after we send XML response in a race condition.
+	<-completeDoneCh
 	if err != nil {
 		switch oErr := err.(type) {
 		case PartTooSmall:

cmd/object-handlers_test.go

@@ -1738,7 +1738,6 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 
 	for i, testCase := range testCases {
 		var req *http.Request
-		var reqV2 *http.Request
 		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
 		rec := httptest.NewRecorder()
 		if !testCase.invalidPartNumber || !testCase.maximumPartNumber {
@@ -1782,37 +1781,6 @@ func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName stri
 				t.Fatalf("Test %d: %s: Expected only one entry returned %d entries", i+1, instanceType, len(results.Parts))
 			}
 		}
-
-		// Verify response of the V2 signed HTTP request.
-		// initialize HTTP NewRecorder, this records any mutations to response writer inside the handler.
-		recV2 := httptest.NewRecorder()
-
-		reqV2, err = newTestRequest("PUT", getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "1"), 0, nil)
-		if err != nil {
-			t.Fatalf("Test %d: Failed to create HTTP request for copy Object: <ERROR> %v", i+1, err)
-		}
-		// "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied.
-		if testCase.copySourceHeader != "" {
-			reqV2.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader)
-		}
-		if testCase.copySourceVersionID != "" {
-			reqV2.Header.Set("X-Amz-Copy-Source-Version-Id", testCase.copySourceVersionID)
-		}
-		if testCase.copySourceRange != "" {
-			reqV2.Header.Set("X-Amz-Copy-Source-Range", testCase.copySourceRange)
-		}
-
-		err = signRequestV2(reqV2, testCase.accessKey, testCase.secretKey)
-		if err != nil {
-			t.Fatalf("Failed to V2 Sign the HTTP request: %v.", err)
-		}
-
-		// Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler.
-		// Call the ServeHTTP to execute the handler.
-		apiRouter.ServeHTTP(recV2, reqV2)
-		if recV2.Code != testCase.expectedRespStatus {
-			t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code)
-		}
 	}
 
 	// HTTP request for testing when `ObjectLayer` is set to `nil`.
@@ -3189,73 +3157,6 @@ func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string
 	ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq)
 }
 
-// TestAPIPutObjectPartHandlerPreSign - Tests validate the response of PutObjectPart HTTP handler
-// when the request signature type is PreSign.
-func TestAPIPutObjectPartHandlerPreSign(t *testing.T) {
-	defer DetectTestLeak(t)()
-	ExecObjectLayerAPITest(t, testAPIPutObjectPartHandlerPreSign, []string{"NewMultipart", "PutObjectPart"})
-}
-
-func testAPIPutObjectPartHandlerPreSign(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler,
-	credentials auth.Credentials, t *testing.T) {
-	testObject := "testobject"
-	rec := httptest.NewRecorder()
-	req, err := newTestSignedRequestV4("POST", getNewMultipartURL("", bucketName, "testobject"),
-		0, nil, credentials.AccessKey, credentials.SecretKey, nil)
-	if err != nil {
-		t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-	apiRouter.ServeHTTP(rec, req)
-
-	// Get uploadID of the mulitpart upload initiated.
-	var mpartResp InitiateMultipartUploadResponse
-	mpartRespBytes, err := ioutil.ReadAll(rec.Result().Body)
-	if err != nil {
-		t.Fatalf("[%s] Failed to read NewMultipartUpload response <ERROR> %v", instanceType, err)
-
-	}
-	err = xml.Unmarshal(mpartRespBytes, &mpartResp)
-	if err != nil {
-		t.Fatalf("[%s] Failed to unmarshal NewMultipartUpload response <ERROR> %v", instanceType, err)
-	}
-
-	rec = httptest.NewRecorder()
-	req, err = newTestRequest("PUT", getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
-		int64(len("hello")), bytes.NewReader([]byte("hello")))
-	if err != nil {
-		t.Fatalf("[%s] - Failed to create an unsigned request to put object part for %s/%s <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-
-	err = preSignV2(req, credentials.AccessKey, credentials.SecretKey, int64(10*60*60))
-	if err != nil {
-		t.Fatalf("[%s] - Failed to presign an unsigned request to put object part for %s/%s <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-	apiRouter.ServeHTTP(rec, req)
-	if rec.Code != http.StatusOK {
-		t.Errorf("Test %d %s expected to succeed but failed with HTTP status code %d", 1, instanceType, rec.Code)
-	}
-
-	rec = httptest.NewRecorder()
-	req, err = newTestRequest("PUT", getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"),
-		int64(len("hello")), bytes.NewReader([]byte("hello")))
-	if err != nil {
-		t.Fatalf("[%s] - Failed to create an unsigned request to put object part for %s/%s <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-	err = preSignV4(req, credentials.AccessKey, credentials.SecretKey, int64(10*60*60))
-	if err != nil {
-		t.Fatalf("[%s] - Failed to presign an unsigned request to put object part for %s/%s <ERROR> %v",
-			instanceType, bucketName, testObject, err)
-	}
-	apiRouter.ServeHTTP(rec, req)
-	if rec.Code != http.StatusOK {
-		t.Errorf("Test %d %s expected to succeed but failed with HTTP status code %d", 1, instanceType, rec.Code)
-	}
-}
-
 // TestAPIPutObjectPartHandlerStreaming - Tests validate the response of PutObjectPart HTTP handler
 // when the request signature type is `streaming signature`.
 func TestAPIPutObjectPartHandlerStreaming(t *testing.T) {

cmd/peer-rpc-client.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -200,6 +200,15 @@ func (rpcClient *PeerRPCClient) ServerInfo() (ServerInfoData, error) {
 	return reply, err
 }
 
+// GetLocksResp stores various info from the client for each lock that is requested.
+type GetLocksResp map[string][]lockRequesterInfo
+
+// GetLocks - returns the lock information on the server to which the RPC call is made.
+func (rpcClient *PeerRPCClient) GetLocks() (resp GetLocksResp, err error) {
+	err = rpcClient.Call(peerServiceName+".GetLocks", &AuthArgs{}, &resp)
+	return resp, err
+}
+
 // NewPeerRPCClient - returns new peer RPC client.
 func NewPeerRPCClient(host *xnet.Host) (*PeerRPCClient, error) {
 	scheme := "http"

cmd/peer-rpc-server.go

@@ -1,5 +1,5 @@
 /*
- * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ * Minio Cloud Storage, (C) 2018, 2019 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -383,6 +383,24 @@ func (receiver *peerRPCReceiver) ServerInfo(args *AuthArgs, reply *ServerInfoDat
 	return nil
 }
 
+// GetLocks - Get Locks receiver.
+func (receiver *peerRPCReceiver) GetLocks(args *AuthArgs, reply *GetLocksResp) error {
+	if globalBootTime.IsZero() {
+		return errServerNotInitialized
+	}
+
+	// Build storage info
+	objLayer := newObjectLayerFn()
+	if objLayer == nil {
+		return errServerNotInitialized
+	}
+
+	// Locks data.
+	*reply = globalLockServer.ll.DupLockMap()
+
+	return nil
+}
+
 // NewPeerRPCServer - returns new peer RPC server.
 func NewPeerRPCServer() (*xrpc.Server, error) {
 	rpcServer := xrpc.NewServer()

cmd/posix.go

@@ -303,9 +303,10 @@ func (s *posix) IsOnline() bool {
 // DiskInfo is an extended type which returns current
 // disk usage per path.
 type DiskInfo struct {
-	Total uint64
-	Free  uint64
-	Used  uint64
+	Total    uint64
+	Free     uint64
+	Used     uint64
+	RootDisk bool
 }
 
 // DiskInfo provides current information about disk space usage,
@@ -319,12 +320,17 @@ func (s *posix) DiskInfo() (info DiskInfo, err error) {
 	if !s.diskMount {
 		used = atomic.LoadUint64(&s.totalUsed)
 	}
-	return DiskInfo{
-		Total: di.Total,
-		Free:  di.Free,
-		Used:  used,
-	}, nil
 
+	rootDisk, err := disk.IsRootDisk(s.diskPath)
+	if err != nil {
+		return info, err
+	}
+	return DiskInfo{
+		Total:    di.Total,
+		Free:     di.Free,
+		Used:     used,
+		RootDisk: rootDisk,
+	}, nil
 }
 
 // getVolDir - will convert incoming volume names to
@@ -1099,8 +1105,9 @@ func (s *posix) WriteAll(volume, path string, buf []byte) (err error) {
 		return errFaultyDisk
 	}
 
-	// Create file if not found
-	w, err := s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_WRONLY)
+	// Create file if not found. Note that it is created with os.O_EXCL flag as the file
+	// always is supposed to be created in the tmp directory with a unique file name.
+	w, err := s.openFile(volume, path, os.O_CREATE|os.O_SYNC|os.O_WRONLY|os.O_EXCL)
 	if err != nil {
 		return err
 	}

cmd/prepare-storage.go

@@ -23,6 +23,7 @@ import (
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
+	"github.com/minio/minio/pkg/sync/errgroup"
 )
 
 var printEndpointError = func() func(Endpoint, error) {
@@ -49,18 +50,26 @@ var printEndpointError = func() func(Endpoint, error) {
 
 // Migrates backend format of local disks.
 func formatXLMigrateLocalEndpoints(endpoints EndpointList) error {
-	for _, endpoint := range endpoints {
+	g := errgroup.WithNErrs(len(endpoints))
+	for index, endpoint := range endpoints {
 		if !endpoint.IsLocal {
 			continue
 		}
-		formatPath := pathJoin(endpoint.Path, minioMetaBucket, formatConfigFile)
-		if _, err := os.Stat(formatPath); err != nil {
-			if os.IsNotExist(err) {
-				continue
+		index := index
+		g.Go(func() error {
+			epPath := endpoints[index].Path
+			formatPath := pathJoin(epPath, minioMetaBucket, formatConfigFile)
+			if _, err := os.Stat(formatPath); err != nil {
+				if os.IsNotExist(err) {
+					return nil
+				}
+				return err
 			}
-			return err
-		}
-		if err := formatXLMigrate(endpoint.Path); err != nil {
+			return formatXLMigrate(epPath)
+		}, index)
+	}
+	for _, err := range g.Wait() {
+		if err != nil {
 			return err
 		}
 	}
@@ -69,21 +78,51 @@ func formatXLMigrateLocalEndpoints(endpoints EndpointList) error {
 
 // Cleans up tmp directory of local disks.
 func formatXLCleanupTmpLocalEndpoints(endpoints EndpointList) error {
-	for _, endpoint := range endpoints {
+	g := errgroup.WithNErrs(len(endpoints))
+	for index, endpoint := range endpoints {
 		if !endpoint.IsLocal {
 			continue
 		}
-		formatPath := pathJoin(endpoint.Path, minioMetaBucket, formatConfigFile)
-		if _, err := os.Stat(formatPath); err != nil {
-			if os.IsNotExist(err) {
-				continue
+		index := index
+		g.Go(func() error {
+			epPath := endpoints[index].Path
+			// If disk is not formatted there is nothing to be cleaned up.
+			formatPath := pathJoin(epPath, minioMetaBucket, formatConfigFile)
+			if _, err := os.Stat(formatPath); err != nil {
+				if os.IsNotExist(err) {
+					return nil
+				}
+				return err
 			}
-			return err
-		}
-		if err := removeAll(pathJoin(endpoint.Path, minioMetaTmpBucket)); err != nil {
-			return err
-		}
-		if err := mkdirAll(pathJoin(endpoint.Path, minioMetaTmpBucket), 0777); err != nil {
+			if _, err := os.Stat(pathJoin(epPath, minioMetaTmpBucket+"-old")); err != nil {
+				if !os.IsNotExist(err) {
+					return err
+				}
+			}
+
+			// Need to move temporary objects left behind from previous run of minio
+			// server to a unique directory under `minioMetaTmpBucket-old` to clean
+			// up `minioMetaTmpBucket` for the current run.
+			//
+			// /disk1/.minio.sys/tmp-old/
+			//  |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62
+			//  |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e
+			//
+			// In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains
+			// temporary objects from one of the previous runs of minio server.
+			if err := renameAll(pathJoin(epPath, minioMetaTmpBucket),
+				pathJoin(epPath, minioMetaTmpBucket+"-old", mustGetUUID())); err != nil {
+				return err
+			}
+
+			// Removal of tmp-old folder is backgrounded completely.
+			go removeAll(pathJoin(epPath, minioMetaTmpBucket+"-old"))
+
+			return mkdirAll(pathJoin(epPath, minioMetaTmpBucket), 0777)
+		}, index)
+	}
+	for _, err := range g.Wait() {
+		if err != nil {
 			return err
 		}
 	}

cmd/rest/client.go

@@ -59,9 +59,9 @@ func (c *Client) Call(method string, values url.Values, body io.Reader, length i
 	}
 
 	if resp.StatusCode != http.StatusOK {
+		defer xhttp.DrainBody(resp.Body)
 		// Limit the ReadAll(), just in case, because of a bug, the server responds with large data.
-		r := io.LimitReader(resp.Body, 1024)
-		b, err := ioutil.ReadAll(r)
+		b, err := ioutil.ReadAll(io.LimitReader(resp.Body, 4096))
 		if err != nil {
 			return nil, err
 		}

cmd/rpc/client.go

@@ -22,8 +22,6 @@ import (
 	"encoding/gob"
 	"errors"
 	"fmt"
-	"io"
-	"io/ioutil"
 	"net"
 	"net/http"
 	"reflect"
@@ -42,28 +40,6 @@ type Client struct {
 	serviceURL *xnet.URL
 }
 
-// closeResponse close non nil response with any response Body.
-// convenient wrapper to drain any remaining data on response body.
-//
-// Subsequently this allows golang http RoundTripper
-// to re-use the same connection for future requests.
-func closeResponse(body io.ReadCloser) {
-	// Callers should close resp.Body when done reading from it.
-	// If resp.Body is not closed, the Client's underlying RoundTripper
-	// (typically Transport) may not be able to re-use a persistent TCP
-	// connection to the server for a subsequent "keep-alive" request.
-	if body != nil {
-		// Drain any remaining Body and then close the connection.
-		// Without this closing connection would disallow re-using
-		// the same connection for future uses.
-		//  - http://stackoverflow.com/a/17961593/4465767
-		bufp := b512pool.Get().(*[]byte)
-		defer b512pool.Put(bufp)
-		io.CopyBuffer(ioutil.Discard, body, *bufp)
-		body.Close()
-	}
-}
-
 // Call - calls service method on RPC server.
 func (client *Client) Call(serviceMethod string, args, reply interface{}) error {
 	replyKind := reflect.TypeOf(reply).Kind()
@@ -93,7 +69,7 @@ func (client *Client) Call(serviceMethod string, args, reply interface{}) error
 	if err != nil {
 		return err
 	}
-	defer closeResponse(response.Body)
+	defer xhttp.DrainBody(response.Body)
 
 	if response.StatusCode != http.StatusOK {
 		return fmt.Errorf("%v rpc call failed with error code %v", serviceMethod, response.StatusCode)

cmd/storage-rest-client.go

@@ -29,6 +29,10 @@ import (
 	"encoding/gob"
 	"encoding/hex"
 
+	"fmt"
+	"strings"
+
+	"github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/cmd/rest"
 	xnet "github.com/minio/minio/pkg/net"
@@ -101,6 +105,15 @@ func toStorageErr(err error) error {
 	case errServerTimeMismatch.Error():
 		return errServerTimeMismatch
 	}
+	if strings.Contains(err.Error(), "Bitrot verification mismatch") {
+		var expected string
+		var received string
+		fmt.Sscanf(err.Error(), "Bitrot verification mismatch - expected %s received %s", &expected, &received)
+		// Go's Sscanf %s scans "," that comes after the expected hash, hence remove it. Providing "," in the format string does not help.
+		expected = strings.TrimSuffix(expected, ",")
+		bitrotErr := hashMismatchError{expected, received}
+		return bitrotErr
+	}
 	return err
 }
 
@@ -152,7 +165,7 @@ func (client *storageRESTClient) DiskInfo() (info DiskInfo, err error) {
 	if err != nil {
 		return
 	}
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	err = gob.NewDecoder(respBody).Decode(&info)
 	return info, err
 }
@@ -162,7 +175,7 @@ func (client *storageRESTClient) MakeVol(volume string) (err error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	respBody, err := client.call(storageRESTMethodMakeVol, values, nil, -1)
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	return err
 }
 
@@ -172,7 +185,7 @@ func (client *storageRESTClient) ListVols() (volinfo []VolInfo, err error) {
 	if err != nil {
 		return
 	}
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	err = gob.NewDecoder(respBody).Decode(&volinfo)
 	return volinfo, err
 }
@@ -185,7 +198,7 @@ func (client *storageRESTClient) StatVol(volume string) (volInfo VolInfo, err er
 	if err != nil {
 		return
 	}
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	err = gob.NewDecoder(respBody).Decode(&volInfo)
 	return volInfo, err
 }
@@ -195,7 +208,7 @@ func (client *storageRESTClient) DeleteVol(volume string) (err error) {
 	values := make(url.Values)
 	values.Set(storageRESTVolume, volume)
 	respBody, err := client.call(storageRESTMethodDeleteVol, values, nil, -1)
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	return err
 }
 
@@ -206,7 +219,7 @@ func (client *storageRESTClient) AppendFile(volume, path string, buffer []byte)
 	values.Set(storageRESTFilePath, path)
 	reader := bytes.NewBuffer(buffer)
 	respBody, err := client.call(storageRESTMethodAppendFile, values, reader, -1)
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	return err
 }
 
@@ -216,7 +229,7 @@ func (client *storageRESTClient) CreateFile(volume, path string, length int64, r
 	values.Set(storageRESTFilePath, path)
 	values.Set(storageRESTLength, strconv.Itoa(int(length)))
 	respBody, err := client.call(storageRESTMethodCreateFile, values, r, length)
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	return err
 }
 
@@ -227,7 +240,7 @@ func (client *storageRESTClient) WriteAll(volume, path string, buffer []byte) er
 	values.Set(storageRESTFilePath, path)
 	reader := bytes.NewBuffer(buffer)
 	respBody, err := client.call(storageRESTMethodWriteAll, values, reader, -1)
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	return err
 }
 
@@ -240,7 +253,7 @@ func (client *storageRESTClient) StatFile(volume, path string) (info FileInfo, e
 	if err != nil {
 		return info, err
 	}
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	err = gob.NewDecoder(respBody).Decode(&info)
 	return info, err
 }
@@ -254,7 +267,7 @@ func (client *storageRESTClient) ReadAll(volume, path string) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	return ioutil.ReadAll(respBody)
 }
 
@@ -290,7 +303,7 @@ func (client *storageRESTClient) ReadFile(volume, path string, offset int64, buf
 	if err != nil {
 		return 0, err
 	}
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	n, err := io.ReadFull(respBody, buffer)
 	return int64(n), err
 }
@@ -305,7 +318,7 @@ func (client *storageRESTClient) ListDir(volume, dirPath string, count int) (ent
 	if err != nil {
 		return nil, err
 	}
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	err = gob.NewDecoder(respBody).Decode(&entries)
 	return entries, err
 }
@@ -316,7 +329,7 @@ func (client *storageRESTClient) DeleteFile(volume, path string) error {
 	values.Set(storageRESTVolume, volume)
 	values.Set(storageRESTFilePath, path)
 	respBody, err := client.call(storageRESTMethodDeleteFile, values, nil, -1)
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	return err
 }
 
@@ -328,7 +341,7 @@ func (client *storageRESTClient) RenameFile(srcVolume, srcPath, dstVolume, dstPa
 	values.Set(storageRESTDstVolume, dstVolume)
 	values.Set(storageRESTDstPath, dstPath)
 	respBody, err := client.call(storageRESTMethodRenameFile, values, nil, -1)
-	defer CloseResponse(respBody)
+	defer http.DrainBody(respBody)
 	return err
 }
 

cmd/sts-handlers.go

@@ -17,6 +17,7 @@
 package cmd
 
 import (
+	"fmt"
 	"net/http"
 
 	"github.com/gorilla/mux"
@@ -28,6 +29,10 @@ import (
 const (
 	// STS API version.
 	stsAPIVersion = "2011-06-15"
+
+	// STS API action constants
+	clientGrants = "AssumeRoleWithClientGrants"
+	webIdentity  = "AssumeRoleWithWebIdentity"
 )
 
 // stsAPIHandlers implements and provides http handlers for AWS STS API.
@@ -41,20 +46,147 @@ func registerSTSRouter(router *mux.Router) {
 	// STS Router
 	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()
 
+	// Assume roles with JWT handler, handles both ClientGrants and WebIdentity.
+	stsRouter.Methods("POST").HeadersRegexp("Content-Type", "application/x-www-form-urlencoded*").
+		HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))
+
 	// AssumeRoleWithClientGrants
 	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)).
-		Queries("Action", "AssumeRoleWithClientGrants").
+		Queries("Action", clientGrants).
 		Queries("Version", stsAPIVersion).
 		Queries("Token", "{Token:.*}")
 
 	// AssumeRoleWithWebIdentity
 	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithWebIdentity)).
-		Queries("Action", "AssumeRoleWithWebIdentity").
+		Queries("Action", webIdentity).
 		Queries("Version", stsAPIVersion).
 		Queries("WebIdentityToken", "{Token:.*}")
 
 }
 
+func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Request) {
+	ctx := newContext(r, w, "AssumeRoleInternalFunction")
+
+	// Parse the incoming form data.
+	if err := r.ParseForm(); err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		return
+	}
+
+	if r.Form.Get("Version") != stsAPIVersion {
+		logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
+		writeSTSErrorResponse(w, ErrSTSMissingParameter)
+		return
+	}
+
+	action := r.Form.Get("Action")
+	switch action {
+	case clientGrants, webIdentity:
+	default:
+		logger.LogIf(ctx, fmt.Errorf("Unsupported action %s", action))
+		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		return
+	}
+
+	ctx = newContext(r, w, action)
+	defer logger.AuditLog(w, r, action, nil)
+
+	if globalIAMValidators == nil {
+		writeSTSErrorResponse(w, ErrSTSNotInitialized)
+		return
+	}
+
+	v, err := globalIAMValidators.Get("jwt")
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		return
+	}
+
+	token := r.Form.Get("Token")
+	if token == "" {
+		token = r.Form.Get("WebIdentityToken")
+	}
+
+	m, err := v.Validate(token, r.Form.Get("DurationSeconds"))
+	if err != nil {
+		switch err {
+		case validator.ErrTokenExpired:
+			switch action {
+			case clientGrants:
+				writeSTSErrorResponse(w, ErrSTSClientGrantsExpiredToken)
+			case webIdentity:
+				writeSTSErrorResponse(w, ErrSTSWebIdentityExpiredToken)
+			}
+			return
+		case validator.ErrInvalidDuration:
+			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+			return
+		}
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
+		return
+	}
+
+	secret := globalServerConfig.GetCredential().SecretKey
+	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
+	if err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInternalError)
+		return
+	}
+
+	// JWT has requested a custom claim with policy value set.
+	// This is a Minio STS API specific value, this value should
+	// be set and configured on your identity provider as part of
+	// JWT custom claims.
+	var policyName string
+	if v, ok := m["policy"]; ok {
+		policyName, _ = v.(string)
+	}
+
+	var subFromToken string
+	if v, ok := m["sub"]; ok {
+		subFromToken, _ = v.(string)
+	}
+
+	// Set the newly generated credentials.
+	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
+		logger.LogIf(ctx, err)
+		writeSTSErrorResponse(w, ErrSTSInternalError)
+		return
+	}
+
+	// Notify all other Minio peers to reload temp users
+	for _, nerr := range globalNotificationSys.LoadUsers() {
+		if nerr.Err != nil {
+			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
+			logger.LogIf(ctx, nerr.Err)
+		}
+	}
+
+	var encodedSuccessResponse []byte
+	switch action {
+	case clientGrants:
+		encodedSuccessResponse = encodeResponse(&AssumeRoleWithClientGrantsResponse{
+			Result: ClientGrantsResult{
+				Credentials:      cred,
+				SubjectFromToken: subFromToken,
+			},
+		})
+	case webIdentity:
+		encodedSuccessResponse = encodeResponse(&AssumeRoleWithWebIdentityResponse{
+			Result: WebIdentityResult{
+				Credentials:                 cred,
+				SubjectFromWebIdentityToken: subFromToken,
+			},
+		})
+	}
+
+	writeSuccessResponseXML(w, encodedSuccessResponse)
+}
+
 // AssumeRoleWithWebIdentity - implementation of AWS STS API supporting OAuth2.0
 // users from web identity provider such as Facebook, Google, or any OpenID
 // Connect-compatible identity provider.
@@ -62,82 +194,7 @@ func registerSTSRouter(router *mux.Router) {
 // Eg:-
 //    $ curl https://minio:9000/?Action=AssumeRoleWithWebIdentity&WebIdentityToken=<jwt>
 func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AssumeRoleWithWebIdentity")
-
-	defer logger.AuditLog(w, r, "AssumeRoleWithWebIdentity", nil)
-
-	if globalIAMValidators == nil {
-		writeSTSErrorResponse(w, ErrSTSNotInitialized)
-		return
-	}
-
-	// NOTE: this API only accepts JWT tokens.
-	v, err := globalIAMValidators.Get("jwt")
-	if err != nil {
-		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
-		return
-	}
-
-	vars := mux.Vars(r)
-	m, err := v.Validate(vars["Token"], r.URL.Query().Get("DurationSeconds"))
-	if err != nil {
-		switch err {
-		case validator.ErrTokenExpired:
-			writeSTSErrorResponse(w, ErrSTSWebIdentityExpiredToken)
-		case validator.ErrInvalidDuration:
-			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
-		default:
-			logger.LogIf(ctx, err)
-			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
-		}
-		return
-	}
-
-	secret := globalServerConfig.GetCredential().SecretKey
-	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeSTSErrorResponse(w, ErrSTSInternalError)
-		return
-	}
-
-	// JWT has requested a custom claim with policy value set.
-	// This is a Minio STS API specific value, this value should
-	// be set and configured on your identity provider as part of
-	// JWT custom claims.
-	var policyName string
-	if v, ok := m["policy"]; ok {
-		policyName, _ = v.(string)
-	}
-
-	var subFromToken string
-	if v, ok := m["sub"]; ok {
-		subFromToken, _ = v.(string)
-	}
-
-	// Set the newly generated credentials.
-	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
-		logger.LogIf(ctx, err)
-		writeSTSErrorResponse(w, ErrSTSInternalError)
-		return
-	}
-
-	// Notify all other Minio peers to reload temp users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-
-	encodedSuccessResponse := encodeResponse(&AssumeRoleWithWebIdentityResponse{
-		Result: WebIdentityResult{
-			Credentials:                 cred,
-			SubjectFromWebIdentityToken: subFromToken,
-		},
-	})
-
-	writeSuccessResponseXML(w, encodedSuccessResponse)
+	sts.AssumeRoleWithJWT(w, r)
 }
 
 // AssumeRoleWithClientGrants - implementation of AWS STS extension API supporting
@@ -146,80 +203,5 @@ func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *h
 // Eg:-
 //    $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token=<jwt>
 func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
-	ctx := newContext(r, w, "AssumeRoleWithClientGrants")
-
-	defer logger.AuditLog(w, r, "AssumeRoleWithClientGrants", nil)
-
-	if globalIAMValidators == nil {
-		writeSTSErrorResponse(w, ErrSTSNotInitialized)
-		return
-	}
-
-	// NOTE: this API only accepts JWT tokens.
-	v, err := globalIAMValidators.Get("jwt")
-	if err != nil {
-		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
-		return
-	}
-
-	vars := mux.Vars(r)
-	m, err := v.Validate(vars["Token"], r.URL.Query().Get("DurationSeconds"))
-	if err != nil {
-		switch err {
-		case validator.ErrTokenExpired:
-			writeSTSErrorResponse(w, ErrSTSClientGrantsExpiredToken)
-		case validator.ErrInvalidDuration:
-			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
-		default:
-			logger.LogIf(ctx, err)
-			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
-		}
-		return
-	}
-
-	secret := globalServerConfig.GetCredential().SecretKey
-	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
-	if err != nil {
-		logger.LogIf(ctx, err)
-		writeSTSErrorResponse(w, ErrSTSInternalError)
-		return
-	}
-
-	// JWT has requested a custom claim with policy value set.
-	// This is a Minio STS API specific value, this value should
-	// be set and configured on your identity provider as part of
-	// JWT custom claims.
-	var policyName string
-	if v, ok := m["policy"]; ok {
-		policyName, _ = v.(string)
-	}
-
-	var subFromToken string
-	if v, ok := m["sub"]; ok {
-		subFromToken, _ = v.(string)
-	}
-
-	// Set the newly generated credentials.
-	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
-		logger.LogIf(ctx, err)
-		writeSTSErrorResponse(w, ErrSTSInternalError)
-		return
-	}
-
-	// Notify all other Minio peers to reload temp users
-	for _, nerr := range globalNotificationSys.LoadUsers() {
-		if nerr.Err != nil {
-			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
-			logger.LogIf(ctx, nerr.Err)
-		}
-	}
-
-	encodedSuccessResponse := encodeResponse(&AssumeRoleWithClientGrantsResponse{
-		Result: ClientGrantsResult{
-			Credentials:      cred,
-			SubjectFromToken: subFromToken,
-		},
-	})
-
-	writeSuccessResponseXML(w, encodedSuccessResponse)
+	sts.AssumeRoleWithJWT(w, r)
 }

cmd/update-main.go

@@ -32,6 +32,7 @@ import (
 
 	"github.com/inconshreveable/go-update"
 	"github.com/minio/cli"
+	xhttp "github.com/minio/minio/cmd/http"
 	"github.com/minio/minio/cmd/logger"
 	_ "github.com/minio/sha256-simd" // Needed for sha256 hash verifier.
 	"github.com/segmentio/go-prompt"
@@ -318,7 +319,7 @@ func downloadReleaseURL(releaseChecksumURL string, timeout time.Duration, mode s
 	if resp == nil {
 		return content, fmt.Errorf("No response from server to download URL %s", releaseChecksumURL)
 	}
-	defer CloseResponse(resp.Body)
+	defer xhttp.DrainBody(resp.Body)
 
 	if resp.StatusCode != http.StatusOK {
 		return content, fmt.Errorf("Error downloading URL %s. Response: %v", releaseChecksumURL, resp.Status)
@@ -465,7 +466,7 @@ func doUpdate(sha256Hex string, latestReleaseTime time.Time, ok bool) (updateSta
 	if err != nil {
 		return updateStatusMsg, err
 	}
-	defer CloseResponse(resp.Body)
+	defer xhttp.DrainBody(resp.Body)
 
 	// FIXME: add support for gpg verification as well.
 	if err = update.Apply(resp.Body,

cmd/utils.go

@@ -34,7 +34,6 @@ import (
 	"path/filepath"
 	"reflect"
 	"strings"
-	"sync"
 	"time"
 
 	"github.com/minio/minio/cmd/logger"
@@ -467,35 +466,6 @@ func isNetworkOrHostDown(err error) bool {
 	return false
 }
 
-var b512pool = sync.Pool{
-	New: func() interface{} {
-		buf := make([]byte, 512)
-		return &buf
-	},
-}
-
-// CloseResponse close non nil response with any response Body.
-// convenient wrapper to drain any remaining data on response body.
-//
-// Subsequently this allows golang http RoundTripper
-// to re-use the same connection for future requests.
-func CloseResponse(respBody io.ReadCloser) {
-	// Callers should close resp.Body when done reading from it.
-	// If resp.Body is not closed, the Client's underlying RoundTripper
-	// (typically Transport) may not be able to re-use a persistent TCP
-	// connection to the server for a subsequent "keep-alive" request.
-	if respBody != nil {
-		// Drain any remaining Body and then close the connection.
-		// Without this closing connection would disallow re-using
-		// the same connection for future uses.
-		//  - http://stackoverflow.com/a/17961593/4465767
-		bufp := b512pool.Get().(*[]byte)
-		defer b512pool.Put(bufp)
-		io.CopyBuffer(ioutil.Discard, respBody, *bufp)
-		respBody.Close()
-	}
-}
-
 // Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example)
 // If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"]
 func restQueries(keys ...string) []string {

cmd/web-handlers.go

@@ -471,7 +471,7 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 			AccountName:     claims.Subject,
 			Action:          iampolicy.ListBucketAction,
 			BucketName:      args.BucketName,
-			ConditionValues: getConditionValues(r, "", ""),
+			ConditionValues: getConditionValues(r, "", claims.Subject),
 			IsOwner:         owner,
 		})
 
@@ -479,7 +479,7 @@ func (web *webAPIHandlers) ListObjects(r *http.Request, args *ListObjectsArgs, r
 			AccountName:     claims.Subject,
 			Action:          iampolicy.PutObjectAction,
 			BucketName:      args.BucketName,
-			ConditionValues: getConditionValues(r, "", ""),
+			ConditionValues: getConditionValues(r, "", claims.Subject),
 			IsOwner:         owner,
 			ObjectName:      args.Prefix + "/",
 		})

cmd/xl-sets.go

@@ -1007,6 +1007,65 @@ func (s *xlSets) ReloadFormat(ctx context.Context, dryRun bool) (err error) {
 	return nil
 }
 
+// If it is a single node XL and all disks are root disks, it is most likely a test setup, else it is a production setup.
+// On a test setup we allow creation of format.json on root disks to help with dev/testing.
+func isTestSetup(infos []DiskInfo, errs []error) bool {
+	if globalIsDistXL {
+		return false
+	}
+	rootDiskCount := 0
+	for i := range errs {
+		if errs[i] != nil {
+			// On error it is safer to assume that this is not a test setup.
+			return false
+		}
+		if infos[i].RootDisk {
+			rootDiskCount++
+		}
+	}
+	// It is a test setup if all disks are root disks.
+	return rootDiskCount == len(infos)
+}
+
+func getAllDiskInfos(storageDisks []StorageAPI) ([]DiskInfo, []error) {
+	infos := make([]DiskInfo, len(storageDisks))
+	errs := make([]error, len(storageDisks))
+	var wg sync.WaitGroup
+	for i := range storageDisks {
+		if storageDisks[i] == nil {
+			errs[i] = errDiskNotFound
+			continue
+		}
+		wg.Add(1)
+		go func(i int) {
+			defer wg.Done()
+			infos[i], errs[i] = storageDisks[i].DiskInfo()
+		}(i)
+	}
+	wg.Wait()
+	return infos, errs
+}
+
+// Mark root disks as down so as not to heal them.
+func markRootDisksAsDown(storageDisks []StorageAPI) {
+	infos, errs := getAllDiskInfos(storageDisks)
+	if isTestSetup(infos, errs) {
+		// Allow healing of disks for test setups to help with testing.
+		return
+	}
+	for i := range storageDisks {
+		if errs[i] != nil {
+			storageDisks[i] = nil
+			continue
+		}
+		if infos[i].RootDisk {
+			// We should not heal on root disk. i.e in a situation where the minio-administrator has unmounted a
+			// defective drive we should not heal a path on the root disk.
+			storageDisks[i] = nil
+		}
+	}
+}
+
 // HealFormat - heals missing `format.json` on fresh unformatted disks.
 // TODO: In future support corrupted disks missing format.json but has erasure
 // coded data in it.
@@ -1029,6 +1088,8 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 		}
 	}(storageDisks)
 
+	markRootDisksAsDown(storageDisks)
+
 	formats, sErrs := loadFormatXLAll(storageDisks)
 	if err = checkFormatXLValues(formats); err != nil {
 		return madmin.HealResultItem{}, err
@@ -1170,7 +1231,7 @@ func (s *xlSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealRe
 }
 
 // HealBucket - heals inconsistent buckets and bucket metadata on all sets.
-func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun bool) (results []madmin.HealResultItem, err error) {
+func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (results []madmin.HealResultItem, err error) {
 	bucketLock := globalNSMutex.NewNSLock(bucket, "")
 	if err := bucketLock.GetLock(globalHealingTimeout); err != nil {
 		return nil, err
@@ -1187,7 +1248,7 @@ func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun bool) (re
 
 	for _, s := range s.sets {
 		var setResults []madmin.HealResultItem
-		setResults, _ = s.HealBucket(ctx, bucket, dryRun)
+		setResults, _ = s.HealBucket(ctx, bucket, dryRun, remove)
 		for _, setResult := range setResults {
 			if setResult.Type == madmin.HealItemBucket {
 				for _, v := range setResult.Before.Drives {
@@ -1255,8 +1316,8 @@ func (s *xlSets) HealBucket(ctx context.Context, bucket string, dryRun bool) (re
 }
 
 // HealObject - heals inconsistent object on a hashedSet based on object name.
-func (s *xlSets) HealObject(ctx context.Context, bucket, object string, dryRun bool) (madmin.HealResultItem, error) {
-	return s.getHashedSet(object).HealObject(ctx, bucket, object, dryRun)
+func (s *xlSets) HealObject(ctx context.Context, bucket, object string, dryRun, remove bool) (madmin.HealResultItem, error) {
+	return s.getHashedSet(object).HealObject(ctx, bucket, object, dryRun, remove)
 }
 
 // Lists all buckets which need healing.

cmd/xl-v1-bucket.go

@@ -218,6 +218,30 @@ func (xl xlObjects) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 	return bucketInfos, nil
 }
 
+// Dangling buckets should be handled appropriately, in this following situation
+// we actually have quorum error to be `nil` but we have some disks where
+// the bucket delete returned `errVolumeNotEmpty` but this is not correct
+// can only happen if there are dangling objects in a bucket. Under such
+// a situation we simply attempt a full delete of the bucket including
+// the dangling objects. All of this happens under a lock and there
+// is no way a user can create buckets and sneak in objects into namespace,
+// so it is safer to do.
+func deleteDanglingBucket(ctx context.Context, storageDisks []StorageAPI, dErrs []error, bucket string) {
+	for index, err := range dErrs {
+		if err == errVolumeNotEmpty {
+			// Attempt to delete bucket again.
+			if derr := storageDisks[index].DeleteVol(bucket); derr == errVolumeNotEmpty {
+				_ = cleanupDir(ctx, storageDisks[index], bucket, "")
+
+				_ = storageDisks[index].DeleteVol(bucket)
+
+				// Cleanup all the previously incomplete multiparts.
+				_ = cleanupDir(ctx, storageDisks[index], minioMetaMultipartBucket, bucket)
+			}
+		}
+	}
+}
+
 // DeleteBucket - deletes a bucket.
 func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 	bucketLock := xl.nsMutex.NewNSLock(bucket, "")
@@ -231,7 +255,8 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 	var dErrs = make([]error, len(xl.getDisks()))
 
 	// Remove a volume entry on all underlying storage disks.
-	for index, disk := range xl.getDisks() {
+	storageDisks := xl.getDisks()
+	for index, disk := range storageDisks {
 		if disk == nil {
 			dErrs[index] = errDiskNotFound
 			continue
@@ -242,18 +267,14 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 			defer wg.Done()
 			// Attempt to delete bucket.
 			err := disk.DeleteVol(bucket)
-
 			if err != nil {
 				dErrs[index] = err
 				return
 			}
+
 			// Cleanup all the previously incomplete multiparts.
 			err = cleanupDir(ctx, disk, minioMetaMultipartBucket, bucket)
-
-			if err != nil {
-				if err == errVolumeNotFound {
-					return
-				}
+			if err != nil && err != errVolumeNotFound {
 				dErrs[index] = err
 			}
 		}(index, disk)
@@ -271,6 +292,11 @@ func (xl xlObjects) DeleteBucket(ctx context.Context, bucket string) error {
 		return toObjectErr(err, bucket)
 	}
 
+	// If we reduce quorum to nil, means we have deleted buckets properly
+	// on some servers in quorum, we should look for volumeNotEmpty errors
+	// and delete those buckets as well.
+	deleteDanglingBucket(ctx, storageDisks, dErrs, bucket)
+
 	return nil
 }
 

cmd/xl-v1-common.go

@@ -19,6 +19,7 @@ package cmd
 import (
 	"context"
 	"path"
+	"sync"
 
 	"github.com/minio/minio/cmd/logger"
 )
@@ -75,26 +76,46 @@ func (xl xlObjects) isObject(bucket, prefix string) (ok bool) {
 
 // isObjectDir returns if the specified path represents an empty directory.
 func (xl xlObjects) isObjectDir(bucket, prefix string) (ok bool) {
-	for _, disk := range xl.getLoadBalancedDisks() {
+	var objectDirs = make([]bool, len(xl.getDisks()))
+	var errs = make([]error, len(xl.getDisks()))
+	var wg sync.WaitGroup
+	for index, disk := range xl.getDisks() {
 		if disk == nil {
 			continue
 		}
-		// Check if 'prefix' is an object on this 'disk', else continue the check the next disk
-		ctnts, err := disk.ListDir(bucket, prefix, 1)
-		if err == nil {
-			if len(ctnts) == 0 {
-				return true
+		wg.Add(1)
+		go func(index int, disk StorageAPI) {
+			defer wg.Done()
+			// Check if 'prefix' is an object on this 'disk', else continue the check the next disk
+			entries, err := disk.ListDir(bucket, prefix, 1)
+			if err != nil {
+				errs[index] = err
+				return
 			}
-			return false
-		}
-		// Ignore for file not found,  disk not found or faulty disk.
-		if IsErrIgnored(err, xlTreeWalkIgnoredErrs...) {
-			continue
-		}
+			objectDirs[index] = len(entries) == 0
+		}(index, disk)
+	}
+
+	wg.Wait()
+
+	readQuorum := len(xl.getDisks()) / 2
+
+	err := reduceReadQuorumErrs(context.Background(), errs, xlTreeWalkIgnoredErrs, readQuorum)
+	if err != nil {
 		reqInfo := &logger.ReqInfo{BucketName: bucket}
 		reqInfo.AppendTags("prefix", prefix)
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
 		logger.LogIf(ctx, err)
-	} // Exhausted all disks - return false.
-	return false
+		return false
+	}
+
+	for _, ok := range objectDirs {
+		if !ok {
+			// We perhaps found some entries in a prefix.
+			return false
+		}
+	}
+
+	// Return true if and only if the all directories are empty.
+	return true
 }

cmd/xl-v1-healing.go

@@ -22,6 +22,7 @@ import (
 	"io"
 	"path"
 	"sync"
+	"time"
 
 	"github.com/minio/minio/cmd/logger"
 	"github.com/minio/minio/pkg/madmin"
@@ -40,7 +41,7 @@ func (xl xlObjects) HealFormat(ctx context.Context, dryRun bool) (madmin.HealRes
 // Heals a bucket if it doesn't exist on one of the disks, additionally
 // also heals the missing entries for bucket metadata files
 // `policy.json, notification.xml, listeners.json`.
-func (xl xlObjects) HealBucket(ctx context.Context, bucket string, dryRun bool) (
+func (xl xlObjects) HealBucket(ctx context.Context, bucket string, dryRun, remove bool) (
 	results []madmin.HealResultItem, err error) {
 
 	storageDisks := xl.getDisks()
@@ -57,7 +58,7 @@ func (xl xlObjects) HealBucket(ctx context.Context, bucket string, dryRun bool)
 	results = append(results, result)
 
 	// Proceed to heal bucket metadata.
-	metaResults, err := healBucketMetadata(xl, bucket, dryRun)
+	metaResults, err := healBucketMetadata(xl, bucket, dryRun, remove)
 	results = append(results, metaResults...)
 	return results, err
 }
@@ -133,30 +134,19 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, bucket string, w
 		DiskCount: len(storageDisks),
 	}
 	for i, before := range beforeState {
-		if storageDisks[i] == nil {
+		if storageDisks[i] != nil {
+			drive := storageDisks[i].String()
 			res.Before.Drives = append(res.Before.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: "",
+				Endpoint: drive,
 				State:    before,
 			})
 			res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
 				UUID:     "",
-				Endpoint: "",
+				Endpoint: drive,
 				State:    afterState[i],
 			})
-			continue
 		}
-		drive := storageDisks[i].String()
-		res.Before.Drives = append(res.Before.Drives, madmin.HealDriveInfo{
-			UUID:     "",
-			Endpoint: drive,
-			State:    before,
-		})
-		res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{
-			UUID:     "",
-			Endpoint: drive,
-			State:    afterState[i],
-		})
 	}
 
 	reducedErr := reduceWriteQuorumErrs(ctx, dErrs, bucketOpIgnoredErrs, writeQuorum)
@@ -169,13 +159,13 @@ func healBucket(ctx context.Context, storageDisks []StorageAPI, bucket string, w
 
 // Heals all the metadata associated for a given bucket, this function
 // heals `policy.json`, `notification.xml` and `listeners.json`.
-func healBucketMetadata(xl xlObjects, bucket string, dryRun bool) (
+func healBucketMetadata(xl xlObjects, bucket string, dryRun, remove bool) (
 	results []madmin.HealResultItem, err error) {
 
 	healBucketMetaFn := func(metaPath string) error {
 		reqInfo := &logger.ReqInfo{BucketName: bucket}
 		ctx := logger.SetReqInfo(context.Background(), reqInfo)
-		result, healErr := xl.HealObject(ctx, minioMetaBucket, metaPath, dryRun)
+		result, healErr := xl.HealObject(ctx, minioMetaBucket, metaPath, dryRun, remove)
 		// If object is not found, skip the file.
 		if isErrObjectNotFound(healErr) {
 			return nil
@@ -248,6 +238,30 @@ func listAllBuckets(storageDisks []StorageAPI) (buckets map[string]VolInfo,
 	return buckets, bucketsOcc, nil
 }
 
+// Only heal on disks where we are sure that healing is needed. We can expand
+// this list as and when we figure out more errors can be added to this list safely.
+func shouldHealObjectOnDisk(xlErr, dataErr error, meta xlMetaV1, quorumModTime time.Time) bool {
+	switch xlErr {
+	case errFileNotFound:
+		return true
+	case errCorruptedFormat:
+		return true
+	}
+	if xlErr == nil {
+		// If xl.json was read fine but there is some problem with the part.N files.
+		if dataErr == errFileNotFound {
+			return true
+		}
+		if _, ok := dataErr.(hashMismatchError); ok {
+			return true
+		}
+		if quorumModTime != meta.Stat.ModTime {
+			return true
+		}
+	}
+	return false
+}
+
 // Heals an object by re-writing corrupt/missing erasure blocks.
 func healObject(ctx context.Context, storageDisks []StorageAPI, bucket string, object string,
 	quorum int, dryRun bool) (result madmin.HealResultItem, err error) {
@@ -316,17 +330,13 @@ func healObject(ctx context.Context, storageDisks []StorageAPI, bucket string, o
 			driveState = madmin.DriveStateCorrupt
 		}
 
-		// an online disk without valid data/metadata is
-		// outdated and can be healed.
-		if errs[i] != errDiskNotFound && v == nil {
+		var drive string
+		if storageDisks[i] != nil {
+			drive = storageDisks[i].String()
+		}
+		if shouldHealObjectOnDisk(errs[i], dataErrs[i], partsMetadata[i], modTime) {
 			outDatedDisks[i] = storageDisks[i]
 			disksToHealCount++
-		}
-		var drive string
-		if v == nil {
-			if errs[i] != errDiskNotFound {
-				drive = outDatedDisks[i].String()
-			}
 			result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 				UUID:     "",
 				Endpoint: drive,
@@ -339,7 +349,6 @@ func healObject(ctx context.Context, storageDisks []StorageAPI, bucket string, o
 			})
 			continue
 		}
-		drive = v.String()
 		result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{
 			UUID:     "",
 			Endpoint: drive,
@@ -526,44 +535,55 @@ func (xl xlObjects) healObjectDir(ctx context.Context, bucket, object string, dr
 		ObjectSize:   0,
 	}
 
+	hr.Before.Drives = make([]madmin.HealDriveInfo, len(storageDisks))
+	hr.After.Drives = make([]madmin.HealDriveInfo, len(storageDisks))
+
+	var wg sync.WaitGroup
+
 	// Prepare object creation in all disks
-	for _, disk := range storageDisks {
-		if disk == nil {
-			hr.Before.Drives = append(hr.Before.Drives, madmin.HealDriveInfo{
-				UUID:  "",
-				State: madmin.DriveStateOffline,
-			})
-			hr.After.Drives = append(hr.After.Drives, madmin.HealDriveInfo{
-				UUID:  "",
-				State: madmin.DriveStateMissing,
-			})
-			continue
-		}
-
-		drive := disk.String()
-		hr.Before.Drives = append(hr.Before.Drives, madmin.HealDriveInfo{
-			UUID:     "",
-			Endpoint: drive,
-			State:    madmin.DriveStateMissing,
-		})
-		hr.After.Drives = append(hr.After.Drives, madmin.HealDriveInfo{
-			UUID:     "",
-			Endpoint: drive,
-			State:    madmin.DriveStateMissing,
-		})
-
-		if !dryRun {
-			if err := disk.MakeVol(pathJoin(bucket, object)); err != nil && err != errVolumeExists {
-				return hr, toObjectErr(err, bucket, object)
+	for i, d := range storageDisks {
+		wg.Add(1)
+		go func(idx int, disk StorageAPI) {
+			defer wg.Done()
+			if disk == nil {
+				hr.Before.Drives[idx] = madmin.HealDriveInfo{State: madmin.DriveStateOffline}
+				hr.After.Drives[idx] = madmin.HealDriveInfo{State: madmin.DriveStateOffline}
+				return
 			}
 
-			for i, v := range hr.Before.Drives {
-				if v.Endpoint == drive {
-					hr.After.Drives[i].State = madmin.DriveStateOk
-				}
+			drive := disk.String()
+			hr.Before.Drives[idx] = madmin.HealDriveInfo{UUID: "", Endpoint: drive, State: madmin.DriveStateOffline}
+			hr.After.Drives[idx] = madmin.HealDriveInfo{UUID: "", Endpoint: drive, State: madmin.DriveStateOffline}
+
+			_, statErr := disk.StatVol(pathJoin(bucket, object))
+			switch statErr {
+			case nil:
+				hr.Before.Drives[idx].State = madmin.DriveStateOk
+				hr.After.Drives[idx].State = madmin.DriveStateOk
+				// Object is fine in this disk, nothing to be done anymore, exiting
+				return
+			case errVolumeNotFound:
+				hr.Before.Drives[idx].State = madmin.DriveStateMissing
+				hr.After.Drives[idx].State = madmin.DriveStateMissing
+			default:
+				logger.LogIf(ctx, err)
+				return
 			}
-		}
+
+			if dryRun {
+				return
+			}
+
+			if err := disk.MakeVol(pathJoin(bucket, object)); err == nil || err == errVolumeExists {
+				hr.After.Drives[idx].State = madmin.DriveStateOk
+			} else {
+				logger.LogIf(ctx, err)
+				hr.After.Drives[idx].State = madmin.DriveStateOffline
+			}
+		}(i, d)
 	}
+
+	wg.Wait()
 	return hr, nil
 }
 
@@ -619,12 +639,45 @@ func defaultHealResult(storageDisks []StorageAPI, errs []error, bucket, object s
 	return result
 }
 
+// Object is considered dangling/corrupted if any only
+// if total disks - a combination of corrupted and missing
+// files is lesser than number of data blocks.
+func (xl xlObjects) isObjectDangling(metaArr []xlMetaV1, errs []error) (validMeta xlMetaV1, ok bool) {
+	// We can consider an object data not reliable
+	// when xl.json is not found in read quorum disks.
+	// or when xl.json is not readable in read quorum disks.
+	var notFoundXLJSON, corruptedXLJSON int
+	for _, readErr := range errs {
+		if readErr == errFileNotFound {
+			notFoundXLJSON++
+		} else if readErr == errCorruptedFormat {
+			corruptedXLJSON++
+		}
+	}
+
+	for _, m := range metaArr {
+		if !m.IsValid() {
+			continue
+		}
+		validMeta = m
+		break
+	}
+
+	// We couldn't find any valid meta we are indeed corrupted, return true right away.
+	if validMeta.Erasure.DataBlocks == 0 {
+		return validMeta, true
+	}
+
+	// We have valid meta, now verify if we have enough files with data blocks.
+	return validMeta, (len(xl.getDisks()) - corruptedXLJSON - notFoundXLJSON) < validMeta.Erasure.DataBlocks
+}
+
 // HealObject - heal the given object.
 //
 // FIXME: If an object object was deleted and one disk was down,
 // and later the disk comes back up again, heal on the object
 // should delete it.
-func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool) (hr madmin.HealResultItem, err error) {
+func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRun bool, remove bool) (hr madmin.HealResultItem, err error) {
 	// Create context that also contains information about the object and bucket.
 	// The top level handler might not have this information.
 	reqInfo := logger.GetReqInfo(ctx)
@@ -647,6 +700,19 @@ func (xl xlObjects) HealObject(ctx context.Context, bucket, object string, dryRu
 	// Read metadata files from all the disks
 	partsMetadata, errs := readAllXLMetadata(healCtx, storageDisks, bucket, object)
 
+	// Check if the object is dangling, if yes and user requested
+	// remove we simply delete it from namespace.
+	if m, ok := xl.isObjectDangling(partsMetadata, errs); ok {
+		writeQuorum := m.Erasure.DataBlocks + 1
+		if m.Erasure.DataBlocks == 0 {
+			writeQuorum = len(xl.getDisks())/2 + 1
+		}
+		if !dryRun && remove {
+			err = xl.deleteObject(healCtx, bucket, object, writeQuorum, false)
+		}
+		return defaultHealResult(storageDisks, errs, bucket, object), err
+	}
+
 	latestXLMeta, err := getLatestXLMeta(healCtx, partsMetadata, errs)
 	if err != nil {
 		return defaultHealResult(storageDisks, errs, bucket, object), toObjectErr(err, bucket, object)

cmd/xl-v1-healing_test.go

@@ -114,7 +114,7 @@ func TestHealObjectXL(t *testing.T) {
 		t.Fatalf("Failed to delete a file - %v", err)
 	}
 
-	_, err = obj.HealObject(context.Background(), bucket, object, false)
+	_, err = obj.HealObject(context.Background(), bucket, object, false, false)
 	if err != nil {
 		t.Fatalf("Failed to heal object - %v", err)
 	}
@@ -130,7 +130,7 @@ func TestHealObjectXL(t *testing.T) {
 	}
 
 	// Try healing now, expect to receive errDiskNotFound.
-	_, err = obj.HealObject(context.Background(), bucket, object, false)
+	_, err = obj.HealObject(context.Background(), bucket, object, false, false)
 	// since majority of xl.jsons are not available, object quorum can't be read properly and error will be errXLReadQuorum
 	if _, ok := err.(InsufficientReadQuorum); !ok {
 		t.Errorf("Expected %v but received %v", InsufficientReadQuorum{}, err)

cmd/xl-v1-multipart.go

@@ -505,11 +505,33 @@ func (xl xlObjects) ListObjectParts(ctx context.Context, bucket, object, uploadI
 
 	uploadIDPath := xl.getUploadIDDir(bucket, object, uploadID)
 
-	xlParts, xlMeta, err := xl.readXLMetaParts(ctx, minioMetaMultipartBucket, uploadIDPath)
+	storageDisks := xl.getDisks()
+
+	// Read metadata associated with the object from all disks.
+	partsMetadata, errs := readAllXLMetadata(ctx, storageDisks, minioMetaMultipartBucket, uploadIDPath)
+
+	// get Quorum for this object
+	_, writeQuorum, err := objectQuorumFromMeta(ctx, xl, partsMetadata, errs)
 	if err != nil {
 		return result, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
 	}
 
+	reducedErr := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum)
+	if reducedErr == errXLWriteQuorum {
+		return result, toObjectErr(err, minioMetaMultipartBucket, uploadIDPath)
+	}
+
+	_, modTime := listOnlineDisks(storageDisks, partsMetadata, errs)
+
+	// Pick one from the first valid metadata.
+	xlValidMeta, err := pickValidXLMeta(ctx, partsMetadata, modTime, writeQuorum)
+	if err != nil {
+		return result, err
+	}
+
+	var xlMeta = xlValidMeta.Meta
+	var xlParts = xlValidMeta.Parts
+
 	// Populate the result stub.
 	result.Bucket = bucket
 	result.Object = object

cmd/xl-v1-object.go

@@ -23,7 +23,6 @@ import (
 	"net/http"
 	"path"
 	"strconv"
-	"strings"
 	"sync"
 
 	"github.com/minio/minio/cmd/logger"
@@ -430,100 +429,6 @@ func (xl xlObjects) GetObjectInfo(ctx context.Context, bucket, object string, op
 	return info, nil
 }
 
-func (xl xlObjects) isObjectCorrupted(metaArr []xlMetaV1, errs []error) (validMeta xlMetaV1, ok bool) {
-	// We can consider an object data not reliable
-	// when xl.json is not found in read quorum disks.
-	var notFoundXLJSON, corruptedXLJSON int
-	for _, readErr := range errs {
-		if readErr == errFileNotFound {
-			notFoundXLJSON++
-		}
-	}
-	for _, readErr := range errs {
-		if readErr == errCorruptedFormat {
-			corruptedXLJSON++
-		}
-	}
-
-	for _, m := range metaArr {
-		if !m.IsValid() {
-			continue
-		}
-		validMeta = m
-		break
-	}
-
-	// Return if the object is indeed corrupted.
-	return validMeta, len(xl.getDisks())-notFoundXLJSON < validMeta.Erasure.DataBlocks || len(xl.getDisks()) == corruptedXLJSON
-}
-
-const xlCorruptedSuffix = ".CORRUPTED"
-
-// Renames the corrupted object and makes it visible.
-func (xl xlObjects) renameCorruptedObject(ctx context.Context, bucket, object string, validMeta xlMetaV1, disks []StorageAPI, errs []error) {
-	// if errs returned are corrupted
-	if validMeta.Erasure.DataBlocks == 0 {
-		validMeta = newXLMetaV1(object, len(disks)/2, len(disks)/2)
-	}
-	writeQuorum := validMeta.Erasure.DataBlocks + 1
-
-	// Move all existing objects into corrupted suffix.
-	oldObj := mustGetUUID()
-
-	rename(ctx, disks, bucket, object, minioMetaTmpBucket, oldObj, true, writeQuorum, []error{errFileNotFound})
-
-	// Delete temporary object in the event of failure.
-	// If PutObject succeeded there would be no temporary
-	// object to delete.
-	defer xl.deleteObject(ctx, minioMetaTmpBucket, oldObj, writeQuorum, false)
-
-	tempObj := mustGetUUID()
-
-	// Get all the disks which do not have the file.
-	var cdisks = make([]StorageAPI, len(disks))
-	for i, merr := range errs {
-		if merr == errFileNotFound || merr == errCorruptedFormat {
-			cdisks[i] = disks[i]
-		}
-	}
-
-	for _, disk := range cdisks {
-		if disk == nil {
-			continue
-		}
-
-		// Write empty part file on missing disks.
-		disk.AppendFile(minioMetaTmpBucket, pathJoin(tempObj, "part.1"), []byte{})
-
-		// Write algorithm hash for empty part file.
-		var algorithm = DefaultBitrotAlgorithm
-		h := algorithm.New()
-		h.Write([]byte{})
-
-		// Update the checksums and part info.
-		validMeta.Erasure.Checksums = []ChecksumInfo{
-			{
-				Name:      "part.1",
-				Algorithm: algorithm,
-				Hash:      h.Sum(nil),
-			},
-		}
-
-		validMeta.Parts = []ObjectPartInfo{
-			{
-				Number: 1,
-				Name:   "part.1",
-			},
-		}
-
-		// Write the `xl.json` with the newly calculated metadata.
-		writeXLMetadata(ctx, disk, minioMetaTmpBucket, tempObj, validMeta)
-	}
-
-	// Finally rename all the parts into their respective locations.
-	rename(ctx, cdisks, minioMetaTmpBucket, tempObj, bucket, object+xlCorruptedSuffix, true, writeQuorum, []error{errFileNotFound})
-}
-
 // getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo.
 func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (objInfo ObjectInfo, err error) {
 	disks := xl.getDisks()
@@ -531,26 +436,9 @@ func (xl xlObjects) getObjectInfo(ctx context.Context, bucket, object string) (o
 	// Read metadata associated with the object from all disks.
 	metaArr, errs := readAllXLMetadata(ctx, disks, bucket, object)
 
-	var readQuorum int
-	// Having read quorum means we have xl.json in at least N/2 disks.
-	if !strings.HasSuffix(object, xlCorruptedSuffix) {
-		if validMeta, ok := xl.isObjectCorrupted(metaArr, errs); ok {
-			xl.renameCorruptedObject(ctx, bucket, object, validMeta, disks, errs)
-			// Return err file not found since we renamed now the corrupted object
-			return objInfo, errFileNotFound
-		}
-
-		// Not a corrupted object, attempt to get readquorum properly.
-		readQuorum, _, err = objectQuorumFromMeta(ctx, xl, metaArr, errs)
-		if err != nil {
-			return objInfo, err
-		}
-
-	} else {
-
-		// If this is a corrupted object, change read quorum to N/2 disks
-		// so it will be visible to users, so they can delete it.
-		readQuorum = len(xl.getDisks()) / 2
+	readQuorum, _, err := objectQuorumFromMeta(ctx, xl, metaArr, errs)
+	if err != nil {
+		return objInfo, err
 	}
 
 	// List all the file commit ids from parts metadata.

cmd/xl-v1-object_test.go

@@ -308,7 +308,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = xl.HealObject(context.Background(), bucket, object, false)
+	_, err = xl.HealObject(context.Background(), bucket, object, false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -323,6 +323,11 @@ func TestHealing(t *testing.T) {
 		t.Fatal("HealObject failed")
 	}
 
+	err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "xl.json"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
 	// Write xl.json with different modtime to simulate the case where a disk had
 	// gone down when an object was replaced by a new object.
 	xlMetaOutDated := xlMetaPreHeal
@@ -332,7 +337,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	_, err = xl.HealObject(context.Background(), bucket, object, false)
+	_, err = xl.HealObject(context.Background(), bucket, object, false, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -354,7 +359,7 @@ func TestHealing(t *testing.T) {
 		t.Fatal(err)
 	}
 	// This would create the bucket.
-	_, err = xl.HealBucket(context.Background(), bucket, false)
+	_, err = xl.HealBucket(context.Background(), bucket, false, false)
 	if err != nil {
 		t.Fatal(err)
 	}

docs/bucket/notifications/README.md

@@ -160,7 +160,6 @@ The Minio server configuration file is stored on the backend in json format. The
 | `broker` | _string_ | (Required) MQTT server endpoint, e.g. `tcp://localhost:1883` |
 | `topic` | _string_ | (Required) Name of the MQTT topic to publish on, e.g. `minio` |
 | `qos` | _int_ | Set the Quality of Service Level |
-| `clientId` | _string_ | Unique ID for the MQTT broker to identify Minio |
 | `username` | _string_ | Username to connect to the MQTT server (if required) |
 | `password` | _string_ | Password to connect to the MQTT server (if required) |
 | `queueDir` | _string_ | Persistent store for events when MQTT broker is offline |
@@ -174,7 +173,6 @@ An example configuration for MQTT is shown below:
         "broker": "tcp://localhost:1883",
         "topic": "minio",
         "qos": 1,
-        "clientId": "minio",
         "username": "",
         "password": "",
         "queueDir": ""
@@ -222,12 +220,14 @@ import paho.mqtt.client as mqtt
 
 def on_connect(client, userdata, flags, rc):
   print("Connected with result code "+str(rc))
-  client.subscribe("minio")
+  # qos level is set to 1
+  client.subscribe("minio", 1)
 
 def on_message(client, userdata, msg):
     print(msg.payload)
 
-client = mqtt.Client()
+# client_id is a randomly generated unique ID for the mqtt broker to identify the connection.
+client = mqtt.Client(client_id="myclientid",clean_session=False)
 
 client.on_connect = on_connect
 client.on_message = on_message

docs/config/config.sample.json

@@ -82,7 +82,6 @@
 				"broker": "",
 				"topic": "",
 				"qos": 0,
-				"clientId": "",
 				"username": "",
 				"password": "",
 				"reconnectInterval": 0,

docs/kms/README.md

@@ -71,7 +71,7 @@ vault auth enable approle    # enable approle style auth
 vault secrets enable transit  # enable transit secrets engine
 vault write -f  transit/keys/my-minio-key  #define a encryption key-ring for the transit path
 vault policy write minio-policy ./vaultpolicy.hcl  #define a policy for AppRole to access transit path
-vault write auth/approle/role/my-role token_num_uses=0  secret_id_num_uses=0  period=60s # period indicates it is renewable if token is renewed before the period is over
+vault write auth/approle/role/my-role token_num_uses=0  secret_id_num_uses=0  period=5m # period indicates it is renewable if token is renewed before the period is over
 # define an AppRole
 vault write auth/approle/role/my-role policies=minio-policy # apply policy to role
 vault read auth/approle/role/my-role/role-id  # get Approle ID

docs/orchestration/docker-compose/docker-compose.yaml

@@ -5,7 +5,7 @@ version: '2'
 # 9001 through 9004.
 services:
  minio1:
-  image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+  image: minio/minio:RELEASE.2019-01-31T00-31-19Z
   volumes:
    - data1:/data
   ports:
@@ -15,7 +15,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio2:
-  image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+  image: minio/minio:RELEASE.2019-01-31T00-31-19Z
   volumes:
    - data2:/data
   ports:
@@ -25,7 +25,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio3:
-  image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+  image: minio/minio:RELEASE.2019-01-31T00-31-19Z
   volumes:
    - data3:/data
   ports:
@@ -35,7 +35,7 @@ services:
    MINIO_SECRET_KEY: minio123
   command: server http://minio1/data http://minio2/data http://minio3/data http://minio4/data 
  minio4:
-  image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+  image: minio/minio:RELEASE.2019-01-31T00-31-19Z
   volumes:
    - data4:/data
   ports:

docs/orchestration/docker-swarm/docker-compose-secrets.yaml

@@ -2,7 +2,7 @@ version: '3.1'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+    image: minio/minio:RELEASE.2019-01-31T00-31-19Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
       - access_key
 
   minio2:
-    image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+    image: minio/minio:RELEASE.2019-01-31T00-31-19Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
       - access_key
 
   minio3:
-    image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+    image: minio/minio:RELEASE.2019-01-31T00-31-19Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
       - access_key
 
   minio4:
-    image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+    image: minio/minio:RELEASE.2019-01-31T00-31-19Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/docker-swarm/docker-compose.yaml

@@ -2,7 +2,7 @@ version: '3'
 
 services:
   minio1:
-    image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+    image: minio/minio:RELEASE.2019-01-31T00-31-19Z
     hostname: minio1
     volumes:
       - minio1-data:/export
@@ -24,7 +24,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio2:
-    image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+    image: minio/minio:RELEASE.2019-01-31T00-31-19Z
     hostname: minio2
     volumes:
       - minio2-data:/export
@@ -46,7 +46,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio3:
-    image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+    image: minio/minio:RELEASE.2019-01-31T00-31-19Z
     hostname: minio3
     volumes:
       - minio3-data:/export
@@ -68,7 +68,7 @@ services:
     command: server http://minio1/export http://minio2/export http://minio3/export http://minio4/export
 
   minio4:
-    image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+    image: minio/minio:RELEASE.2019-01-31T00-31-19Z
     hostname: minio4
     volumes:
       - minio4-data:/export

docs/orchestration/kubernetes/minio-distributed-daemonset.yaml

@@ -30,7 +30,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+        image: minio/minio:RELEASE.2019-01-31T00-31-19Z
         # Unfortunately you must manually define each server. Perhaps autodiscovery via DNS can be implemented in the future.
         args:
         - server

docs/orchestration/kubernetes/minio-distributed-statefulset.yaml

@@ -21,7 +21,7 @@ spec:
           value: "minio"
         - name: MINIO_SECRET_KEY
           value: "minio123"
-        image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+        image: minio/minio:RELEASE.2019-01-31T00-31-19Z
         args:
         - server
         - http://minio-0.minio.default.svc.cluster.local/data

docs/orchestration/kubernetes/minio-gcs-gateway-deployment.yaml

@@ -21,7 +21,7 @@ spec:
       containers:
       - name: minio
         # Pulls the default Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+        image: minio/minio:RELEASE.2019-01-31T00-31-19Z
         args:
         - gateway
         - gcs

docs/orchestration/kubernetes/minio-standalone-deployment.yaml

@@ -29,7 +29,7 @@ spec:
         - name: data 
           mountPath: "/data"
         # Pulls the lastest Minio image from Docker Hub
-        image: minio/minio:RELEASE.2019-01-16T21-44-08Z
+        image: minio/minio:RELEASE.2019-01-31T00-31-19Z
         args:
         - server
         - /data

docs/select/README.md

@@ -1,11 +1,18 @@
 # Select API Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io)
 Traditional retrieval of objects is always as whole entities, i.e GetObject for a 5 GiB object, will always return 5 GiB of data. S3 Select API allows us to retrieve a subset of data by using simple SQL expressions. By using Select API to retrieve only the data needed by the application, drastic performance improvements can be achieved.
 
-> This implementation is compatible with AWS S3 Select API
+You can use the Select API to query objects with following features:
+
+- CSV, JSON and Parquet - Objects must be in CSV, JSON, or Parquet format.
+- UTF-8 is the only encoding type the Select API supports.
+- GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP or BZIP2. The Select API supports columnar compression for Parquet using GZIP, Snappy, LZO, LZ4. Whole object compression is not supported for Parquet objects.
+- Server-side encryption - The Select API supports querying objects that are protected with server-side encryption.
+
+Type inference and automatic conversion of values is performed based on the context when the value is un-typed (such as when reading CSV data). If present, the CAST function overrides automatic conversion.
 
 ## 1. Prerequisites
 - Install Minio Server from [here](http://docs.minio.io/docs/minio-quickstart-guide).
-- Familiarity with AWS S3 API
+- Familiarity with AWS S3 API.
 - Familiarity with Python and installing dependencies.
 
 ## 2. Install boto3
@@ -53,7 +60,7 @@ for event in r['Payload']:
 ```
 
 ## 4. Run the Program
-Upload first a sample dataset downloaded from [TotalPopulation.csv](https://esa.un.org/unpd/wpp/DVD/Files/1_Indicators%20(Standard)/CSV_FILES/WPP2017_TotalPopulationBySex.csv) using the following commands.
+Upload a sample dataset to Minio using the following commands.
 ```sh
 $ curl "https://esa.un.org/unpd/wpp/DVD/Files/1_Indicators%20(Standard)/CSV_FILES/WPP2017_TotalPopulationBySex.csv" > TotalPopulation.csv
 $ mc mb myminio/mycsvbucket
@@ -81,9 +88,21 @@ Stats details bytesProcessed:
 25786743
 ```
 
+For a more detailed SELECT SQL reference, please see [here](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-select.html)
+
 ## 5. Explore Further
 - [Use `mc` with Minio Server](https://docs.minio.io/docs/minio-client-quickstart-guide)
+- [Use `mc sql` with Minio Server](https://docs.minio.io/docs/minio-client-complete-guide.html#sql)
 - [Use `minio-go` SDK with Minio Server](https://docs.minio.io/docs/golang-client-quickstart-guide)
 - [Use `aws-cli` with Minio Server](https://docs.minio.io/docs/aws-cli-with-minio)
 - [Use `s3cmd` with Minio Server](https://docs.minio.io/docs/s3cmd-with-minio)
 - [The Minio documentation website](https://docs.minio.io)
+
+## 6. Implementation Status
+- Full AWS S3 [SELECT SQL](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-select.html) syntax is supported.
+- All [operators](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-operators.html) are supported.
+- All aggregation, conditional, type-conversion and string functions are supported.
+- JSON path expressions such as `FROM S3Object[*].path` are not yet evaluated.
+- Large numbers (outside of the signed 64-bit range) are not yet supported.
+- The Date [functions](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-date.html) `DATE_ADD`, `DATE_DIFF`, `EXTRACT` and `UTCNOW` along with type conversion using `CAST` to the `TIMESTAMP` data type are currently supported.
+- AWS S3's [reserved keywords](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-keyword-list.html) list is not yet respected.

pkg/cpu/cpu.go

@@ -17,11 +17,44 @@
 package cpu
 
 import (
-	"fmt"
 	"sync"
 	"time"
 )
 
+// rollingAvg holds the rolling average of the cpu load on the minio
+// server over its lifetime
+var rollingAvg *Load
+
+// cpuMeasureInterval is the interval of time between two
+// measurements of CPU load
+const cpuLoadMeasureInterval = 5 * time.Second
+
+// triggers the average load computation at server spawn
+func init() {
+	rollingAvg = &Load{
+		Min: float64(0),
+		Max: float64(0),
+		Avg: float64(0),
+	}
+	var rollingSum float64
+	var cycles float64
+	go func() {
+		for {
+			time.Sleep(cpuLoadMeasureInterval)
+			cycles = cycles + 1
+			currLoad := GetLoad()
+			if rollingAvg.Max < currLoad.Max || rollingAvg.Max == 0 {
+				rollingAvg.Max = currLoad.Max
+			}
+			if rollingAvg.Min > currLoad.Min || rollingAvg.Min == 0 {
+				rollingAvg.Min = currLoad.Min
+			}
+			rollingSum = rollingSum + currLoad.Avg
+			rollingAvg.Avg = rollingSum / cycles
+		}
+	}()
+}
+
 const (
 	// cpuLoadWindow is the interval of time for which the
 	// cpu utilization is measured
@@ -37,15 +70,34 @@ const (
 
 // Load holds CPU utilization % measured in three intervals of 200ms each
 type Load struct {
-	Avg   string `json:"avg"`
-	Max   string `json:"max"`
-	Min   string `json:"min"`
-	Error string `json:"error,omitempty"`
+	Avg   float64 `json:"avg"`
+	Max   float64 `json:"max"`
+	Min   float64 `json:"min"`
+	Error string  `json:"error,omitempty"`
 }
 
 type counter struct{}
 
-// GetLoad returns the CPU utilization % of the current process
+// GetHistoricLoad returns the historic CPU utilization of the current process
+func GetHistoricLoad() Load {
+	return *rollingAvg
+}
+
+// GetLoad returns the CPU utilization of the current process
+// This function works by calcualating the amount of cpu clock
+// cycles the current process used in a given time window
+//
+// This corresponds to the CPU utilization calculation done by
+// tools like top. Here, we use the getclocktime with the
+// CLOCK_PROCESS_CPUTIME_ID parameter to obtain the total number of
+// clock ticks used by the process so far. Then we sleep for
+// 200ms and obtain the the total number of clock ticks again. The
+// difference between the two counts provides us the number of
+// clock ticks used by the process in the 200ms interval.
+//
+// The ratio of clock ticks used (measured in nanoseconds) to number
+// of nanoseconds in 200 milliseconds provides us the CPU usage
+// for the process currently
 func GetLoad() Load {
 	vals := make(chan time.Duration, 3)
 	wg := sync.WaitGroup{}
@@ -83,9 +135,9 @@ func GetLoad() Load {
 	close(vals)
 	avg := sum / 3
 	return Load{
-		Avg:   fmt.Sprintf("%.2f%%", toFixed4(float64(avg)/float64(200*time.Millisecond))*100),
-		Max:   fmt.Sprintf("%.2f%%", toFixed4(float64(max)/float64(200*time.Millisecond))*100),
-		Min:   fmt.Sprintf("%.2f%%", toFixed4(float64(min)/float64(200*time.Millisecond))*100),
+		Avg:   toFixed4(float64(avg)/float64(200*time.Millisecond)) * 100,
+		Max:   toFixed4(float64(max)/float64(200*time.Millisecond)) * 100,
+		Min:   toFixed4(float64(min)/float64(200*time.Millisecond)) * 100,
 		Error: "",
 	}
 }

pkg/disk/root-disk-unix.go

@@ -0,0 +1,31 @@
+// +build !windows
+
+package disk
+
+import (
+	"os"
+	"syscall"
+)
+
+// IsRootDisk returns if diskPath belongs to root-disk, i.e the disk mounted at "/"
+func IsRootDisk(diskPath string) (bool, error) {
+	rootDisk := false
+	diskInfo, err := os.Stat(diskPath)
+	if err != nil {
+		return false, err
+	}
+	rootInfo, err := os.Stat("/")
+	if err != nil {
+		return false, err
+	}
+	diskStat, diskStatOK := diskInfo.Sys().(*syscall.Stat_t)
+	rootStat, rootStatOK := rootInfo.Sys().(*syscall.Stat_t)
+	if diskStatOK && rootStatOK {
+		if diskStat.Dev == rootStat.Dev {
+			// Indicate if the disk path is on root disk. This is used to indicate the healing
+			// process not to format the drive and end up healing it.
+			rootDisk = true
+		}
+	}
+	return rootDisk, nil
+}

pkg/disk/root-disk-windows.go

@@ -0,0 +1,9 @@
+// +build windows
+
+package disk
+
+// IsRootDisk returns if diskPath belongs to root-disk, i.e the disk mounted at "/"
+func IsRootDisk(diskPath string) (bool, error) {
+	// On windows a disk can never be mounted on a subpath.
+	return false, nil
+}

pkg/event/target/mqtt.go

@@ -36,7 +36,6 @@ type MQTTArgs struct {
 	Broker               xnet.URL       `json:"broker"`
 	Topic                string         `json:"topic"`
 	QoS                  byte           `json:"qos"`
-	ClientID             string         `json:"clientId"`
 	User                 string         `json:"username"`
 	Password             string         `json:"password"`
 	MaxReconnectInterval time.Duration  `json:"reconnectInterval"`
@@ -59,9 +58,15 @@ func (m MQTTArgs) Validate() error {
 	default:
 		return errors.New("unknown protocol in broker address")
 	}
-	if m.QueueDir != "" && !filepath.IsAbs(m.QueueDir) {
-		return errors.New("queueDir path should be absolute")
+	if m.QueueDir != "" {
+		if !filepath.IsAbs(m.QueueDir) {
+			return errors.New("queueDir path should be absolute")
+		}
+		if m.QoS == 0 {
+			return errors.New("qos should be set to 1 or 2 if queueDir is set")
+		}
 	}
+
 	return nil
 }
 
@@ -120,7 +125,7 @@ func (target *MQTTTarget) Close() error {
 // NewMQTTTarget - creates new MQTT target.
 func NewMQTTTarget(id string, args MQTTArgs) (*MQTTTarget, error) {
 	options := mqtt.NewClientOptions().
-		SetClientID(args.ClientID).
+		SetClientID("").
 		SetCleanSession(true).
 		SetUsername(args.User).
 		SetPassword(args.Password).

pkg/madmin/API.md

@@ -36,12 +36,12 @@ func main() {
 
 ```
 
-| Service operations         | Info operations  | Healing operations                    | Config operations        | IAM operations | Misc                                |
-|:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:------------------------------------|:------------------------------------|
-| [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`AddUser`](#AddUser) | [`SetAdminCredentials`](#SetAdminCredentials) |
-| [`ServiceSendAction`](#ServiceSendAction) | [`ServerDrivesPerfInfo`](#ServerDrivesPerfInfo) | [`ServerCPULoadInfo`](#ServerCPULoadInfo) | [`ServerMemUsageInfo`](#ServerMemUsageInfo)  | [`SetConfig`](#SetConfig) | [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
-| | |            | [`GetConfigKeys`](#GetConfigKeys) | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) |
-| | |            | [`SetConfigKeys`](#SetConfigKeys) | [`AddCannedPolicy`](#AddCannedPolicy) | |
+| Service operations         | Info operations  | Healing operations                    | Config operations        | Top operations        | IAM operations | Misc                                |
+|:----------------------------|:----------------------------|:--------------------------------------|:--------------------------|:--------------------------|:------------------------------------|:------------------------------------|
+| [`ServiceStatus`](#ServiceStatus) | [`ServerInfo`](#ServerInfo) | [`Heal`](#Heal) | [`GetConfig`](#GetConfig) | [`TopLocks`](#TopLocks) | [`AddUser`](#AddUser) | [`SetAdminCredentials`](#SetAdminCredentials) |
+| [`ServiceSendAction`](#ServiceSendAction) | [`ServerCPULoadInfo`](#ServerCPULoadInfo) | | [`SetConfig`](#SetConfig) | |  [`SetUserPolicy`](#SetUserPolicy) | [`StartProfiling`](#StartProfiling) |
+| |[`ServerMemUsageInfo`](#ServerMemUsageInfo) |            | [`GetConfigKeys`](#GetConfigKeys) | | [`ListUsers`](#ListUsers) | [`DownloadProfilingData`](#DownloadProfilingData) |
+| | |            | [`SetConfigKeys`](#SetConfigKeys) | | [`AddCannedPolicy`](#AddCannedPolicy) | |
 
 
 ## 1. Constructor
@@ -235,9 +235,9 @@ Fetches CPU utilization for all cluster nodes. Returned value is in Bytes.
 
 | Param | Type | Description |
 |-------|------|-------------|
-|`cpu.Load.Avg` | _string_ | The average utilization % of the CPU measured in a 200ms interval |
-|`cpu.Load.Min` | _string_ | The minimum utilization % of the CPU measured in a 200ms interval |
-|`cpu.Load.Max` | _string_ | The maximum utilization % of the CPU measured in a 200ms interval |
+|`cpu.Load.Avg` | _float64_ | The average utilization of the CPU measured in a 200ms interval |
+|`cpu.Load.Min` | _float64_ | The minimum utilization of the CPU measured in a 200ms interval |
+|`cpu.Load.Max` | _float64_ | The maximum utilization of the CPU measured in a 200ms interval |
 |`cpu.Load.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
 
 <a name="ServerMemUsageInfo"></a>
@@ -253,7 +253,7 @@ Fetches Mem utilization for all cluster nodes. Returned value is in Bytes.
 
 | Param | Type | Description |
 |-------|------|-------------|
-|`mem.Usage.Mem` | _string_ | The total number of bytes obtained from the OS |
+|`mem.Usage.Mem` | _uint64_ | The total number of bytes obtained from the OS |
 |`mem.Usage.Error` | _string_ | Error (if any) encountered while accesing the CPU info |
 
 ## 6. Heal operations
@@ -399,7 +399,29 @@ __Example__
     log.Println("New configuration successfully set")
 ```
 
-## 8. IAM operations
+## 8. Top operations
+
+<a name="TopLocks"></a>
+### TopLocks() (LockEntries, error)
+Get the oldest locks from Minio server.
+
+__Example__
+
+``` go
+    locks, err := madmClnt.TopLocks()
+    if err != nil {
+        log.Fatalf("failed due to: %v", err)
+    }
+
+    out, err := json.Marshal(locks)
+    if err != nil {
+        log.Fatalf("Marshal failed due to: %v", err)
+    }
+
+    log.Println("TopLocks received successfully: ", string(out))
+```
+
+## 9. IAM operations
 
 <a name="AddCannedPolicy"></a>
 ### AddCannedPolicy(policyName string, policy string) error
@@ -455,7 +477,7 @@ __Example__
     }
 ```
 
-## 9. Misc operations
+## 10. Misc operations
 
 <a name="SetAdminCredentials"></a>
 ### SetAdminCredentials() error

pkg/madmin/examples/top-locks.go

@@ -0,0 +1,50 @@
+// +build ignore
+
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package main
+
+import (
+	"encoding/json"
+	"log"
+
+	"github.com/minio/minio/pkg/madmin"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are
+	// dummy values, please replace them with original values.
+
+	// API requests are secure (HTTPS) if secure=true and insecure (HTTPS) otherwise.
+	// New returns an Minio Admin client object.
+	madmClnt, err := madmin.New("your-minio.example.com:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	locks, err := madmClnt.TopLocks()
+	if err != nil {
+		log.Fatalf("failed due to: %v", err)
+	}
+
+	out, err := json.Marshal(locks)
+	if err != nil {
+		log.Fatalf("Marshal failed due to: %v", err)
+	}
+	log.Println("Top Locks received successfully: ", string(out))
+}

pkg/madmin/heal-commands.go

@@ -30,6 +30,7 @@ import (
 type HealOpts struct {
 	Recursive bool `json:"recursive"`
 	DryRun    bool `json:"dryRun"`
+	Remove    bool `json:"remove"`
 }
 
 // HealStartSuccess - holds information about a successfully started

pkg/madmin/info-commands.go

@@ -199,9 +199,10 @@ func (adm *AdminClient) ServerDrivesPerfInfo() ([]ServerDrivesPerfInfo, error) {
 // ServerCPULoadInfo holds information about address and cpu load of
 // a single server node
 type ServerCPULoadInfo struct {
-	Addr  string     `json:"addr"`
-	Error string     `json:"error,omitempty"`
-	Load  []cpu.Load `json:"load"`
+	Addr         string     `json:"addr"`
+	Error        string     `json:"error,omitempty"`
+	Load         []cpu.Load `json:"load"`
+	HistoricLoad []cpu.Load `json:"historicLoad"`
 }
 
 // ServerCPULoadInfo - Returns cpu utilization information
@@ -242,9 +243,10 @@ func (adm *AdminClient) ServerCPULoadInfo() ([]ServerCPULoadInfo, error) {
 // ServerMemUsageInfo holds information about address and memory utilization of
 // a single server node
 type ServerMemUsageInfo struct {
-	Addr  string      `json:"addr"`
-	Error string      `json:"error,omitempty"`
-	Usage []mem.Usage `json:"usage"`
+	Addr          string      `json:"addr"`
+	Error         string      `json:"error,omitempty"`
+	Usage         []mem.Usage `json:"usage"`
+	HistoricUsage []mem.Usage `json:"historicUsage"`
 }
 
 // ServerMemUsageInfo - Returns mem utilization information

pkg/madmin/top-commands.go

@@ -0,0 +1,78 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package madmin
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"time"
+)
+
+// LockEntry holds information about client requesting the lock,
+// servers holding the lock, source on the client machine,
+// ID, type(read or write) and time stamp.
+type LockEntry struct {
+	Timestamp  time.Time `json:"time"`       // Timestamp set at the time of initialization.
+	Resource   string    `json:"resource"`   // Resource contains info like bucket, object etc
+	Type       string    `json:"type"`       // Bool whether write or read lock.
+	Source     string    `json:"source"`     // Source which created the lock
+	ServerList []string  `json:"serverlist"` // RPC path of servers issuing the lock.
+	Owner      string    `json:"owner"`      // RPC path of client claiming lock.
+	ID         string    `json:"id"`         // UID to uniquely identify request of client.
+}
+
+// LockEntries - To sort the locks
+type LockEntries []LockEntry
+
+func (l LockEntries) Len() int {
+	return len(l)
+}
+
+func (l LockEntries) Less(i, j int) bool {
+	return l[i].Timestamp.Before(l[j].Timestamp)
+}
+
+func (l LockEntries) Swap(i, j int) {
+	l[i], l[j] = l[j], l[i]
+}
+
+// TopLocks - returns the oldest locks in a minio setup.
+func (adm *AdminClient) TopLocks() (LockEntries, error) {
+	// Execute GET on /minio/admin/v1/top/locks
+	// to get the oldest locks in a minio setup.
+	resp, err := adm.executeMethod("GET",
+		requestData{relPath: "/v1/top/locks"})
+	defer closeResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, httpRespToErrorResponse(resp)
+	}
+
+	response, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return LockEntries{}, err
+	}
+
+	var lockEntries LockEntries
+	err = json.Unmarshal(response, &lockEntries)
+	return lockEntries, err
+}

pkg/mem/mem.go

@@ -18,22 +18,51 @@ package mem
 
 import (
 	"runtime"
-
-	humanize "github.com/dustin/go-humanize"
+	"time"
 )
 
+// historicUsage holds the rolling average of memory used by
+// minio server
+var historicUsage *Usage
+
+// memUsageMeasureInterval is the window of time between
+// two measurements of memory usage
+const memUsageMeasureInterval = 5 * time.Second
+
+// triggers the collection of historic stats about the memory
+// utilized by minio server
+func init() {
+	historicUsage = &Usage{}
+	var cycles uint64
+	go func() {
+		for {
+			time.Sleep(memUsageMeasureInterval)
+			currUsage := GetUsage()
+			currSum := cycles * historicUsage.Mem
+			cycles = cycles + 1
+			historicUsage.Mem = (currSum + currUsage.Mem) / cycles
+		}
+	}()
+}
+
 // Usage holds memory utilization information in human readable format
 type Usage struct {
-	Mem   string `json:"mem"`
+	Mem   uint64 `json:"mem"`
 	Error string `json:"error,omitempty"`
 }
 
+// GetHistoricUsage measures the historic average of memory utilized by
+// current process
+func GetHistoricUsage() Usage {
+	return *historicUsage
+}
+
 // GetUsage measures the total memory provisioned for the current process
 // from the OS
 func GetUsage() Usage {
 	memStats := new(runtime.MemStats)
 	runtime.ReadMemStats(memStats)
 	return Usage{
-		Mem: humanize.IBytes(memStats.Sys),
+		Mem: memStats.Sys,
 	}
 }

pkg/s3select/csv/record.go

@@ -32,7 +32,11 @@ type Record struct {
 	nameIndexMap map[string]int64
 }
 
-// Get - gets the value for a column name.
+// Get - gets the value for a column name. CSV fields do not have any
+// defined type (other than the default string). So this function
+// always returns fields using sql.FromBytes so that the type
+// specified/implied by the query can be used, or can be automatically
+// converted based on the query.
 func (r *Record) Get(name string) (*sql.Value, error) {
 	index, found := r.nameIndexMap[name]
 	if !found {
@@ -40,11 +44,12 @@ func (r *Record) Get(name string) (*sql.Value, error) {
 	}
 
 	if index >= int64(len(r.csvRecord)) {
-		// No value found for column 'name', hence return empty string for compatibility.
-		return sql.NewString(""), nil
+		// No value found for column 'name', hence return null
+		// value
+		return sql.FromNull(), nil
 	}
 
-	return sql.NewString(r.csvRecord[index]), nil
+	return sql.FromBytes([]byte(r.csvRecord[index])), nil
 }
 
 // Set - sets the value for a column name.

pkg/s3select/json/record.go

@@ -37,15 +37,15 @@ func (r *Record) Get(name string) (*sql.Value, error) {
 	result := gjson.GetBytes(r.data, name)
 	switch result.Type {
 	case gjson.Null:
-		return sql.NewNull(), nil
+		return sql.FromNull(), nil
 	case gjson.False:
-		return sql.NewBool(false), nil
+		return sql.FromBool(false), nil
 	case gjson.Number:
-		return sql.NewFloat(result.Float()), nil
+		return sql.FromFloat(result.Float()), nil
 	case gjson.String:
-		return sql.NewString(result.String()), nil
+		return sql.FromString(result.String()), nil
 	case gjson.True:
-		return sql.NewBool(true), nil
+		return sql.FromBool(true), nil
 	}
 
 	return nil, fmt.Errorf("unsupported gjson value %v; %v", result, result.Type)
@@ -54,19 +54,22 @@ func (r *Record) Get(name string) (*sql.Value, error) {
 // Set - sets the value for a column name.
 func (r *Record) Set(name string, value *sql.Value) (err error) {
 	var v interface{}
-	switch value.Type() {
-	case sql.Null:
-		v = value.NullValue()
-	case sql.Bool:
-		v = value.BoolValue()
-	case sql.Int:
-		v = value.IntValue()
-	case sql.Float:
-		v = value.FloatValue()
-	case sql.String:
-		v = value.StringValue()
-	default:
-		return fmt.Errorf("unsupported sql value %v and type %v", value, value.Type())
+	if b, ok := value.ToBool(); ok {
+		v = b
+	} else if f, ok := value.ToFloat(); ok {
+		v = f
+	} else if i, ok := value.ToInt(); ok {
+		v = i
+	} else if t, ok := value.ToTimestamp(); ok {
+		v = sql.FormatSQLTimestamp(t)
+	} else if s, ok := value.ToString(); ok {
+		v = s
+	} else if value.IsNull() {
+		v = nil
+	} else if b, ok := value.ToBytes(); ok {
+		v = string(b)
+	} else {
+		return fmt.Errorf("unsupported sql value %v and type %v", value, value.GetTypeString())
 	}
 
 	name = strings.Replace(name, "*", "__ALL__", -1)

pkg/s3select/message.go

@@ -224,11 +224,14 @@ type messageWriter struct {
 	getProgressFunc func() (int64, int64)
 	bytesReturned   int64
 
-	dataCh  chan []byte
-	doneCh  chan struct{}
-	closeCh chan struct{}
-	stopped uint32
-	closed  uint32
+	payloadBuffer      []byte
+	payloadBufferIndex int
+	payloadCh          chan []byte
+
+	finBytesScanned, finBytesProcessed int64
+
+	errCh  chan []byte
+	doneCh chan struct{}
 }
 
 func (writer *messageWriter) write(data []byte) bool {
@@ -243,142 +246,159 @@ func (writer *messageWriter) write(data []byte) bool {
 func (writer *messageWriter) start() {
 	keepAliveTicker := time.NewTicker(1 * time.Second)
 	var progressTicker *time.Ticker
+	var progressTickerC <-chan time.Time
 	if writer.getProgressFunc != nil {
 		progressTicker = time.NewTicker(1 * time.Minute)
+		progressTickerC = progressTicker.C
 	}
+	recordStagingTicker := time.NewTicker(500 * time.Millisecond)
 
-	go func() {
-		quitFlag := 0
-		for quitFlag == 0 {
-			if progressTicker == nil {
-				select {
-				case data, ok := <-writer.dataCh:
-					if !ok {
-						quitFlag = 1
-						break
-					}
-					if !writer.write(data) {
-						quitFlag = 1
-					}
-				case <-writer.doneCh:
-					quitFlag = 2
-				case <-keepAliveTicker.C:
-					if !writer.write(continuationMessage) {
-						quitFlag = 1
-					}
-				}
-			} else {
-				select {
-				case data, ok := <-writer.dataCh:
-					if !ok {
-						quitFlag = 1
-						break
-					}
-					if !writer.write(data) {
-						quitFlag = 1
-					}
-				case <-writer.doneCh:
-					quitFlag = 2
-				case <-keepAliveTicker.C:
-					if !writer.write(continuationMessage) {
-						quitFlag = 1
-					}
-				case <-progressTicker.C:
-					bytesScanned, bytesProcessed := writer.getProgressFunc()
-					bytesReturned := atomic.LoadInt64(&writer.bytesReturned)
-					if !writer.write(newProgressMessage(bytesScanned, bytesProcessed, bytesReturned)) {
-						quitFlag = 1
-					}
-				}
+	// Exit conditions:
+	//
+	// 1. If a writer.write() returns false, select loop below exits and
+	// closes `doneCh` to indicate to caller to also exit.
+	//
+	// 2. If caller (Evaluate()) has an error, it sends an error
+	// message and waits for this go-routine to quit in
+	// FinishWithError()
+	//
+	// 3. If caller is done, it waits for this go-routine to exit
+	// in Finish()
+
+	quitFlag := false
+	for !quitFlag {
+		select {
+		case data := <-writer.errCh:
+			quitFlag = true
+			// Flush collected records before sending error message
+			if !writer.flushRecords() {
+				break
 			}
-		}
+			writer.write(data)
 
-		atomic.StoreUint32(&writer.stopped, 1)
-		close(writer.closeCh)
+		case payload, ok := <-writer.payloadCh:
+			if !ok {
+				// payloadCh is closed by caller to
+				// indicate finish with success
+				quitFlag = true
 
-		keepAliveTicker.Stop()
-		if progressTicker != nil {
-			progressTicker.Stop()
-		}
-
-		if quitFlag == 2 {
-			for data := range writer.dataCh {
-				if _, err := writer.writer.Write(data); err != nil {
+				if !writer.flushRecords() {
 					break
 				}
+				// Write Stats message, then End message
+				bytesReturned := atomic.LoadInt64(&writer.bytesReturned)
+				if !writer.write(newStatsMessage(writer.finBytesScanned, writer.finBytesProcessed, bytesReturned)) {
+					break
+				}
+				writer.write(endMessage)
+			} else {
+				// Write record payload to staging buffer
+				freeSpace := bufLength - writer.payloadBufferIndex
+				if len(payload) > freeSpace {
+					if !writer.flushRecords() {
+						quitFlag = true
+						break
+					}
+				}
+				copy(writer.payloadBuffer[writer.payloadBufferIndex:], payload)
+				writer.payloadBufferIndex += len(payload)
+			}
+
+		case <-recordStagingTicker.C:
+			if !writer.flushRecords() {
+				quitFlag = true
+				break
+			}
+
+		case <-keepAliveTicker.C:
+			if !writer.write(continuationMessage) {
+				quitFlag = true
+			}
+
+		case <-progressTickerC:
+			bytesScanned, bytesProcessed := writer.getProgressFunc()
+			bytesReturned := atomic.LoadInt64(&writer.bytesReturned)
+			if !writer.write(newProgressMessage(bytesScanned, bytesProcessed, bytesReturned)) {
+				quitFlag = true
 			}
 		}
-	}()
-}
+	}
+	close(writer.doneCh)
 
-func (writer *messageWriter) close() {
-	if atomic.SwapUint32(&writer.closed, 1) == 0 {
-		close(writer.doneCh)
-		for range writer.closeCh {
-			close(writer.dataCh)
-		}
+	recordStagingTicker.Stop()
+	keepAliveTicker.Stop()
+	if progressTicker != nil {
+		progressTicker.Stop()
 	}
 }
 
-func (writer *messageWriter) send(data []byte) error {
-	err := func() error {
-		if atomic.LoadUint32(&writer.stopped) == 1 {
-			return fmt.Errorf("writer already closed")
-		}
-
-		select {
-		case writer.dataCh <- data:
-		case <-writer.doneCh:
-			return fmt.Errorf("closed writer")
-		}
+const (
+	bufLength = maxRecordSize
+)
 
+// Sends a single whole record.
+func (writer *messageWriter) SendRecord(payload []byte) error {
+	select {
+	case writer.payloadCh <- payload:
 		return nil
-	}()
-
-	if err != nil {
-		writer.close()
+	case <-writer.doneCh:
+		return fmt.Errorf("messageWriter is done")
 	}
-
-	return err
 }
 
-func (writer *messageWriter) SendRecords(payload []byte) error {
-	err := writer.send(newRecordsMessage(payload))
-	if err == nil {
-		atomic.AddInt64(&writer.bytesReturned, int64(len(payload)))
+func (writer *messageWriter) flushRecords() bool {
+	if writer.payloadBufferIndex == 0 {
+		return true
 	}
-	return err
+	result := writer.write(newRecordsMessage(writer.payloadBuffer[0:writer.payloadBufferIndex]))
+	if result {
+		atomic.AddInt64(&writer.bytesReturned, int64(writer.payloadBufferIndex))
+		writer.payloadBufferIndex = 0
+	}
+	return result
 }
 
-func (writer *messageWriter) SendStats(bytesScanned, bytesProcessed int64) error {
-	bytesReturned := atomic.LoadInt64(&writer.bytesReturned)
-	err := writer.send(newStatsMessage(bytesScanned, bytesProcessed, bytesReturned))
-	if err != nil {
-		return err
+// Finish is the last call to the message writer - it sends any
+// remaining record payload, then sends statistics and finally the end
+// message.
+func (writer *messageWriter) Finish(bytesScanned, bytesProcessed int64) error {
+	select {
+	case <-writer.doneCh:
+		return fmt.Errorf("messageWriter is done")
+	default:
+		writer.finBytesScanned = bytesScanned
+		writer.finBytesProcessed = bytesProcessed
+		close(writer.payloadCh)
+		// Wait until the `start` go-routine is done.
+		<-writer.doneCh
+		return nil
 	}
-
-	err = writer.send(endMessage)
-	writer.close()
-	return err
 }
 
-func (writer *messageWriter) SendError(errorCode, errorMessage string) error {
-	err := writer.send(newErrorMessage([]byte(errorCode), []byte(errorMessage)))
-	if err == nil {
-		writer.close()
+func (writer *messageWriter) FinishWithError(errorCode, errorMessage string) error {
+	select {
+	case <-writer.doneCh:
+		return fmt.Errorf("messageWriter is done")
+	case writer.errCh <- newErrorMessage([]byte(errorCode), []byte(errorMessage)):
+		// Wait until the `start` go-routine is done.
+		<-writer.doneCh
+		return nil
 	}
-	return err
 }
 
+// newMessageWriter creates a message writer that writes to the HTTP
+// response writer
 func newMessageWriter(w http.ResponseWriter, getProgressFunc func() (bytesScanned, bytesProcessed int64)) *messageWriter {
 	writer := &messageWriter{
 		writer:          w,
 		getProgressFunc: getProgressFunc,
 
-		dataCh:  make(chan []byte),
-		doneCh:  make(chan struct{}),
-		closeCh: make(chan struct{}),
+		payloadBuffer: make([]byte, bufLength),
+		payloadCh:     make(chan []byte),
+
+		errCh:  make(chan []byte),
+		doneCh: make(chan struct{}),
 	}
-	writer.start()
+	go writer.start()
 	return writer
 }

pkg/s3select/parquet/reader.go

@@ -32,7 +32,7 @@ type Reader struct {
 }
 
 // Read - reads single record.
-func (r *Reader) Read() (sql.Record, error) {
+func (r *Reader) Read() (rec sql.Record, rerr error) {
 	parquetRecord, err := r.file.Read()
 	if err != nil {
 		if err != io.EOF {
@@ -43,39 +43,41 @@ func (r *Reader) Read() (sql.Record, error) {
 	}
 
 	record := json.NewRecord()
-	for name, v := range parquetRecord {
+	f := func(name string, v parquetgo.Value) bool {
 		if v.Value == nil {
-			if err = record.Set(name, sql.NewNull()); err != nil {
-				return nil, errParquetParsingError(err)
+			if err := record.Set(name, sql.FromNull()); err != nil {
+				rerr = errParquetParsingError(err)
 			}
-
-			continue
+			return rerr == nil
 		}
 
 		var value *sql.Value
 		switch v.Type {
 		case parquetgen.Type_BOOLEAN:
-			value = sql.NewBool(v.Value.(bool))
+			value = sql.FromBool(v.Value.(bool))
 		case parquetgen.Type_INT32:
-			value = sql.NewInt(int64(v.Value.(int32)))
+			value = sql.FromInt(int64(v.Value.(int32)))
 		case parquetgen.Type_INT64:
-			value = sql.NewInt(v.Value.(int64))
+			value = sql.FromInt(int64(v.Value.(int64)))
 		case parquetgen.Type_FLOAT:
-			value = sql.NewFloat(float64(v.Value.(float32)))
+			value = sql.FromFloat(float64(v.Value.(float32)))
 		case parquetgen.Type_DOUBLE:
-			value = sql.NewFloat(v.Value.(float64))
+			value = sql.FromFloat(v.Value.(float64))
 		case parquetgen.Type_INT96, parquetgen.Type_BYTE_ARRAY, parquetgen.Type_FIXED_LEN_BYTE_ARRAY:
-			value = sql.NewString(string(v.Value.([]byte)))
+			value = sql.FromString(string(v.Value.([]byte)))
 		default:
-			return nil, errParquetParsingError(nil)
+			rerr = errParquetParsingError(nil)
+			return false
 		}
 
 		if err = record.Set(name, value); err != nil {
-			return nil, errParquetParsingError(err)
+			rerr = errParquetParsingError(err)
 		}
+		return rerr == nil
 	}
 
-	return record, nil
+	parquetRecord.Range(f)
+	return record, rerr
 }
 
 // Close - closes underlaying readers.

pkg/s3select/select.go

@@ -49,6 +49,10 @@ const (
 	bzip2Type CompressionType = "bzip2"
 )
 
+const (
+	maxRecordSize = 1 << 20 // 1 MiB
+)
+
 // UnmarshalXML - decodes XML data.
 func (c *CompressionType) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
 	var s string
@@ -105,7 +109,7 @@ func (input *InputSerialization) UnmarshalXML(d *xml.Decoder, start xml.StartEle
 		found++
 	}
 	if !parsedInput.ParquetArgs.IsEmpty() {
-		if parsedInput.CompressionType != noneType {
+		if parsedInput.CompressionType != "" && parsedInput.CompressionType != noneType {
 			return errInvalidRequestParameter(fmt.Errorf("CompressionType must be NONE for Parquet format"))
 		}
 
@@ -178,7 +182,7 @@ type S3Select struct {
 	Output         OutputSerialization `xml:"OutputSerialization"`
 	Progress       RequestProgress     `xml:"RequestProgress"`
 
-	statement      *sql.Select
+	statement      *sql.SelectStatement
 	progressReader *progressReader
 	recordReader   recordReader
 }
@@ -209,12 +213,12 @@ func (s3Select *S3Select) UnmarshalXML(d *xml.Decoder, start xml.StartElement) e
 		return errMissingRequiredParameter(fmt.Errorf("OutputSerialization must be provided"))
 	}
 
-	statement, err := sql.NewSelect(parsedS3Select.Expression)
+	statement, err := sql.ParseSelectStatement(parsedS3Select.Expression)
 	if err != nil {
 		return err
 	}
 
-	parsedS3Select.statement = statement
+	parsedS3Select.statement = &statement
 
 	*s3Select = S3Select(parsedS3Select)
 	return nil
@@ -324,7 +328,12 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
 			return false
 		}
 
-		if err = writer.SendRecords(data); err != nil {
+		if len(data) > maxRecordSize {
+			writer.FinishWithError("OverMaxRecordSize", "The length of a record in the input or result is greater than maxCharsPerRecord of 1 MB.")
+			return false
+		}
+
+		if err = writer.SendRecord(data); err != nil {
 			// FIXME: log this error.
 			err = nil
 			return false
@@ -334,6 +343,14 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
 	}
 
 	for {
+		if s3Select.statement.LimitReached() {
+			if err = writer.Finish(s3Select.getProgress()); err != nil {
+				// FIXME: log this error.
+				err = nil
+			}
+			break
+		}
+
 		if inputRecord, err = s3Select.recordReader.Read(); err != nil {
 			if err != io.EOF {
 				break
@@ -350,20 +367,23 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
 				}
 			}
 
-			if err = writer.SendStats(s3Select.getProgress()); err != nil {
+			if err = writer.Finish(s3Select.getProgress()); err != nil {
 				// FIXME: log this error.
 				err = nil
 			}
-
 			break
 		}
 
-		outputRecord = s3Select.outputRecord()
-		if outputRecord, err = s3Select.statement.Eval(inputRecord, outputRecord); err != nil {
-			break
-		}
+		if s3Select.statement.IsAggregated() {
+			if err = s3Select.statement.AggregateRow(inputRecord); err != nil {
+				break
+			}
+		} else {
+			outputRecord = s3Select.outputRecord()
+			if outputRecord, err = s3Select.statement.Eval(inputRecord, outputRecord); err != nil {
+				break
+			}
 
-		if !s3Select.statement.IsAggregated() {
 			if !sendRecord() {
 				break
 			}
@@ -371,7 +391,7 @@ func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
 	}
 
 	if err != nil {
-		if serr := writer.SendError("InternalError", err.Error()); serr != nil {
+		if serr := writer.FinishWithError("InternalError", err.Error()); serr != nil {
 			// FIXME: log errors.
 		}
 	}

pkg/s3select/select_test.go

@@ -49,7 +49,7 @@ func (w *testResponseWriter) WriteHeader(statusCode int) {
 func (w *testResponseWriter) Flush() {
 }
 
-func TestCSVINput(t *testing.T) {
+func TestCSVInput(t *testing.T) {
 	var requestXML = []byte(`
 <?xml version="1.0" encoding="UTF-8"?>
 <SelectObjectContentRequest>
@@ -78,7 +78,7 @@ func TestCSVINput(t *testing.T) {
 `)
 
 	var expectedResult = []byte{
-		0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 155, 11, 218, 231, 0, 0, 0, 112, 0, 0, 0, 85, 135, 241, 154, 221, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 146, 219, 90, 187, 0, 0, 0, 114, 0, 0, 0, 85, 253, 49, 201, 189, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 172, 136, 9, 26, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 53, 48, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 53, 48, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 253, 105, 8, 216, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
+		0, 0, 0, 137, 0, 0, 0, 85, 194, 213, 168, 241, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 75, 182, 193, 80, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 53, 48, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 53, 48, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 253, 105, 8, 216, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
 	}
 
 	s3Select, err := NewS3Select(bytes.NewReader(requestXML))
@@ -129,7 +129,7 @@ func TestJSONInput(t *testing.T) {
 `)
 
 	var expectedResult = []byte{
-		0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 155, 11, 218, 231, 0, 0, 0, 112, 0, 0, 0, 85, 135, 241, 154, 221, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 146, 219, 90, 187, 0, 0, 0, 114, 0, 0, 0, 85, 253, 49, 201, 189, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 172, 136, 9, 26, 0, 0, 0, 237, 0, 0, 0, 67, 90, 179, 204, 45, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 49, 49, 50, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 49, 49, 50, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 181, 40, 50, 250, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
+		0, 0, 0, 137, 0, 0, 0, 85, 194, 213, 168, 241, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 75, 182, 193, 80, 0, 0, 0, 237, 0, 0, 0, 67, 90, 179, 204, 45, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 49, 49, 50, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 49, 49, 50, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 181, 40, 50, 250, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
 	}
 
 	s3Select, err := NewS3Select(bytes.NewReader(requestXML))
@@ -197,7 +197,7 @@ func TestParquetInput(t *testing.T) {
 	}
 
 	var expectedResult = []byte{
-		0, 0, 0, 113, 0, 0, 0, 85, 186, 145, 179, 109, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 155, 11, 218, 231, 0, 0, 0, 112, 0, 0, 0, 85, 135, 241, 154, 221, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 146, 219, 90, 187, 0, 0, 0, 114, 0, 0, 0, 85, 253, 49, 201, 189, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 172, 136, 9, 26, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 45, 49, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 45, 49, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 128, 96, 253, 66, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
+		0, 0, 0, 137, 0, 0, 0, 85, 194, 213, 168, 241, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 24, 97, 112, 112, 108, 105, 99, 97, 116, 105, 111, 110, 47, 111, 99, 116, 101, 116, 45, 115, 116, 114, 101, 97, 109, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 7, 82, 101, 99, 111, 114, 100, 115, 45, 49, 44, 102, 111, 111, 44, 116, 114, 117, 101, 10, 44, 98, 97, 114, 44, 102, 97, 108, 115, 101, 10, 50, 46, 53, 44, 98, 97, 122, 44, 116, 114, 117, 101, 10, 75, 182, 193, 80, 0, 0, 0, 235, 0, 0, 0, 67, 213, 243, 57, 141, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 13, 58, 99, 111, 110, 116, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 8, 116, 101, 120, 116, 47, 120, 109, 108, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 5, 83, 116, 97, 116, 115, 60, 63, 120, 109, 108, 32, 118, 101, 114, 115, 105, 111, 110, 61, 34, 49, 46, 48, 34, 32, 101, 110, 99, 111, 100, 105, 110, 103, 61, 34, 85, 84, 70, 45, 56, 34, 63, 62, 60, 83, 116, 97, 116, 115, 62, 60, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 45, 49, 60, 47, 66, 121, 116, 101, 115, 83, 99, 97, 110, 110, 101, 100, 62, 60, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 45, 49, 60, 47, 66, 121, 116, 101, 115, 80, 114, 111, 99, 101, 115, 115, 101, 100, 62, 60, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 51, 54, 60, 47, 66, 121, 116, 101, 115, 82, 101, 116, 117, 114, 110, 101, 100, 62, 60, 47, 83, 116, 97, 116, 115, 62, 128, 96, 253, 66, 0, 0, 0, 56, 0, 0, 0, 40, 193, 198, 132, 212, 13, 58, 109, 101, 115, 115, 97, 103, 101, 45, 116, 121, 112, 101, 7, 0, 5, 101, 118, 101, 110, 116, 11, 58, 101, 118, 101, 110, 116, 45, 116, 121, 112, 101, 7, 0, 3, 69, 110, 100, 207, 151, 211, 146,
 	}
 
 	s3Select, err := NewS3Select(bytes.NewReader(requestXML))

pkg/s3select/sql/aggregation.go

@@ -0,0 +1,318 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"errors"
+	"fmt"
+)
+
+// Aggregation Function name constants
+const (
+	aggFnAvg   FuncName = "AVG"
+	aggFnCount FuncName = "COUNT"
+	aggFnMax   FuncName = "MAX"
+	aggFnMin   FuncName = "MIN"
+	aggFnSum   FuncName = "SUM"
+)
+
+var (
+	errNonNumericArg = func(fnStr FuncName) error {
+		return fmt.Errorf("%s() requires a numeric argument", fnStr)
+	}
+	errInvalidAggregation = errors.New("Invalid aggregation seen")
+)
+
+type aggVal struct {
+	runningSum             *Value
+	runningCount           int64
+	runningMax, runningMin *Value
+
+	// Stores if at least one record has been seen
+	seen bool
+}
+
+func newAggVal(fn FuncName) *aggVal {
+	switch fn {
+	case aggFnAvg, aggFnSum:
+		return &aggVal{runningSum: FromInt(0)}
+	case aggFnMin:
+		return &aggVal{runningMin: FromInt(0)}
+	case aggFnMax:
+		return &aggVal{runningMax: FromInt(0)}
+	default:
+		return &aggVal{}
+	}
+}
+
+// evalAggregationNode - performs partial computation using the
+// current row and stores the result.
+//
+// On success, it returns (nil, nil).
+func (e *FuncExpr) evalAggregationNode(r Record) error {
+	// It is assumed that this function is called only when
+	// `e` is an aggregation function.
+
+	var val *Value
+	var err error
+	funcName := e.getFunctionName()
+	if aggFnCount == funcName {
+		if e.Count.StarArg {
+			// Handle COUNT(*)
+			e.aggregate.runningCount++
+			return nil
+		}
+
+		val, err = e.Count.ExprArg.evalNode(r)
+		if err != nil {
+			return err
+		}
+	} else {
+		// Evaluate the (only) argument
+		val, err = e.SFunc.ArgsList[0].evalNode(r)
+		if err != nil {
+			return err
+		}
+	}
+
+	if val.IsNull() {
+		// E.g. the column or field does not exist in the
+		// record - in all such cases the aggregation is not
+		// updated.
+		return nil
+	}
+
+	argVal := val
+	if funcName != aggFnCount {
+		// All aggregation functions, except COUNT require a
+		// numeric argument.
+
+		// Here, we diverge from Amazon S3 behavior by
+		// inferring untyped values are numbers.
+		if i, ok := argVal.bytesToInt(); ok {
+			argVal.setInt(i)
+		} else if f, ok := argVal.bytesToFloat(); ok {
+			argVal.setFloat(f)
+		} else {
+			return errNonNumericArg(funcName)
+		}
+	}
+
+	// Mark that we have seen one non-null value.
+	isFirstRow := false
+	if !e.aggregate.seen {
+		e.aggregate.seen = true
+		isFirstRow = true
+	}
+
+	switch funcName {
+	case aggFnCount:
+		// For all non-null values, the count is incremented.
+		e.aggregate.runningCount++
+
+	case aggFnAvg:
+		e.aggregate.runningCount++
+		err = e.aggregate.runningSum.arithOp(opPlus, argVal)
+
+	case aggFnMin:
+		err = e.aggregate.runningMin.minmax(argVal, false, isFirstRow)
+
+	case aggFnMax:
+		err = e.aggregate.runningMax.minmax(argVal, true, isFirstRow)
+
+	case aggFnSum:
+		err = e.aggregate.runningSum.arithOp(opPlus, argVal)
+
+	default:
+		err = errInvalidAggregation
+	}
+
+	return err
+}
+
+func (e *AliasedExpression) aggregateRow(r Record) error {
+	return e.Expression.aggregateRow(r)
+}
+
+func (e *Expression) aggregateRow(r Record) error {
+	for _, ex := range e.And {
+		err := ex.aggregateRow(r)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (e *AndCondition) aggregateRow(r Record) error {
+	for _, ex := range e.Condition {
+		err := ex.aggregateRow(r)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (e *Condition) aggregateRow(r Record) error {
+	if e.Operand != nil {
+		return e.Operand.aggregateRow(r)
+	}
+	return e.Not.aggregateRow(r)
+}
+
+func (e *ConditionOperand) aggregateRow(r Record) error {
+	err := e.Operand.aggregateRow(r)
+	if err != nil {
+		return err
+	}
+
+	if e.ConditionRHS == nil {
+		return nil
+	}
+
+	switch {
+	case e.ConditionRHS.Compare != nil:
+		return e.ConditionRHS.Compare.Operand.aggregateRow(r)
+	case e.ConditionRHS.Between != nil:
+		err = e.ConditionRHS.Between.Start.aggregateRow(r)
+		if err != nil {
+			return err
+		}
+		return e.ConditionRHS.Between.End.aggregateRow(r)
+	case e.ConditionRHS.In != nil:
+		for _, elt := range e.ConditionRHS.In.Expressions {
+			err = elt.aggregateRow(r)
+			if err != nil {
+				return err
+			}
+		}
+		return nil
+	case e.ConditionRHS.Like != nil:
+		err = e.ConditionRHS.Like.Pattern.aggregateRow(r)
+		if err != nil {
+			return err
+		}
+		return e.ConditionRHS.Like.EscapeChar.aggregateRow(r)
+	default:
+		return errInvalidASTNode
+	}
+}
+
+func (e *Operand) aggregateRow(r Record) error {
+	err := e.Left.aggregateRow(r)
+	if err != nil {
+		return err
+	}
+	for _, rt := range e.Right {
+		err = rt.Right.aggregateRow(r)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (e *MultOp) aggregateRow(r Record) error {
+	err := e.Left.aggregateRow(r)
+	if err != nil {
+		return err
+	}
+	for _, rt := range e.Right {
+		err = rt.Right.aggregateRow(r)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (e *UnaryTerm) aggregateRow(r Record) error {
+	if e.Negated != nil {
+		return e.Negated.Term.aggregateRow(r)
+	}
+	return e.Primary.aggregateRow(r)
+}
+
+func (e *PrimaryTerm) aggregateRow(r Record) error {
+	switch {
+	case e.SubExpression != nil:
+		return e.SubExpression.aggregateRow(r)
+	case e.FuncCall != nil:
+		return e.FuncCall.aggregateRow(r)
+	}
+	return nil
+}
+
+func (e *FuncExpr) aggregateRow(r Record) error {
+	switch e.getFunctionName() {
+	case aggFnAvg, aggFnSum, aggFnMax, aggFnMin, aggFnCount:
+		return e.evalAggregationNode(r)
+	default:
+		// TODO: traverse arguments and call aggregateRow on
+		// them if they could be an ancestor of an
+		// aggregation.
+	}
+	return nil
+}
+
+// getAggregate() implementation for each AST node follows. This is
+// called after calling aggregateRow() on each input row, to calculate
+// the final aggregate result.
+
+func (e *Expression) getAggregate() (*Value, error) {
+	return e.evalNode(nil)
+}
+
+func (e *FuncExpr) getAggregate() (*Value, error) {
+	switch e.getFunctionName() {
+	case aggFnCount:
+		return FromFloat(float64(e.aggregate.runningCount)), nil
+
+	case aggFnAvg:
+		if e.aggregate.runningCount == 0 {
+			// No rows were seen by AVG.
+			return FromNull(), nil
+		}
+		err := e.aggregate.runningSum.arithOp(opDivide, FromInt(e.aggregate.runningCount))
+		return e.aggregate.runningSum, err
+
+	case aggFnMin:
+		if !e.aggregate.seen {
+			// No rows were seen by MIN
+			return FromNull(), nil
+		}
+		return e.aggregate.runningMin, nil
+
+	case aggFnMax:
+		if !e.aggregate.seen {
+			// No rows were seen by MAX
+			return FromNull(), nil
+		}
+		return e.aggregate.runningMax, nil
+
+	case aggFnSum:
+		// TODO: check if returning 0 when no rows were seen
+		// by SUM is expected behavior.
+		return e.aggregate.runningSum, nil
+
+	default:
+		// TODO:
+	}
+
+	return nil, errInvalidAggregation
+}

pkg/s3select/sql/analysis.go

@@ -0,0 +1,305 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"errors"
+	"fmt"
+)
+
+// Query analysis - The query is analyzed to determine if it involves
+// aggregation.
+//
+// Aggregation functions - An expression that involves aggregation of
+// rows in some manner. Requires all input rows to be processed,
+// before a result is returned.
+//
+// Row function - An expression that depends on a value in the
+// row. They have an output for each input row.
+//
+// Some types of a queries are not valid. For example, an aggregation
+// function combined with a row function is meaningless ("AVG(s.Age) +
+// s.Salary"). Analysis determines if such a scenario exists so an
+// error can be returned.
+
+var (
+	// Fatal error for query processing.
+	errNestedAggregation      = errors.New("Cannot nest aggregations")
+	errFunctionNotImplemented = errors.New("Function is not yet implemented")
+	errUnexpectedInvalidNode  = errors.New("Unexpected node value")
+	errInvalidKeypath         = errors.New("A provided keypath is invalid")
+)
+
+// qProp contains analysis info about an SQL term.
+type qProp struct {
+	isAggregation, isRowFunc bool
+
+	err error
+}
+
+// `combine` combines a pair of `qProp`s, so that errors are
+// propagated correctly, and checks that an aggregation is not being
+// combined with a row-function term.
+func (p *qProp) combine(q qProp) {
+	switch {
+	case p.err != nil:
+		// Do nothing
+	case q.err != nil:
+		p.err = q.err
+	default:
+		p.isAggregation = p.isAggregation || q.isAggregation
+		p.isRowFunc = p.isRowFunc || q.isRowFunc
+		if p.isAggregation && p.isRowFunc {
+			p.err = errNestedAggregation
+		}
+	}
+}
+
+func (e *SelectExpression) analyze(s *Select) (result qProp) {
+	if e.All {
+		return qProp{isRowFunc: true}
+	}
+
+	for _, ex := range e.Expressions {
+		result.combine(ex.analyze(s))
+	}
+	return
+}
+
+func (e *AliasedExpression) analyze(s *Select) qProp {
+	return e.Expression.analyze(s)
+}
+
+func (e *Expression) analyze(s *Select) (result qProp) {
+	for _, ac := range e.And {
+		result.combine(ac.analyze(s))
+	}
+	return
+}
+
+func (e *AndCondition) analyze(s *Select) (result qProp) {
+	for _, ac := range e.Condition {
+		result.combine(ac.analyze(s))
+	}
+	return
+}
+
+func (e *Condition) analyze(s *Select) (result qProp) {
+	if e.Operand != nil {
+		result = e.Operand.analyze(s)
+	} else {
+		result = e.Not.analyze(s)
+	}
+	return
+}
+
+func (e *ConditionOperand) analyze(s *Select) (result qProp) {
+	if e.ConditionRHS == nil {
+		result = e.Operand.analyze(s)
+	} else {
+		result.combine(e.Operand.analyze(s))
+		result.combine(e.ConditionRHS.analyze(s))
+	}
+	return
+}
+
+func (e *ConditionRHS) analyze(s *Select) (result qProp) {
+	switch {
+	case e.Compare != nil:
+		result = e.Compare.Operand.analyze(s)
+	case e.Between != nil:
+		result.combine(e.Between.Start.analyze(s))
+		result.combine(e.Between.End.analyze(s))
+	case e.In != nil:
+		for _, elt := range e.In.Expressions {
+			result.combine(elt.analyze(s))
+		}
+	case e.Like != nil:
+		result.combine(e.Like.Pattern.analyze(s))
+		if e.Like.EscapeChar != nil {
+			result.combine(e.Like.EscapeChar.analyze(s))
+		}
+	default:
+		result = qProp{err: errUnexpectedInvalidNode}
+	}
+	return
+}
+
+func (e *Operand) analyze(s *Select) (result qProp) {
+	result.combine(e.Left.analyze(s))
+	for _, r := range e.Right {
+		result.combine(r.Right.analyze(s))
+	}
+	return
+}
+
+func (e *MultOp) analyze(s *Select) (result qProp) {
+	result.combine(e.Left.analyze(s))
+	for _, r := range e.Right {
+		result.combine(r.Right.analyze(s))
+	}
+	return
+}
+
+func (e *UnaryTerm) analyze(s *Select) (result qProp) {
+	if e.Negated != nil {
+		result = e.Negated.Term.analyze(s)
+	} else {
+		result = e.Primary.analyze(s)
+	}
+	return
+}
+
+func (e *PrimaryTerm) analyze(s *Select) (result qProp) {
+	switch {
+	case e.Value != nil:
+		result = qProp{}
+
+	case e.JPathExpr != nil:
+		// Check if the path expression is valid
+		if len(e.JPathExpr.PathExpr) > 0 {
+			if e.JPathExpr.BaseKey.String() != s.From.As {
+				result = qProp{err: errInvalidKeypath}
+				return
+			}
+		}
+		result = qProp{isRowFunc: true}
+
+	case e.SubExpression != nil:
+		result = e.SubExpression.analyze(s)
+
+	case e.FuncCall != nil:
+		result = e.FuncCall.analyze(s)
+
+	default:
+		result = qProp{err: errUnexpectedInvalidNode}
+	}
+	return
+}
+
+func (e *FuncExpr) analyze(s *Select) (result qProp) {
+	funcName := e.getFunctionName()
+
+	switch funcName {
+	case sqlFnCast:
+		return e.Cast.Expr.analyze(s)
+
+	case sqlFnExtract:
+		return e.Extract.From.analyze(s)
+
+	case sqlFnDateAdd:
+		result.combine(e.DateAdd.Quantity.analyze(s))
+		result.combine(e.DateAdd.Timestamp.analyze(s))
+		return result
+
+	case sqlFnDateDiff:
+		result.combine(e.DateDiff.Timestamp1.analyze(s))
+		result.combine(e.DateDiff.Timestamp2.analyze(s))
+		return result
+
+	// Handle aggregation function calls
+	case aggFnAvg, aggFnMax, aggFnMin, aggFnSum, aggFnCount:
+		// Initialize accumulator
+		e.aggregate = newAggVal(funcName)
+
+		var exprA qProp
+		if funcName == aggFnCount {
+			if e.Count.StarArg {
+				return qProp{isAggregation: true}
+			}
+
+			exprA = e.Count.ExprArg.analyze(s)
+		} else {
+			if len(e.SFunc.ArgsList) != 1 {
+				return qProp{err: fmt.Errorf("%s takes exactly one argument", funcName)}
+			}
+			exprA = e.SFunc.ArgsList[0].analyze(s)
+		}
+
+		if exprA.err != nil {
+			return exprA
+		}
+		if exprA.isAggregation {
+			return qProp{err: errNestedAggregation}
+		}
+		return qProp{isAggregation: true}
+
+	case sqlFnCoalesce:
+		if len(e.SFunc.ArgsList) == 0 {
+			return qProp{err: fmt.Errorf("%s needs at least one argument", string(funcName))}
+		}
+		for _, arg := range e.SFunc.ArgsList {
+			result.combine(arg.analyze(s))
+		}
+		return result
+
+	case sqlFnNullIf:
+		if len(e.SFunc.ArgsList) != 2 {
+			return qProp{err: fmt.Errorf("%s needs exactly 2 arguments", string(funcName))}
+		}
+		for _, arg := range e.SFunc.ArgsList {
+			result.combine(arg.analyze(s))
+		}
+		return result
+
+	case sqlFnCharLength, sqlFnCharacterLength:
+		if len(e.SFunc.ArgsList) != 1 {
+			return qProp{err: fmt.Errorf("%s needs exactly 2 arguments", string(funcName))}
+		}
+		for _, arg := range e.SFunc.ArgsList {
+			result.combine(arg.analyze(s))
+		}
+		return result
+
+	case sqlFnLower, sqlFnUpper:
+		if len(e.SFunc.ArgsList) != 1 {
+			return qProp{err: fmt.Errorf("%s needs exactly 2 arguments", string(funcName))}
+		}
+		for _, arg := range e.SFunc.ArgsList {
+			result.combine(arg.analyze(s))
+		}
+		return result
+
+	case sqlFnSubstring:
+		errVal := fmt.Errorf("Invalid argument(s) to %s", string(funcName))
+		result.combine(e.Substring.Expr.analyze(s))
+		switch {
+		case e.Substring.From != nil:
+			result.combine(e.Substring.From.analyze(s))
+			if e.Substring.For != nil {
+				result.combine(e.Substring.Expr.analyze(s))
+			}
+		case e.Substring.Arg2 != nil:
+			result.combine(e.Substring.Arg2.analyze(s))
+			if e.Substring.Arg3 != nil {
+				result.combine(e.Substring.Arg3.analyze(s))
+			}
+		default:
+			result.err = errVal
+		}
+		return result
+
+	case sqlFnUTCNow:
+		if len(e.SFunc.ArgsList) != 0 {
+			result.err = fmt.Errorf("%s() takes no arguments", string(funcName))
+		}
+		return result
+	}
+
+	// TODO: implement other functions
+	return qProp{err: errFunctionNotImplemented}
+}

pkg/s3select/sql/arithexpr.go

@@ -1,175 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2019 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sql
-
-import "fmt"
-
-// ArithOperator - arithmetic operator.
-type ArithOperator string
-
-const (
-	// Add operator '+'.
-	Add ArithOperator = "+"
-
-	// Subtract operator '-'.
-	Subtract ArithOperator = "-"
-
-	// Multiply operator '*'.
-	Multiply ArithOperator = "*"
-
-	// Divide operator '/'.
-	Divide ArithOperator = "/"
-
-	// Modulo operator '%'.
-	Modulo ArithOperator = "%"
-)
-
-// arithExpr - arithmetic function.
-type arithExpr struct {
-	left     Expr
-	right    Expr
-	operator ArithOperator
-	funcType Type
-}
-
-// String - returns string representation of this function.
-func (f *arithExpr) String() string {
-	return fmt.Sprintf("(%v %v %v)", f.left, f.operator, f.right)
-}
-
-func (f *arithExpr) compute(lv, rv *Value) (*Value, error) {
-	leftValueType := lv.Type()
-	rightValueType := rv.Type()
-	if !leftValueType.isNumber() {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to number", f, leftValueType)
-		return nil, errExternalEvalException(err)
-	}
-	if !rightValueType.isNumber() {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to number", f, rightValueType)
-		return nil, errExternalEvalException(err)
-	}
-
-	leftValue := lv.FloatValue()
-	rightValue := rv.FloatValue()
-
-	var result float64
-	switch f.operator {
-	case Add:
-		result = leftValue + rightValue
-	case Subtract:
-		result = leftValue - rightValue
-	case Multiply:
-		result = leftValue * rightValue
-	case Divide:
-		result = leftValue / rightValue
-	case Modulo:
-		result = float64(int64(leftValue) % int64(rightValue))
-	}
-
-	if leftValueType == Float || rightValueType == Float {
-		return NewFloat(result), nil
-	}
-
-	return NewInt(int64(result)), nil
-}
-
-// Call - evaluates this function for given arg values and returns result as Value.
-func (f *arithExpr) Eval(record Record) (*Value, error) {
-	leftValue, err := f.left.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	rightValue, err := f.right.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	if f.funcType == aggregateFunction {
-		return nil, nil
-	}
-
-	return f.compute(leftValue, rightValue)
-}
-
-// AggregateValue - returns aggregated value.
-func (f *arithExpr) AggregateValue() (*Value, error) {
-	if f.funcType != aggregateFunction {
-		err := fmt.Errorf("%v is not aggreate expression", f)
-		return nil, errExternalEvalException(err)
-	}
-
-	lv, err := f.left.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-
-	rv, err := f.right.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-
-	return f.compute(lv, rv)
-}
-
-// Type - returns arithmeticFunction or aggregateFunction type.
-func (f *arithExpr) Type() Type {
-	return f.funcType
-}
-
-// ReturnType - returns Float as return type.
-func (f *arithExpr) ReturnType() Type {
-	return Float
-}
-
-// newArithExpr - creates new arithmetic function.
-func newArithExpr(operator ArithOperator, left, right Expr) (*arithExpr, error) {
-	if !left.ReturnType().isNumberKind() {
-		err := fmt.Errorf("operator %v: left side expression %v evaluate to %v, not number", operator, left, left.ReturnType())
-		return nil, errInvalidDataType(err)
-	}
-
-	if !right.ReturnType().isNumberKind() {
-		err := fmt.Errorf("operator %v: right side expression %v evaluate to %v; not number", operator, right, right.ReturnType())
-		return nil, errInvalidDataType(err)
-	}
-
-	funcType := arithmeticFunction
-	if left.Type() == aggregateFunction || right.Type() == aggregateFunction {
-		funcType = aggregateFunction
-		switch left.Type() {
-		case Int, Float, aggregateFunction:
-		default:
-			err := fmt.Errorf("operator %v: left side expression %v return type %v is incompatible for aggregate evaluation", operator, left, left.Type())
-			return nil, errUnsupportedSQLOperation(err)
-		}
-
-		switch right.Type() {
-		case Int, Float, aggregateFunction:
-		default:
-			err := fmt.Errorf("operator %v: right side expression %v return type %v is incompatible for aggregate evaluation", operator, right, right.Type())
-			return nil, errUnsupportedSQLOperation(err)
-		}
-	}
-
-	return &arithExpr{
-		left:     left,
-		right:    right,
-		operator: operator,
-		funcType: funcType,
-	}, nil
-}

pkg/s3select/sql/compexpr.go

@@ -1,636 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2019 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sql
-
-import (
-	"fmt"
-	"regexp"
-	"strings"
-)
-
-// ComparisonOperator - comparison operator.
-type ComparisonOperator string
-
-const (
-	// Equal operator '='.
-	Equal ComparisonOperator = "="
-
-	// NotEqual operator '!=' or '<>'.
-	NotEqual ComparisonOperator = "!="
-
-	// LessThan operator '<'.
-	LessThan ComparisonOperator = "<"
-
-	// GreaterThan operator '>'.
-	GreaterThan ComparisonOperator = ">"
-
-	// LessThanEqual operator '<='.
-	LessThanEqual ComparisonOperator = "<="
-
-	// GreaterThanEqual operator '>='.
-	GreaterThanEqual ComparisonOperator = ">="
-
-	// Between operator 'BETWEEN'
-	Between ComparisonOperator = "between"
-
-	// In operator 'IN'
-	In ComparisonOperator = "in"
-
-	// Like operator 'LIKE'
-	Like ComparisonOperator = "like"
-
-	// NotBetween operator 'NOT BETWEEN'
-	NotBetween ComparisonOperator = "not between"
-
-	// NotIn operator 'NOT IN'
-	NotIn ComparisonOperator = "not in"
-
-	// NotLike operator 'NOT LIKE'
-	NotLike ComparisonOperator = "not like"
-
-	// IsNull operator 'IS NULL'
-	IsNull ComparisonOperator = "is null"
-
-	// IsNotNull operator 'IS NOT NULL'
-	IsNotNull ComparisonOperator = "is not null"
-)
-
-// String - returns string representation of this operator.
-func (operator ComparisonOperator) String() string {
-	return strings.ToUpper((string(operator)))
-}
-
-func equal(leftValue, rightValue *Value) (bool, error) {
-	switch {
-	case leftValue.Type() == Null && rightValue.Type() == Null:
-		return true, nil
-	case leftValue.Type() == Bool && rightValue.Type() == Bool:
-		return leftValue.BoolValue() == rightValue.BoolValue(), nil
-	case (leftValue.Type() == Int || leftValue.Type() == Float) &&
-		(rightValue.Type() == Int || rightValue.Type() == Float):
-		return leftValue.FloatValue() == rightValue.FloatValue(), nil
-	case leftValue.Type() == String && rightValue.Type() == String:
-		return leftValue.StringValue() == rightValue.StringValue(), nil
-	case leftValue.Type() == Timestamp && rightValue.Type() == Timestamp:
-		return leftValue.TimeValue() == rightValue.TimeValue(), nil
-	}
-
-	return false, fmt.Errorf("left value type %v and right value type %v are incompatible for equality check", leftValue.Type(), rightValue.Type())
-}
-
-// comparisonExpr - comparison function.
-type comparisonExpr struct {
-	left     Expr
-	right    Expr
-	to       Expr
-	operator ComparisonOperator
-	funcType Type
-}
-
-// String - returns string representation of this function.
-func (f *comparisonExpr) String() string {
-	switch f.operator {
-	case Equal, NotEqual, LessThan, GreaterThan, LessThanEqual, GreaterThanEqual, In, Like, NotIn, NotLike:
-		return fmt.Sprintf("(%v %v %v)", f.left, f.operator, f.right)
-	case Between, NotBetween:
-		return fmt.Sprintf("(%v %v %v AND %v)", f.left, f.operator, f.right, f.to)
-	}
-
-	return fmt.Sprintf("(%v %v %v %v)", f.left, f.right, f.to, f.operator)
-}
-
-func (f *comparisonExpr) equal(leftValue, rightValue *Value) (*Value, error) {
-	result, err := equal(leftValue, rightValue)
-	if err != nil {
-		err = fmt.Errorf("%v: %v", f, err)
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(result), nil
-}
-
-func (f *comparisonExpr) notEqual(leftValue, rightValue *Value) (*Value, error) {
-	result, err := equal(leftValue, rightValue)
-	if err != nil {
-		err = fmt.Errorf("%v: %v", f, err)
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(!result), nil
-}
-
-func (f *comparisonExpr) lessThan(leftValue, rightValue *Value) (*Value, error) {
-	if !leftValue.Type().isNumber() {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to number", f, leftValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-	if !rightValue.Type().isNumber() {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to number", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(leftValue.FloatValue() < rightValue.FloatValue()), nil
-}
-
-func (f *comparisonExpr) greaterThan(leftValue, rightValue *Value) (*Value, error) {
-	if !leftValue.Type().isNumber() {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to number", f, leftValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-	if !rightValue.Type().isNumber() {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to number", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(leftValue.FloatValue() > rightValue.FloatValue()), nil
-}
-
-func (f *comparisonExpr) lessThanEqual(leftValue, rightValue *Value) (*Value, error) {
-	if !leftValue.Type().isNumber() {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to number", f, leftValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-	if !rightValue.Type().isNumber() {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to number", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(leftValue.FloatValue() <= rightValue.FloatValue()), nil
-}
-
-func (f *comparisonExpr) greaterThanEqual(leftValue, rightValue *Value) (*Value, error) {
-	if !leftValue.Type().isNumber() {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to number", f, leftValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-	if !rightValue.Type().isNumber() {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to number", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(leftValue.FloatValue() >= rightValue.FloatValue()), nil
-}
-
-func (f *comparisonExpr) computeBetween(leftValue, fromValue, toValue *Value) (bool, error) {
-	if !leftValue.Type().isNumber() {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to number", f, leftValue.Type())
-		return false, errExternalEvalException(err)
-	}
-	if !fromValue.Type().isNumber() {
-		err := fmt.Errorf("%v: from side expression evaluated to %v; not to number", f, fromValue.Type())
-		return false, errExternalEvalException(err)
-	}
-	if !toValue.Type().isNumber() {
-		err := fmt.Errorf("%v: to side expression evaluated to %v; not to number", f, toValue.Type())
-		return false, errExternalEvalException(err)
-	}
-
-	return leftValue.FloatValue() >= fromValue.FloatValue() &&
-		leftValue.FloatValue() <= toValue.FloatValue(), nil
-}
-
-func (f *comparisonExpr) between(leftValue, fromValue, toValue *Value) (*Value, error) {
-	result, err := f.computeBetween(leftValue, fromValue, toValue)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewBool(result), nil
-}
-
-func (f *comparisonExpr) notBetween(leftValue, fromValue, toValue *Value) (*Value, error) {
-	result, err := f.computeBetween(leftValue, fromValue, toValue)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewBool(!result), nil
-}
-
-func (f *comparisonExpr) computeIn(leftValue, rightValue *Value) (found bool, err error) {
-	if rightValue.Type() != Array {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to Array", f, rightValue.Type())
-		return false, errExternalEvalException(err)
-	}
-
-	values := rightValue.ArrayValue()
-
-	for i := range values {
-		found, err = equal(leftValue, values[i])
-		if err != nil {
-			return false, err
-		}
-
-		if found {
-			return true, nil
-		}
-	}
-
-	return false, nil
-}
-
-func (f *comparisonExpr) in(leftValue, rightValue *Value) (*Value, error) {
-	result, err := f.computeIn(leftValue, rightValue)
-	if err != nil {
-		err = fmt.Errorf("%v: %v", f, err)
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(result), nil
-}
-
-func (f *comparisonExpr) notIn(leftValue, rightValue *Value) (*Value, error) {
-	result, err := f.computeIn(leftValue, rightValue)
-	if err != nil {
-		err = fmt.Errorf("%v: %v", f, err)
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(!result), nil
-}
-
-func (f *comparisonExpr) computeLike(leftValue, rightValue *Value) (matched bool, err error) {
-	if leftValue.Type() != String {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to string", f, leftValue.Type())
-		return false, errExternalEvalException(err)
-	}
-	if rightValue.Type() != String {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to string", f, rightValue.Type())
-		return false, errExternalEvalException(err)
-	}
-
-	matched, err = regexp.MatchString(rightValue.StringValue(), leftValue.StringValue())
-	if err != nil {
-		err = fmt.Errorf("%v: %v", f, err)
-		return false, errExternalEvalException(err)
-	}
-
-	return matched, nil
-}
-
-func (f *comparisonExpr) like(leftValue, rightValue *Value) (*Value, error) {
-	result, err := f.computeLike(leftValue, rightValue)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewBool(result), nil
-}
-
-func (f *comparisonExpr) notLike(leftValue, rightValue *Value) (*Value, error) {
-	result, err := f.computeLike(leftValue, rightValue)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewBool(!result), nil
-}
-
-func (f *comparisonExpr) compute(leftValue, rightValue, toValue *Value) (*Value, error) {
-	switch f.operator {
-	case Equal:
-		return f.equal(leftValue, rightValue)
-	case NotEqual:
-		return f.notEqual(leftValue, rightValue)
-	case LessThan:
-		return f.lessThan(leftValue, rightValue)
-	case GreaterThan:
-		return f.greaterThan(leftValue, rightValue)
-	case LessThanEqual:
-		return f.lessThanEqual(leftValue, rightValue)
-	case GreaterThanEqual:
-		return f.greaterThanEqual(leftValue, rightValue)
-	case Between:
-		return f.between(leftValue, rightValue, toValue)
-	case In:
-		return f.in(leftValue, rightValue)
-	case Like:
-		return f.like(leftValue, rightValue)
-	case NotBetween:
-		return f.notBetween(leftValue, rightValue, toValue)
-	case NotIn:
-		return f.notIn(leftValue, rightValue)
-	case NotLike:
-		return f.notLike(leftValue, rightValue)
-	}
-
-	panic(fmt.Errorf("unexpected expression %v", f))
-}
-
-// Call - evaluates this function for given arg values and returns result as Value.
-func (f *comparisonExpr) Eval(record Record) (*Value, error) {
-	leftValue, err := f.left.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	rightValue, err := f.right.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	var toValue *Value
-	if f.to != nil {
-		toValue, err = f.to.Eval(record)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	if f.funcType == aggregateFunction {
-		return nil, nil
-	}
-
-	return f.compute(leftValue, rightValue, toValue)
-}
-
-// AggregateValue - returns aggregated value.
-func (f *comparisonExpr) AggregateValue() (*Value, error) {
-	if f.funcType != aggregateFunction {
-		err := fmt.Errorf("%v is not aggreate expression", f)
-		return nil, errExternalEvalException(err)
-	}
-
-	leftValue, err := f.left.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-
-	rightValue, err := f.right.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-
-	var toValue *Value
-	if f.to != nil {
-		toValue, err = f.to.AggregateValue()
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return f.compute(leftValue, rightValue, toValue)
-}
-
-// Type - returns comparisonFunction or aggregateFunction type.
-func (f *comparisonExpr) Type() Type {
-	return f.funcType
-}
-
-// ReturnType - returns Bool as return type.
-func (f *comparisonExpr) ReturnType() Type {
-	return Bool
-}
-
-// newComparisonExpr - creates new comparison function.
-func newComparisonExpr(operator ComparisonOperator, funcs ...Expr) (*comparisonExpr, error) {
-	funcType := comparisonFunction
-	switch operator {
-	case Equal, NotEqual:
-		if len(funcs) != 2 {
-			panic(fmt.Sprintf("exactly two arguments are expected, but found %v", len(funcs)))
-		}
-
-		left := funcs[0]
-		if !left.ReturnType().isBaseKind() {
-			err := fmt.Errorf("operator %v: left side expression %v evaluate to %v is incompatible for equality check", operator, left, left.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		right := funcs[1]
-		if !right.ReturnType().isBaseKind() {
-			err := fmt.Errorf("operator %v: right side expression %v evaluate to %v is incompatible for equality check", operator, right, right.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		if left.Type() == aggregateFunction || right.Type() == aggregateFunction {
-			funcType = aggregateFunction
-			switch left.Type() {
-			case column, Array, function, arithmeticFunction, comparisonFunction, logicalFunction, record:
-				err := fmt.Errorf("operator %v: left side expression %v return type %v is incompatible for equality check", operator, left, left.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-			switch right.Type() {
-			case column, Array, function, arithmeticFunction, comparisonFunction, logicalFunction, record:
-				err := fmt.Errorf("operator %v: right side expression %v return type %v is incompatible for equality check", operator, right, right.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-		}
-
-		return &comparisonExpr{
-			left:     left,
-			right:    right,
-			operator: operator,
-			funcType: funcType,
-		}, nil
-
-	case LessThan, GreaterThan, LessThanEqual, GreaterThanEqual:
-		if len(funcs) != 2 {
-			panic(fmt.Sprintf("exactly two arguments are expected, but found %v", len(funcs)))
-		}
-
-		left := funcs[0]
-		if !left.ReturnType().isNumberKind() {
-			err := fmt.Errorf("operator %v: left side expression %v evaluate to %v, not number", operator, left, left.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		right := funcs[1]
-		if !right.ReturnType().isNumberKind() {
-			err := fmt.Errorf("operator %v: right side expression %v evaluate to %v; not number", operator, right, right.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		if left.Type() == aggregateFunction || right.Type() == aggregateFunction {
-			funcType = aggregateFunction
-			switch left.Type() {
-			case Int, Float, aggregateFunction:
-			default:
-				err := fmt.Errorf("operator %v: left side expression %v return type %v is incompatible for aggregate evaluation", operator, left, left.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-
-			switch right.Type() {
-			case Int, Float, aggregateFunction:
-			default:
-				err := fmt.Errorf("operator %v: right side expression %v return type %v is incompatible for aggregate evaluation", operator, right, right.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-		}
-
-		return &comparisonExpr{
-			left:     left,
-			right:    right,
-			operator: operator,
-			funcType: funcType,
-		}, nil
-
-	case In, NotIn:
-		if len(funcs) != 2 {
-			panic(fmt.Sprintf("exactly two arguments are expected, but found %v", len(funcs)))
-		}
-
-		left := funcs[0]
-		if !left.ReturnType().isBaseKind() {
-			err := fmt.Errorf("operator %v: left side expression %v evaluate to %v is incompatible for equality check", operator, left, left.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		right := funcs[1]
-		if right.ReturnType() != Array {
-			err := fmt.Errorf("operator %v: right side expression %v evaluate to %v is incompatible for equality check", operator, right, right.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		if left.Type() == aggregateFunction || right.Type() == aggregateFunction {
-			funcType = aggregateFunction
-			switch left.Type() {
-			case column, Array, function, arithmeticFunction, comparisonFunction, logicalFunction, record:
-				err := fmt.Errorf("operator %v: left side expression %v return type %v is incompatible for aggregate evaluation", operator, left, left.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-			switch right.Type() {
-			case Array, aggregateFunction:
-			default:
-				err := fmt.Errorf("operator %v: right side expression %v return type %v is incompatible for aggregate evaluation", operator, right, right.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-		}
-
-		return &comparisonExpr{
-			left:     left,
-			right:    right,
-			operator: operator,
-			funcType: funcType,
-		}, nil
-
-	case Like, NotLike:
-		if len(funcs) != 2 {
-			panic(fmt.Sprintf("exactly two arguments are expected, but found %v", len(funcs)))
-		}
-
-		left := funcs[0]
-		if !left.ReturnType().isStringKind() {
-			err := fmt.Errorf("operator %v: left side expression %v evaluate to %v, not string", operator, left, left.ReturnType())
-			return nil, errLikeInvalidInputs(err)
-		}
-
-		right := funcs[1]
-		if !right.ReturnType().isStringKind() {
-			err := fmt.Errorf("operator %v: right side expression %v evaluate to %v, not string", operator, right, right.ReturnType())
-			return nil, errLikeInvalidInputs(err)
-		}
-
-		if left.Type() == aggregateFunction || right.Type() == aggregateFunction {
-			funcType = aggregateFunction
-			switch left.Type() {
-			case String, aggregateFunction:
-			default:
-				err := fmt.Errorf("operator %v: left side expression %v return type %v is incompatible for aggregate evaluation", operator, left, left.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-			switch right.Type() {
-			case String, aggregateFunction:
-			default:
-				err := fmt.Errorf("operator %v: right side expression %v return type %v is incompatible for aggregate evaluation", operator, right, right.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-		}
-
-		return &comparisonExpr{
-			left:     left,
-			right:    right,
-			operator: operator,
-			funcType: funcType,
-		}, nil
-	case Between, NotBetween:
-		if len(funcs) != 3 {
-			panic(fmt.Sprintf("too many values in funcs %v", funcs))
-		}
-
-		left := funcs[0]
-		if !left.ReturnType().isNumberKind() {
-			err := fmt.Errorf("operator %v: left side expression %v evaluate to %v, not number", operator, left, left.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		from := funcs[1]
-		if !from.ReturnType().isNumberKind() {
-			err := fmt.Errorf("operator %v: from expression %v evaluate to %v, not number", operator, from, from.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		to := funcs[2]
-		if !to.ReturnType().isNumberKind() {
-			err := fmt.Errorf("operator %v: to expression %v evaluate to %v, not number", operator, to, to.ReturnType())
-			return nil, errInvalidDataType(err)
-		}
-
-		if left.Type() == aggregateFunction || from.Type() == aggregateFunction || to.Type() == aggregateFunction {
-			funcType = aggregateFunction
-			switch left.Type() {
-			case Int, Float, aggregateFunction:
-			default:
-				err := fmt.Errorf("operator %v: left side expression %v return type %v is incompatible for aggregate evaluation", operator, left, left.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-			switch from.Type() {
-			case Int, Float, aggregateFunction:
-			default:
-				err := fmt.Errorf("operator %v: from expression %v return type %v is incompatible for aggregate evaluation", operator, from, from.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-			switch to.Type() {
-			case Int, Float, aggregateFunction:
-			default:
-				err := fmt.Errorf("operator %v: to expression %v return type %v is incompatible for aggregate evaluation", operator, to, to.Type())
-				return nil, errUnsupportedSQLOperation(err)
-			}
-		}
-
-		return &comparisonExpr{
-			left:     left,
-			right:    from,
-			to:       to,
-			operator: operator,
-			funcType: funcType,
-		}, nil
-	case IsNull, IsNotNull:
-		if len(funcs) != 1 {
-			panic(fmt.Sprintf("too many values in funcs %v", funcs))
-		}
-
-		if funcs[0].Type() == aggregateFunction {
-			funcType = aggregateFunction
-		}
-
-		if operator == IsNull {
-			operator = Equal
-		} else {
-			operator = NotEqual
-		}
-
-		return &comparisonExpr{
-			left:     funcs[0],
-			right:    newValueExpr(NewNull()),
-			operator: operator,
-			funcType: funcType,
-		}, nil
-	}
-
-	return nil, errParseUnknownOperator(fmt.Errorf("unknown operator %v", operator))
-}

pkg/s3select/sql/errors.go

@@ -43,42 +43,6 @@ func (err *s3Error) Error() string {
 	return err.message
 }
 
-func errUnsupportedSQLStructure(err error) *s3Error {
-	return &s3Error{
-		code:       "UnsupportedSqlStructure",
-		message:    "Encountered an unsupported SQL structure. Check the SQL Reference.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errParseUnsupportedSelect(err error) *s3Error {
-	return &s3Error{
-		code:       "ParseUnsupportedSelect",
-		message:    "The SQL expression contains an unsupported use of SELECT.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errParseAsteriskIsNotAloneInSelectList(err error) *s3Error {
-	return &s3Error{
-		code:       "ParseAsteriskIsNotAloneInSelectList",
-		message:    "Other expressions are not allowed in the SELECT list when '*' is used without dot notation in the SQL expression.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errParseInvalidContextForWildcardInSelectList(err error) *s3Error {
-	return &s3Error{
-		code:       "ParseInvalidContextForWildcardInSelectList",
-		message:    "Invalid use of * in SELECT list in the SQL expression.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
 func errInvalidDataType(err error) *s3Error {
 	return &s3Error{
 		code:       "InvalidDataType",
@@ -88,55 +52,10 @@ func errInvalidDataType(err error) *s3Error {
 	}
 }
 
-func errUnsupportedFunction(err error) *s3Error {
-	return &s3Error{
-		code:       "UnsupportedFunction",
-		message:    "Encountered an unsupported SQL function.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errParseNonUnaryAgregateFunctionCall(err error) *s3Error {
-	return &s3Error{
-		code:       "ParseNonUnaryAgregateFunctionCall",
-		message:    "Only one argument is supported for aggregate functions in the SQL expression.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
 func errIncorrectSQLFunctionArgumentType(err error) *s3Error {
 	return &s3Error{
 		code:       "IncorrectSqlFunctionArgumentType",
-		message:    "Incorrect type of arguments in function call in the SQL expression.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errEvaluatorInvalidArguments(err error) *s3Error {
-	return &s3Error{
-		code:       "EvaluatorInvalidArguments",
-		message:    "Incorrect number of arguments in the function call in the SQL expression.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errUnsupportedSQLOperation(err error) *s3Error {
-	return &s3Error{
-		code:       "UnsupportedSqlOperation",
-		message:    "Encountered an unsupported SQL operation.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errParseUnknownOperator(err error) *s3Error {
-	return &s3Error{
-		code:       "ParseUnknownOperator",
-		message:    "The SQL expression contains an invalid operator.",
+		message:    "Incorrect type of arguments in function call.",
 		statusCode: 400,
 		cause:      err,
 	}
@@ -151,64 +70,28 @@ func errLikeInvalidInputs(err error) *s3Error {
 	}
 }
 
-func errExternalEvalException(err error) *s3Error {
+func errQueryParseFailure(err error) *s3Error {
 	return &s3Error{
-		code:       "ExternalEvalException",
-		message:    "The query cannot be evaluated. Check the file and try again.",
+		code:       "ParseSelectFailure",
+		message:    err.Error(),
 		statusCode: 400,
 		cause:      err,
 	}
 }
 
-func errValueParseFailure(err error) *s3Error {
+func errQueryAnalysisFailure(err error) *s3Error {
 	return &s3Error{
-		code:       "ValueParseFailure",
-		message:    "Time stamp parse failure in the SQL expression.",
+		code:       "InvalidQuery",
+		message:    err.Error(),
 		statusCode: 400,
 		cause:      err,
 	}
 }
 
-func errEvaluatorBindingDoesNotExist(err error) *s3Error {
+func errBadTableName(err error) *s3Error {
 	return &s3Error{
-		code:       "EvaluatorBindingDoesNotExist",
-		message:    "A column name or a path provided does not exist in the SQL expression.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errInternalError(err error) *s3Error {
-	return &s3Error{
-		code:       "InternalError",
-		message:    "Encountered an internal error.",
-		statusCode: 500,
-		cause:      err,
-	}
-}
-
-func errParseInvalidTypeParam(err error) *s3Error {
-	return &s3Error{
-		code:       "ParseInvalidTypeParam",
-		message:    "The SQL expression contains an invalid parameter value.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errParseUnsupportedSyntax(err error) *s3Error {
-	return &s3Error{
-		code:       "ParseUnsupportedSyntax",
-		message:    "The SQL expression contains unsupported syntax.",
-		statusCode: 400,
-		cause:      err,
-	}
-}
-
-func errInvalidKeyPath(err error) *s3Error {
-	return &s3Error{
-		code:       "InvalidKeyPath",
-		message:    "Key path in the SQL expression is invalid.",
+		code:       "BadTableName",
+		message:    "The table name is not supported",
 		statusCode: 400,
 		cause:      err,
 	}

pkg/s3select/sql/evaluate.go

@@ -0,0 +1,360 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"errors"
+	"strings"
+)
+
+var (
+	errInvalidASTNode    = errors.New("invalid AST Node")
+	errExpectedBool      = errors.New("expected bool")
+	errLikeNonStrArg     = errors.New("LIKE clause requires string arguments")
+	errLikeInvalidEscape = errors.New("LIKE clause has invalid ESCAPE character")
+	errNotImplemented    = errors.New("not implemented")
+)
+
+// AST Node Evaluation functions
+//
+// During evaluation, the query is known to be valid, as analysis is
+// complete. The only errors possible are due to value type
+// mismatches, etc.
+//
+// If an aggregation node is present as a descendant (when
+// e.prop.isAggregation is true), we call evalNode on all child nodes,
+// check for errors, but do not perform any combining of the results
+// of child nodes. The final result row is returned after all rows are
+// processed, and the `getAggregate` function is called.
+
+func (e *AliasedExpression) evalNode(r Record) (*Value, error) {
+	return e.Expression.evalNode(r)
+}
+
+func (e *Expression) evalNode(r Record) (*Value, error) {
+	if len(e.And) == 1 {
+		// In this case, result is not required to be boolean
+		// type.
+		return e.And[0].evalNode(r)
+	}
+
+	// Compute OR of conditions
+	result := false
+	for _, ex := range e.And {
+		res, err := ex.evalNode(r)
+		if err != nil {
+			return nil, err
+		}
+		b, ok := res.ToBool()
+		if !ok {
+			return nil, errExpectedBool
+		}
+		result = result || b
+	}
+	return FromBool(result), nil
+}
+
+func (e *AndCondition) evalNode(r Record) (*Value, error) {
+	if len(e.Condition) == 1 {
+		// In this case, result does not have to be boolean
+		return e.Condition[0].evalNode(r)
+	}
+
+	// Compute AND of conditions
+	result := true
+	for _, ex := range e.Condition {
+		res, err := ex.evalNode(r)
+		if err != nil {
+			return nil, err
+		}
+		b, ok := res.ToBool()
+		if !ok {
+			return nil, errExpectedBool
+		}
+		result = result && b
+	}
+	return FromBool(result), nil
+}
+
+func (e *Condition) evalNode(r Record) (*Value, error) {
+	if e.Operand != nil {
+		// In this case, result does not have to be boolean
+		return e.Operand.evalNode(r)
+	}
+
+	// Compute NOT of condition
+	res, err := e.Not.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	b, ok := res.ToBool()
+	if !ok {
+		return nil, errExpectedBool
+	}
+	return FromBool(!b), nil
+}
+
+func (e *ConditionOperand) evalNode(r Record) (*Value, error) {
+	opVal, opErr := e.Operand.evalNode(r)
+	if opErr != nil || e.ConditionRHS == nil {
+		return opVal, opErr
+	}
+
+	// Need to evaluate the ConditionRHS
+	switch {
+	case e.ConditionRHS.Compare != nil:
+		cmpRight, cmpRErr := e.ConditionRHS.Compare.Operand.evalNode(r)
+		if cmpRErr != nil {
+			return nil, cmpRErr
+		}
+
+		b, err := opVal.compareOp(e.ConditionRHS.Compare.Operator, cmpRight)
+		return FromBool(b), err
+
+	case e.ConditionRHS.Between != nil:
+		return e.ConditionRHS.Between.evalBetweenNode(r, opVal)
+
+	case e.ConditionRHS.Like != nil:
+		return e.ConditionRHS.Like.evalLikeNode(r, opVal)
+
+	case e.ConditionRHS.In != nil:
+		return e.ConditionRHS.In.evalInNode(r, opVal)
+
+	default:
+		return nil, errInvalidASTNode
+	}
+}
+
+func (e *Between) evalBetweenNode(r Record, arg *Value) (*Value, error) {
+	stVal, stErr := e.Start.evalNode(r)
+	if stErr != nil {
+		return nil, stErr
+	}
+
+	endVal, endErr := e.End.evalNode(r)
+	if endErr != nil {
+		return nil, endErr
+	}
+
+	part1, err1 := stVal.compareOp(opLte, arg)
+	if err1 != nil {
+		return nil, err1
+	}
+
+	part2, err2 := arg.compareOp(opLte, endVal)
+	if err2 != nil {
+		return nil, err2
+	}
+
+	result := part1 && part2
+	if e.Not {
+		result = !result
+	}
+
+	return FromBool(result), nil
+}
+
+func (e *Like) evalLikeNode(r Record, arg *Value) (*Value, error) {
+	inferTypeAsString(arg)
+
+	s, ok := arg.ToString()
+	if !ok {
+		err := errLikeNonStrArg
+		return nil, errLikeInvalidInputs(err)
+	}
+
+	pattern, err1 := e.Pattern.evalNode(r)
+	if err1 != nil {
+		return nil, err1
+	}
+
+	// Infer pattern as string (in case it is untyped)
+	inferTypeAsString(pattern)
+
+	patternStr, ok := pattern.ToString()
+	if !ok {
+		err := errLikeNonStrArg
+		return nil, errLikeInvalidInputs(err)
+	}
+
+	escape := runeZero
+	if e.EscapeChar != nil {
+		escapeVal, err2 := e.EscapeChar.evalNode(r)
+		if err2 != nil {
+			return nil, err2
+		}
+
+		inferTypeAsString(escapeVal)
+
+		escapeStr, ok := escapeVal.ToString()
+		if !ok {
+			err := errLikeNonStrArg
+			return nil, errLikeInvalidInputs(err)
+		}
+
+		if len([]rune(escapeStr)) > 1 {
+			err := errLikeInvalidEscape
+			return nil, errLikeInvalidInputs(err)
+		}
+	}
+
+	matchResult, err := evalSQLLike(s, patternStr, escape)
+	if err != nil {
+		return nil, err
+	}
+
+	if e.Not {
+		matchResult = !matchResult
+	}
+
+	return FromBool(matchResult), nil
+}
+
+func (e *In) evalInNode(r Record, arg *Value) (*Value, error) {
+	result := false
+	for _, elt := range e.Expressions {
+		eltVal, err := elt.evalNode(r)
+		if err != nil {
+			return nil, err
+		}
+
+		// FIXME: type inference?
+
+		// Types must match.
+		if arg.vType != eltVal.vType {
+			// match failed.
+			continue
+		}
+
+		if arg.value == eltVal.value {
+			result = true
+			break
+		}
+	}
+	return FromBool(result), nil
+}
+
+func (e *Operand) evalNode(r Record) (*Value, error) {
+	lval, lerr := e.Left.evalNode(r)
+	if lerr != nil || len(e.Right) == 0 {
+		return lval, lerr
+	}
+
+	// Process remaining child nodes - result must be
+	// numeric. This AST node is for terms separated by + or -
+	// symbols.
+	for _, rightTerm := range e.Right {
+		op := rightTerm.Op
+		rval, rerr := rightTerm.Right.evalNode(r)
+		if rerr != nil {
+			return nil, rerr
+		}
+		err := lval.arithOp(op, rval)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return lval, nil
+}
+
+func (e *MultOp) evalNode(r Record) (*Value, error) {
+	lval, lerr := e.Left.evalNode(r)
+	if lerr != nil || len(e.Right) == 0 {
+		return lval, lerr
+	}
+
+	// Process other child nodes - result must be numeric. This
+	// AST node is for terms separated by *, / or % symbols.
+	for _, rightTerm := range e.Right {
+		op := rightTerm.Op
+		rval, rerr := rightTerm.Right.evalNode(r)
+		if rerr != nil {
+			return nil, rerr
+		}
+
+		err := lval.arithOp(op, rval)
+		if err != nil {
+			return nil, err
+		}
+	}
+	return lval, nil
+}
+
+func (e *UnaryTerm) evalNode(r Record) (*Value, error) {
+	if e.Negated == nil {
+		return e.Primary.evalNode(r)
+	}
+
+	v, err := e.Negated.Term.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+
+	inferTypeForArithOp(v)
+	v.negate()
+	if v.isNumeric() {
+		return v, nil
+	}
+	return nil, errArithMismatchedTypes
+}
+
+func (e *JSONPath) evalNode(r Record) (*Value, error) {
+	// Strip the table name from the keypath.
+	keypath := e.String()
+	ps := strings.SplitN(keypath, ".", 2)
+	if len(ps) == 2 {
+		keypath = ps[1]
+	}
+	return r.Get(keypath)
+}
+
+func (e *PrimaryTerm) evalNode(r Record) (res *Value, err error) {
+	switch {
+	case e.Value != nil:
+		return e.Value.evalNode(r)
+	case e.JPathExpr != nil:
+		return e.JPathExpr.evalNode(r)
+	case e.SubExpression != nil:
+		return e.SubExpression.evalNode(r)
+	case e.FuncCall != nil:
+		return e.FuncCall.evalNode(r)
+	}
+	return nil, errInvalidASTNode
+}
+
+func (e *FuncExpr) evalNode(r Record) (res *Value, err error) {
+	switch e.getFunctionName() {
+	case aggFnCount, aggFnAvg, aggFnMax, aggFnMin, aggFnSum:
+		return e.getAggregate()
+	default:
+		return e.evalSQLFnNode(r)
+	}
+}
+
+// evalNode on a literal value is independent of the node being an
+// aggregation or a row function - it always returns a value.
+func (e *LitValue) evalNode(_ Record) (res *Value, err error) {
+	switch {
+	case e.Number != nil:
+		return floatToValue(*e.Number), nil
+	case e.String != nil:
+		return FromString(string(*e.String)), nil
+	case e.Boolean != nil:
+		return FromBool(bool(*e.Boolean)), nil
+	}
+	return FromNull(), nil
+}

pkg/s3select/sql/expr.go

@@ -1,160 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2019 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sql
-
-import (
-	"fmt"
-)
-
-// Expr - a SQL expression type.
-type Expr interface {
-	AggregateValue() (*Value, error)
-	Eval(record Record) (*Value, error)
-	ReturnType() Type
-	Type() Type
-}
-
-// aliasExpr - aliases expression by alias.
-type aliasExpr struct {
-	alias string
-	expr  Expr
-}
-
-// String - returns string representation of this expression.
-func (expr *aliasExpr) String() string {
-	return fmt.Sprintf("(%v AS %v)", expr.expr, expr.alias)
-}
-
-// Eval - evaluates underlaying expression for given record and returns evaluated result.
-func (expr *aliasExpr) Eval(record Record) (*Value, error) {
-	return expr.expr.Eval(record)
-}
-
-// AggregateValue - returns aggregated value from underlaying expression.
-func (expr *aliasExpr) AggregateValue() (*Value, error) {
-	return expr.expr.AggregateValue()
-}
-
-// Type - returns underlaying expression type.
-func (expr *aliasExpr) Type() Type {
-	return expr.expr.Type()
-}
-
-// ReturnType - returns underlaying expression's return type.
-func (expr *aliasExpr) ReturnType() Type {
-	return expr.expr.ReturnType()
-}
-
-// newAliasExpr - creates new alias expression.
-func newAliasExpr(alias string, expr Expr) *aliasExpr {
-	return &aliasExpr{alias, expr}
-}
-
-// starExpr - asterisk (*) expression.
-type starExpr struct {
-}
-
-// String - returns string representation of this expression.
-func (expr *starExpr) String() string {
-	return "*"
-}
-
-// Eval - returns given args as map value.
-func (expr *starExpr) Eval(record Record) (*Value, error) {
-	return newRecordValue(record), nil
-}
-
-// AggregateValue - returns nil value.
-func (expr *starExpr) AggregateValue() (*Value, error) {
-	return nil, nil
-}
-
-// Type - returns record type.
-func (expr *starExpr) Type() Type {
-	return record
-}
-
-// ReturnType - returns record as return type.
-func (expr *starExpr) ReturnType() Type {
-	return record
-}
-
-// newStarExpr - returns new asterisk (*) expression.
-func newStarExpr() *starExpr {
-	return &starExpr{}
-}
-
-type valueExpr struct {
-	value *Value
-}
-
-func (expr *valueExpr) String() string {
-	return expr.value.String()
-}
-
-func (expr *valueExpr) Eval(record Record) (*Value, error) {
-	return expr.value, nil
-}
-
-func (expr *valueExpr) AggregateValue() (*Value, error) {
-	return expr.value, nil
-}
-
-func (expr *valueExpr) Type() Type {
-	return expr.value.Type()
-}
-
-func (expr *valueExpr) ReturnType() Type {
-	return expr.value.Type()
-}
-
-func newValueExpr(value *Value) *valueExpr {
-	return &valueExpr{value: value}
-}
-
-type columnExpr struct {
-	name string
-}
-
-func (expr *columnExpr) String() string {
-	return expr.name
-}
-
-func (expr *columnExpr) Eval(record Record) (*Value, error) {
-	value, err := record.Get(expr.name)
-	if err != nil {
-		return nil, errEvaluatorBindingDoesNotExist(err)
-	}
-
-	return value, nil
-}
-
-func (expr *columnExpr) AggregateValue() (*Value, error) {
-	return nil, nil
-}
-
-func (expr *columnExpr) Type() Type {
-	return column
-}
-
-func (expr *columnExpr) ReturnType() Type {
-	return column
-}
-
-func newColumnExpr(columnName string) *columnExpr {
-	return &columnExpr{name: columnName}
-}

pkg/s3select/sql/funceval.go

@@ -0,0 +1,565 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// FuncName - SQL function name.
+type FuncName string
+
+// SQL Function name constants
+const (
+	// Conditionals
+	sqlFnCoalesce FuncName = "COALESCE"
+	sqlFnNullIf   FuncName = "NULLIF"
+
+	// Conversion
+	sqlFnCast FuncName = "CAST"
+
+	// Date and time
+	sqlFnDateAdd     FuncName = "DATE_ADD"
+	sqlFnDateDiff    FuncName = "DATE_DIFF"
+	sqlFnExtract     FuncName = "EXTRACT"
+	sqlFnToString    FuncName = "TO_STRING"
+	sqlFnToTimestamp FuncName = "TO_TIMESTAMP"
+	sqlFnUTCNow      FuncName = "UTCNOW"
+
+	// String
+	sqlFnCharLength      FuncName = "CHAR_LENGTH"
+	sqlFnCharacterLength FuncName = "CHARACTER_LENGTH"
+	sqlFnLower           FuncName = "LOWER"
+	sqlFnSubstring       FuncName = "SUBSTRING"
+	sqlFnTrim            FuncName = "TRIM"
+	sqlFnUpper           FuncName = "UPPER"
+)
+
+var (
+	errUnimplementedCast = errors.New("This cast not yet implemented")
+	errNonStringTrimArg  = errors.New("TRIM() received a non-string argument")
+	errNonTimestampArg   = errors.New("Expected a timestamp argument")
+)
+
+func (e *FuncExpr) getFunctionName() FuncName {
+	switch {
+	case e.SFunc != nil:
+		return FuncName(strings.ToUpper(e.SFunc.FunctionName))
+	case e.Count != nil:
+		return FuncName(aggFnCount)
+	case e.Cast != nil:
+		return sqlFnCast
+	case e.Substring != nil:
+		return sqlFnSubstring
+	case e.Extract != nil:
+		return sqlFnExtract
+	case e.Trim != nil:
+		return sqlFnTrim
+	case e.DateAdd != nil:
+		return sqlFnDateAdd
+	case e.DateDiff != nil:
+		return sqlFnDateDiff
+	default:
+		return ""
+	}
+}
+
+// evalSQLFnNode assumes that the FuncExpr is not an aggregation
+// function.
+func (e *FuncExpr) evalSQLFnNode(r Record) (res *Value, err error) {
+	// Handle functions that have phrase arguments
+	switch e.getFunctionName() {
+	case sqlFnCast:
+		expr := e.Cast.Expr
+		res, err = expr.castTo(r, strings.ToUpper(e.Cast.CastType))
+		return
+
+	case sqlFnSubstring:
+		return handleSQLSubstring(r, e.Substring)
+
+	case sqlFnExtract:
+		return handleSQLExtract(r, e.Extract)
+
+	case sqlFnTrim:
+		return handleSQLTrim(r, e.Trim)
+
+	case sqlFnDateAdd:
+		return handleDateAdd(r, e.DateAdd)
+
+	case sqlFnDateDiff:
+		return handleDateDiff(r, e.DateDiff)
+
+	}
+
+	// For all simple argument functions, we evaluate the arguments here
+	argVals := make([]*Value, len(e.SFunc.ArgsList))
+	for i, arg := range e.SFunc.ArgsList {
+		argVals[i], err = arg.evalNode(r)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	switch e.getFunctionName() {
+	case sqlFnCoalesce:
+		return coalesce(argVals)
+
+	case sqlFnNullIf:
+		return nullif(argVals[0], argVals[1])
+
+	case sqlFnCharLength, sqlFnCharacterLength:
+		return charlen(argVals[0])
+
+	case sqlFnLower:
+		return lowerCase(argVals[0])
+
+	case sqlFnUpper:
+		return upperCase(argVals[0])
+
+	case sqlFnUTCNow:
+		return handleUTCNow()
+
+	case sqlFnToString, sqlFnToTimestamp:
+		// TODO: implement
+		fallthrough
+
+	default:
+		return nil, errNotImplemented
+	}
+}
+
+func coalesce(args []*Value) (res *Value, err error) {
+	for _, arg := range args {
+		if arg.IsNull() {
+			continue
+		}
+		return arg, nil
+	}
+	return FromNull(), nil
+}
+
+func nullif(v1, v2 *Value) (res *Value, err error) {
+	// Handle Null cases
+	if v1.IsNull() || v2.IsNull() {
+		return v1, nil
+	}
+
+	err = inferTypesForCmp(v1, v2)
+	if err != nil {
+		return nil, err
+	}
+
+	atleastOneNumeric := v1.isNumeric() || v2.isNumeric()
+	bothNumeric := v1.isNumeric() && v2.isNumeric()
+	if atleastOneNumeric || !bothNumeric {
+		return v1, nil
+	}
+
+	if v1.vType != v2.vType {
+		return v1, nil
+	}
+
+	cmpResult, cmpErr := v1.compareOp(opEq, v2)
+	if cmpErr != nil {
+		return nil, cmpErr
+	}
+
+	if cmpResult {
+		return FromNull(), nil
+	}
+
+	return v1, nil
+}
+
+func charlen(v *Value) (*Value, error) {
+	inferTypeAsString(v)
+	s, ok := v.ToString()
+	if !ok {
+		err := fmt.Errorf("%s/%s expects a string argument", sqlFnCharLength, sqlFnCharacterLength)
+		return nil, errIncorrectSQLFunctionArgumentType(err)
+	}
+	return FromInt(int64(len(s))), nil
+}
+
+func lowerCase(v *Value) (*Value, error) {
+	inferTypeAsString(v)
+	s, ok := v.ToString()
+	if !ok {
+		err := fmt.Errorf("%s expects a string argument", sqlFnLower)
+		return nil, errIncorrectSQLFunctionArgumentType(err)
+	}
+	return FromString(strings.ToLower(s)), nil
+}
+
+func upperCase(v *Value) (*Value, error) {
+	inferTypeAsString(v)
+	s, ok := v.ToString()
+	if !ok {
+		err := fmt.Errorf("%s expects a string argument", sqlFnUpper)
+		return nil, errIncorrectSQLFunctionArgumentType(err)
+	}
+	return FromString(strings.ToUpper(s)), nil
+}
+
+func handleDateAdd(r Record, d *DateAddFunc) (*Value, error) {
+	q, err := d.Quantity.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeForArithOp(q)
+	qty, ok := q.ToFloat()
+	if !ok {
+		return nil, fmt.Errorf("QUANTITY must be a numeric argument to %s()", sqlFnDateAdd)
+	}
+
+	ts, err := d.Timestamp.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeAsTimestamp(ts)
+	t, ok := ts.ToTimestamp()
+	if !ok {
+		return nil, fmt.Errorf("%s() expects a timestamp argument", sqlFnDateAdd)
+	}
+
+	return dateAdd(strings.ToUpper(d.DatePart), qty, t)
+}
+
+func handleDateDiff(r Record, d *DateDiffFunc) (*Value, error) {
+	tval1, err := d.Timestamp1.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeAsTimestamp(tval1)
+	ts1, ok := tval1.ToTimestamp()
+	if !ok {
+		return nil, fmt.Errorf("%s() expects two timestamp arguments", sqlFnDateDiff)
+	}
+
+	tval2, err := d.Timestamp2.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeAsTimestamp(tval2)
+	ts2, ok := tval2.ToTimestamp()
+	if !ok {
+		return nil, fmt.Errorf("%s() expects two timestamp arguments", sqlFnDateDiff)
+	}
+
+	return dateDiff(strings.ToUpper(d.DatePart), ts1, ts2)
+}
+
+func handleUTCNow() (*Value, error) {
+	return FromTimestamp(time.Now().UTC()), nil
+}
+
+func handleSQLSubstring(r Record, e *SubstringFunc) (val *Value, err error) {
+	// Both forms `SUBSTRING('abc' FROM 2 FOR 1)` and
+	// SUBSTRING('abc', 2, 1) are supported.
+
+	// Evaluate the string argument
+	v1, err := e.Expr.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeAsString(v1)
+	s, ok := v1.ToString()
+	if !ok {
+		err := fmt.Errorf("Incorrect argument type passed to %s", sqlFnSubstring)
+		return nil, errIncorrectSQLFunctionArgumentType(err)
+	}
+
+	// Assemble other arguments
+	arg2, arg3 := e.From, e.For
+	// Check if the second form of substring is being used
+	if e.From == nil {
+		arg2, arg3 = e.Arg2, e.Arg3
+	}
+
+	// Evaluate the FROM argument
+	v2, err := arg2.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+	inferTypeForArithOp(v2)
+	startIdx, ok := v2.ToInt()
+	if !ok {
+		err := fmt.Errorf("Incorrect type for start index argument in %s", sqlFnSubstring)
+		return nil, errIncorrectSQLFunctionArgumentType(err)
+	}
+
+	length := -1
+	// Evaluate the optional FOR argument
+	if arg3 != nil {
+		v3, err := arg3.evalNode(r)
+		if err != nil {
+			return nil, err
+		}
+		inferTypeForArithOp(v3)
+		lenInt, ok := v3.ToInt()
+		if !ok {
+			err := fmt.Errorf("Incorrect type for length argument in %s", sqlFnSubstring)
+			return nil, errIncorrectSQLFunctionArgumentType(err)
+		}
+		length = int(lenInt)
+		if length < 0 {
+			err := fmt.Errorf("Negative length argument in %s", sqlFnSubstring)
+			return nil, errIncorrectSQLFunctionArgumentType(err)
+		}
+	}
+
+	res, err := evalSQLSubstring(s, int(startIdx), length)
+	return FromString(res), err
+}
+
+func handleSQLTrim(r Record, e *TrimFunc) (res *Value, err error) {
+	charsV, cerr := e.TrimChars.evalNode(r)
+	if cerr != nil {
+		return nil, cerr
+	}
+	inferTypeAsString(charsV)
+	chars, ok := charsV.ToString()
+	if !ok {
+		return nil, errNonStringTrimArg
+	}
+
+	fromV, ferr := e.TrimFrom.evalNode(r)
+	if ferr != nil {
+		return nil, ferr
+	}
+	from, ok := fromV.ToString()
+	if !ok {
+		return nil, errNonStringTrimArg
+	}
+
+	result, terr := evalSQLTrim(e.TrimWhere, chars, from)
+	if terr != nil {
+		return nil, terr
+	}
+	return FromString(result), nil
+}
+
+func handleSQLExtract(r Record, e *ExtractFunc) (res *Value, err error) {
+	timeVal, verr := e.From.evalNode(r)
+	if verr != nil {
+		return nil, verr
+	}
+
+	inferTypeAsTimestamp(timeVal)
+
+	t, ok := timeVal.ToTimestamp()
+	if !ok {
+		return nil, errNonTimestampArg
+	}
+
+	return extract(strings.ToUpper(e.Timeword), t)
+}
+
+func errUnsupportedCast(fromType, toType string) error {
+	return fmt.Errorf("Cannot cast from %v to %v", fromType, toType)
+}
+
+func errCastFailure(msg string) error {
+	return fmt.Errorf("Error casting: %s", msg)
+}
+
+// Allowed cast types
+const (
+	castBool      = "BOOL"
+	castInt       = "INT"
+	castInteger   = "INTEGER"
+	castString    = "STRING"
+	castFloat     = "FLOAT"
+	castDecimal   = "DECIMAL"
+	castNumeric   = "NUMERIC"
+	castTimestamp = "TIMESTAMP"
+)
+
+func (e *Expression) castTo(r Record, castType string) (res *Value, err error) {
+	v, err := e.evalNode(r)
+	if err != nil {
+		return nil, err
+	}
+
+	switch castType {
+	case castInt, castInteger:
+		i, err := intCast(v)
+		return FromInt(i), err
+
+	case castFloat:
+		f, err := floatCast(v)
+		return FromFloat(f), err
+
+	case castString:
+		s, err := stringCast(v)
+		return FromString(s), err
+
+	case castTimestamp:
+		t, err := timestampCast(v)
+		return FromTimestamp(t), err
+
+	case castBool:
+		b, err := boolCast(v)
+		return FromBool(b), err
+
+	case castDecimal, castNumeric:
+		fallthrough
+
+	default:
+		return nil, errUnimplementedCast
+	}
+}
+
+func intCast(v *Value) (int64, error) {
+	// This conversion truncates floating point numbers to
+	// integer.
+	strToInt := func(s string) (int64, bool) {
+		i, errI := strconv.ParseInt(s, 10, 64)
+		if errI == nil {
+			return i, true
+		}
+		f, errF := strconv.ParseFloat(s, 64)
+		if errF == nil {
+			return int64(f), true
+		}
+		return 0, false
+	}
+
+	switch v.vType {
+	case typeFloat:
+		// Truncate fractional part
+		return int64(v.value.(float64)), nil
+	case typeInt:
+		return v.value.(int64), nil
+	case typeString:
+		// Parse as number, truncate floating point if
+		// needed.
+		s, _ := v.ToString()
+		res, ok := strToInt(s)
+		if !ok {
+			return 0, errCastFailure("could not parse as int")
+		}
+		return res, nil
+	case typeBytes:
+		// Parse as number, truncate floating point if
+		// needed.
+		b, _ := v.ToBytes()
+		s := string(b)
+		res, ok := strToInt(s)
+		if !ok {
+			return 0, errCastFailure("could not parse as int")
+		}
+		return res, nil
+
+	default:
+		return 0, errUnsupportedCast(v.GetTypeString(), castInt)
+	}
+}
+
+func floatCast(v *Value) (float64, error) {
+	switch v.vType {
+	case typeFloat:
+		return v.value.(float64), nil
+	case typeInt:
+		return float64(v.value.(int64)), nil
+	case typeString:
+		f, err := strconv.ParseFloat(v.value.(string), 64)
+		if err != nil {
+			return 0, errCastFailure("could not parse as float")
+		}
+		return f, nil
+	case typeBytes:
+		b, _ := v.ToBytes()
+		f, err := strconv.ParseFloat(string(b), 64)
+		if err != nil {
+			return 0, errCastFailure("could not parse as float")
+		}
+		return f, nil
+	default:
+		return 0, errUnsupportedCast(v.GetTypeString(), castFloat)
+	}
+}
+
+func stringCast(v *Value) (string, error) {
+	switch v.vType {
+	case typeFloat:
+		f, _ := v.ToFloat()
+		return fmt.Sprintf("%v", f), nil
+	case typeInt:
+		i, _ := v.ToInt()
+		return fmt.Sprintf("%v", i), nil
+	case typeString:
+		s, _ := v.ToString()
+		return s, nil
+	case typeBytes:
+		b, _ := v.ToBytes()
+		return string(b), nil
+	case typeBool:
+		b, _ := v.ToBool()
+		return fmt.Sprintf("%v", b), nil
+	case typeNull:
+		// FIXME: verify this case is correct
+		return fmt.Sprintf("NULL"), nil
+	}
+	// This does not happen
+	return "", nil
+}
+
+func timestampCast(v *Value) (t time.Time, _ error) {
+	switch v.vType {
+	case typeString:
+		s, _ := v.ToString()
+		return parseSQLTimestamp(s)
+	case typeBytes:
+		b, _ := v.ToBytes()
+		return parseSQLTimestamp(string(b))
+	case typeTimestamp:
+		t, _ = v.ToTimestamp()
+		return t, nil
+	default:
+		return t, errCastFailure(fmt.Sprintf("cannot cast %v to Timestamp type", v.GetTypeString()))
+	}
+}
+
+func boolCast(v *Value) (b bool, _ error) {
+	sToB := func(s string) (bool, error) {
+		if s == "true" {
+			return true, nil
+		} else if s == "false" {
+			return false, nil
+		} else {
+			return false, errCastFailure("cannot cast to Bool")
+		}
+	}
+	switch v.vType {
+	case typeBool:
+		b, _ := v.ToBool()
+		return b, nil
+	case typeString:
+		s, _ := v.ToString()
+		return sToB(strings.ToLower(s))
+	case typeBytes:
+		b, _ := v.ToBytes()
+		return sToB(strings.ToLower(string(b)))
+	default:
+		return false, errCastFailure("cannot cast %v to Bool")
+	}
+}

pkg/s3select/sql/funcexpr.go

@@ -1,550 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2019 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sql
-
-import (
-	"fmt"
-	"strings"
-	"time"
-)
-
-// FuncName - SQL function name.
-type FuncName string
-
-const (
-	// Avg - aggregate SQL function AVG().
-	Avg FuncName = "AVG"
-
-	// Count - aggregate SQL function COUNT().
-	Count FuncName = "COUNT"
-
-	// Max - aggregate SQL function MAX().
-	Max FuncName = "MAX"
-
-	// Min - aggregate SQL function MIN().
-	Min FuncName = "MIN"
-
-	// Sum - aggregate SQL function SUM().
-	Sum FuncName = "SUM"
-
-	// Coalesce - conditional SQL function COALESCE().
-	Coalesce FuncName = "COALESCE"
-
-	// NullIf - conditional SQL function NULLIF().
-	NullIf FuncName = "NULLIF"
-
-	// ToTimestamp - conversion SQL function TO_TIMESTAMP().
-	ToTimestamp FuncName = "TO_TIMESTAMP"
-
-	// UTCNow - date SQL function UTCNOW().
-	UTCNow FuncName = "UTCNOW"
-
-	// CharLength - string SQL function CHAR_LENGTH().
-	CharLength FuncName = "CHAR_LENGTH"
-
-	// CharacterLength - string SQL function CHARACTER_LENGTH() same as CHAR_LENGTH().
-	CharacterLength FuncName = "CHARACTER_LENGTH"
-
-	// Lower - string SQL function LOWER().
-	Lower FuncName = "LOWER"
-
-	// Substring - string SQL function SUBSTRING().
-	Substring FuncName = "SUBSTRING"
-
-	// Trim - string SQL function TRIM().
-	Trim FuncName = "TRIM"
-
-	// Upper - string SQL function UPPER().
-	Upper FuncName = "UPPER"
-
-	// DateAdd         FuncName = "DATE_ADD"
-	// DateDiff        FuncName = "DATE_DIFF"
-	// Extract         FuncName = "EXTRACT"
-	// ToString        FuncName = "TO_STRING"
-	// Cast            FuncName = "CAST" // CAST('2007-04-05T14:30Z' AS TIMESTAMP)
-)
-
-func isAggregateFuncName(s string) bool {
-	switch FuncName(s) {
-	case Avg, Count, Max, Min, Sum:
-		return true
-	}
-
-	return false
-}
-
-func callForNumber(f Expr, record Record) (*Value, error) {
-	value, err := f.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	if !value.Type().isNumber() {
-		err := fmt.Errorf("%v evaluated to %v; not to number", f, value.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return value, nil
-}
-
-func callForInt(f Expr, record Record) (*Value, error) {
-	value, err := f.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	if value.Type() != Int {
-		err := fmt.Errorf("%v evaluated to %v; not to int", f, value.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return value, nil
-}
-
-func callForString(f Expr, record Record) (*Value, error) {
-	value, err := f.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	if value.Type() != String {
-		err := fmt.Errorf("%v evaluated to %v; not to string", f, value.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return value, nil
-}
-
-// funcExpr - SQL function.
-type funcExpr struct {
-	args []Expr
-	name FuncName
-
-	sumValue   float64
-	countValue int64
-	maxValue   float64
-	minValue   float64
-}
-
-// String - returns string representation of this function.
-func (f *funcExpr) String() string {
-	var argStrings []string
-	for _, arg := range f.args {
-		argStrings = append(argStrings, fmt.Sprintf("%v", arg))
-	}
-
-	return fmt.Sprintf("%v(%v)", f.name, strings.Join(argStrings, ","))
-}
-
-func (f *funcExpr) sum(record Record) (*Value, error) {
-	value, err := callForNumber(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	f.sumValue += value.FloatValue()
-	f.countValue++
-	return nil, nil
-}
-
-func (f *funcExpr) count(record Record) (*Value, error) {
-	value, err := f.args[0].Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	if value.valueType != Null {
-		f.countValue++
-	}
-
-	return nil, nil
-}
-
-func (f *funcExpr) max(record Record) (*Value, error) {
-	value, err := callForNumber(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	v := value.FloatValue()
-	if v > f.maxValue {
-		f.maxValue = v
-	}
-
-	return nil, nil
-}
-
-func (f *funcExpr) min(record Record) (*Value, error) {
-	value, err := callForNumber(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	v := value.FloatValue()
-	if v < f.minValue {
-		f.minValue = v
-	}
-	return nil, nil
-}
-
-func (f *funcExpr) charLength(record Record) (*Value, error) {
-	value, err := callForString(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewInt(int64(len(value.StringValue()))), nil
-}
-
-func (f *funcExpr) trim(record Record) (*Value, error) {
-	value, err := callForString(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewString(strings.TrimSpace(value.StringValue())), nil
-}
-
-func (f *funcExpr) lower(record Record) (*Value, error) {
-	value, err := callForString(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewString(strings.ToLower(value.StringValue())), nil
-}
-
-func (f *funcExpr) upper(record Record) (*Value, error) {
-	value, err := callForString(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	return NewString(strings.ToUpper(value.StringValue())), nil
-}
-
-func (f *funcExpr) substring(record Record) (*Value, error) {
-	stringValue, err := callForString(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	offsetValue, err := callForInt(f.args[1], record)
-	if err != nil {
-		return nil, err
-	}
-
-	var lengthValue *Value
-	if len(f.args) == 3 {
-		lengthValue, err = callForInt(f.args[2], record)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	value := stringValue.StringValue()
-	offset := int(offsetValue.FloatValue())
-	if offset < 0 || offset > len(value) {
-		offset = 0
-	}
-	length := len(value)
-	if lengthValue != nil {
-		length = int(lengthValue.FloatValue())
-		if length < 0 || length > len(value) {
-			length = len(value)
-		}
-	}
-
-	return NewString(value[offset:length]), nil
-}
-
-func (f *funcExpr) coalesce(record Record) (*Value, error) {
-	values := make([]*Value, len(f.args))
-	var err error
-	for i := range f.args {
-		values[i], err = f.args[i].Eval(record)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	for i := range values {
-		if values[i].Type() != Null {
-			return values[i], nil
-		}
-	}
-
-	return values[0], nil
-}
-
-func (f *funcExpr) nullIf(record Record) (*Value, error) {
-	value1, err := f.args[0].Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	value2, err := f.args[1].Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	result, err := equal(value1, value2)
-	if err != nil {
-		return nil, err
-	}
-
-	if result {
-		return NewNull(), nil
-	}
-
-	return value1, nil
-}
-
-func (f *funcExpr) toTimeStamp(record Record) (*Value, error) {
-	value, err := callForString(f.args[0], record)
-	if err != nil {
-		return nil, err
-	}
-
-	t, err := time.Parse(time.RFC3339, value.StringValue())
-	if err != nil {
-		err := fmt.Errorf("%v: value '%v': %v", f, value, err)
-		return nil, errValueParseFailure(err)
-	}
-
-	return NewTime(t), nil
-}
-
-func (f *funcExpr) utcNow(record Record) (*Value, error) {
-	return NewTime(time.Now().UTC()), nil
-}
-
-// Call - evaluates this function for given arg values and returns result as Value.
-func (f *funcExpr) Eval(record Record) (*Value, error) {
-	switch f.name {
-	case Avg, Sum:
-		return f.sum(record)
-	case Count:
-		return f.count(record)
-	case Max:
-		return f.max(record)
-	case Min:
-		return f.min(record)
-	case Coalesce:
-		return f.coalesce(record)
-	case NullIf:
-		return f.nullIf(record)
-	case ToTimestamp:
-		return f.toTimeStamp(record)
-	case UTCNow:
-		return f.utcNow(record)
-	case Substring:
-		return f.substring(record)
-	case CharLength, CharacterLength:
-		return f.charLength(record)
-	case Trim:
-		return f.trim(record)
-	case Lower:
-		return f.lower(record)
-	case Upper:
-		return f.upper(record)
-	}
-
-	panic(fmt.Sprintf("unsupported aggregate function %v", f.name))
-}
-
-// AggregateValue - returns aggregated value.
-func (f *funcExpr) AggregateValue() (*Value, error) {
-	switch f.name {
-	case Avg:
-		return NewFloat(f.sumValue / float64(f.countValue)), nil
-	case Count:
-		return NewInt(f.countValue), nil
-	case Max:
-		return NewFloat(f.maxValue), nil
-	case Min:
-		return NewFloat(f.minValue), nil
-	case Sum:
-		return NewFloat(f.sumValue), nil
-	}
-
-	err := fmt.Errorf("%v is not aggreate function", f)
-	return nil, errExternalEvalException(err)
-}
-
-// Type - returns Function or aggregateFunction type.
-func (f *funcExpr) Type() Type {
-	switch f.name {
-	case Avg, Count, Max, Min, Sum:
-		return aggregateFunction
-	}
-
-	return function
-}
-
-// ReturnType - returns respective primitive type depending on SQL function.
-func (f *funcExpr) ReturnType() Type {
-	switch f.name {
-	case Avg, Max, Min, Sum:
-		return Float
-	case Count:
-		return Int
-	case CharLength, CharacterLength, Trim, Lower, Upper, Substring:
-		return String
-	case ToTimestamp, UTCNow:
-		return Timestamp
-	case Coalesce, NullIf:
-		return column
-	}
-
-	return function
-}
-
-// newFuncExpr - creates new SQL function.
-func newFuncExpr(funcName FuncName, funcs ...Expr) (*funcExpr, error) {
-	switch funcName {
-	case Avg, Max, Min, Sum:
-		if len(funcs) != 1 {
-			err := fmt.Errorf("%v(): exactly one argument expected; got %v", funcName, len(funcs))
-			return nil, errParseNonUnaryAgregateFunctionCall(err)
-		}
-
-		if !funcs[0].ReturnType().isNumberKind() {
-			err := fmt.Errorf("%v(): argument %v evaluate to %v, not number", funcName, funcs[0], funcs[0].ReturnType())
-			return nil, errIncorrectSQLFunctionArgumentType(err)
-		}
-
-		return &funcExpr{
-			args: funcs,
-			name: funcName,
-		}, nil
-
-	case Count:
-		if len(funcs) != 1 {
-			err := fmt.Errorf("%v(): exactly one argument expected; got %v", funcName, len(funcs))
-			return nil, errParseNonUnaryAgregateFunctionCall(err)
-		}
-
-		switch funcs[0].ReturnType() {
-		case Null, Bool, Int, Float, String, Timestamp, column, record:
-		default:
-			err := fmt.Errorf("%v(): argument %v evaluate to %v is incompatible", funcName, funcs[0], funcs[0].ReturnType())
-			return nil, errIncorrectSQLFunctionArgumentType(err)
-		}
-
-		return &funcExpr{
-			args: funcs,
-			name: funcName,
-		}, nil
-
-	case CharLength, CharacterLength, Trim, Lower, Upper, ToTimestamp:
-		if len(funcs) != 1 {
-			err := fmt.Errorf("%v(): exactly one argument expected; got %v", funcName, len(funcs))
-			return nil, errEvaluatorInvalidArguments(err)
-		}
-
-		if !funcs[0].ReturnType().isStringKind() {
-			err := fmt.Errorf("%v(): argument %v evaluate to %v, not string", funcName, funcs[0], funcs[0].ReturnType())
-			return nil, errIncorrectSQLFunctionArgumentType(err)
-		}
-
-		return &funcExpr{
-			args: funcs,
-			name: funcName,
-		}, nil
-
-	case Coalesce:
-		if len(funcs) < 1 {
-			err := fmt.Errorf("%v(): one or more argument expected; got %v", funcName, len(funcs))
-			return nil, errEvaluatorInvalidArguments(err)
-		}
-
-		for i := range funcs {
-			if !funcs[i].ReturnType().isBaseKind() {
-				err := fmt.Errorf("%v(): argument-%v %v evaluate to %v is incompatible", funcName, i+1, funcs[i], funcs[i].ReturnType())
-				return nil, errIncorrectSQLFunctionArgumentType(err)
-			}
-		}
-
-		return &funcExpr{
-			args: funcs,
-			name: funcName,
-		}, nil
-
-	case NullIf:
-		if len(funcs) != 2 {
-			err := fmt.Errorf("%v(): exactly two arguments expected; got %v", funcName, len(funcs))
-			return nil, errEvaluatorInvalidArguments(err)
-		}
-
-		if !funcs[0].ReturnType().isBaseKind() {
-			err := fmt.Errorf("%v(): argument-1 %v evaluate to %v is incompatible", funcName, funcs[0], funcs[0].ReturnType())
-			return nil, errIncorrectSQLFunctionArgumentType(err)
-		}
-
-		if !funcs[1].ReturnType().isBaseKind() {
-			err := fmt.Errorf("%v(): argument-2 %v evaluate to %v is incompatible", funcName, funcs[1], funcs[1].ReturnType())
-			return nil, errIncorrectSQLFunctionArgumentType(err)
-		}
-
-		return &funcExpr{
-			args: funcs,
-			name: funcName,
-		}, nil
-
-	case UTCNow:
-		if len(funcs) != 0 {
-			err := fmt.Errorf("%v(): no argument expected; got %v", funcName, len(funcs))
-			return nil, errEvaluatorInvalidArguments(err)
-		}
-
-		return &funcExpr{
-			args: funcs,
-			name: funcName,
-		}, nil
-
-	case Substring:
-		if len(funcs) < 2 || len(funcs) > 3 {
-			err := fmt.Errorf("%v(): exactly two or three arguments expected; got %v", funcName, len(funcs))
-			return nil, errEvaluatorInvalidArguments(err)
-		}
-
-		if !funcs[0].ReturnType().isStringKind() {
-			err := fmt.Errorf("%v(): argument-1 %v evaluate to %v, not string", funcName, funcs[0], funcs[0].ReturnType())
-			return nil, errIncorrectSQLFunctionArgumentType(err)
-		}
-
-		if !funcs[1].ReturnType().isIntKind() {
-			err := fmt.Errorf("%v(): argument-2 %v evaluate to %v, not int", funcName, funcs[1], funcs[1].ReturnType())
-			return nil, errIncorrectSQLFunctionArgumentType(err)
-		}
-
-		if len(funcs) > 2 {
-			if !funcs[2].ReturnType().isIntKind() {
-				err := fmt.Errorf("%v(): argument-3 %v evaluate to %v, not int", funcName, funcs[2], funcs[2].ReturnType())
-				return nil, errIncorrectSQLFunctionArgumentType(err)
-			}
-		}
-
-		return &funcExpr{
-			args: funcs,
-			name: funcName,
-		}, nil
-	}
-
-	return nil, errUnsupportedFunction(fmt.Errorf("unknown function name %v", funcName))
-}

pkg/s3select/sql/logicalexpr.go

@@ -1,336 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2019 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sql
-
-import "fmt"
-
-// andExpr - logical AND function.
-type andExpr struct {
-	left     Expr
-	right    Expr
-	funcType Type
-}
-
-// String - returns string representation of this function.
-func (f *andExpr) String() string {
-	return fmt.Sprintf("(%v AND %v)", f.left, f.right)
-}
-
-// Call - evaluates this function for given arg values and returns result as Value.
-func (f *andExpr) Eval(record Record) (*Value, error) {
-	leftValue, err := f.left.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	if f.funcType == aggregateFunction {
-		_, err = f.right.Eval(record)
-		return nil, err
-	}
-
-	if leftValue.Type() != Bool {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to bool", f, leftValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	if !leftValue.BoolValue() {
-		return leftValue, nil
-	}
-
-	rightValue, err := f.right.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-	if rightValue.Type() != Bool {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to bool", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return rightValue, nil
-}
-
-// AggregateValue - returns aggregated value.
-func (f *andExpr) AggregateValue() (*Value, error) {
-	if f.funcType != aggregateFunction {
-		err := fmt.Errorf("%v is not aggreate expression", f)
-		return nil, errExternalEvalException(err)
-	}
-
-	leftValue, err := f.left.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-	if leftValue.Type() != Bool {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to bool", f, leftValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	if !leftValue.BoolValue() {
-		return leftValue, nil
-	}
-
-	rightValue, err := f.right.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-	if rightValue.Type() != Bool {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to bool", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return rightValue, nil
-}
-
-// Type - returns logicalFunction or aggregateFunction type.
-func (f *andExpr) Type() Type {
-	return f.funcType
-}
-
-// ReturnType - returns Bool as return type.
-func (f *andExpr) ReturnType() Type {
-	return Bool
-}
-
-// newAndExpr - creates new AND logical function.
-func newAndExpr(left, right Expr) (*andExpr, error) {
-	if !left.ReturnType().isBoolKind() {
-		err := fmt.Errorf("operator AND: left side expression %v evaluate to %v, not bool", left, left.ReturnType())
-		return nil, errInvalidDataType(err)
-	}
-
-	if !right.ReturnType().isBoolKind() {
-		err := fmt.Errorf("operator AND: right side expression %v evaluate to %v; not bool", right, right.ReturnType())
-		return nil, errInvalidDataType(err)
-	}
-
-	funcType := logicalFunction
-	if left.Type() == aggregateFunction || right.Type() == aggregateFunction {
-		funcType = aggregateFunction
-		if left.Type() == column {
-			err := fmt.Errorf("operator AND: left side expression %v return type %v is incompatible for aggregate evaluation", left, left.Type())
-			return nil, errUnsupportedSQLOperation(err)
-		}
-
-		if right.Type() == column {
-			err := fmt.Errorf("operator AND: right side expression %v return type %v is incompatible for aggregate evaluation", right, right.Type())
-			return nil, errUnsupportedSQLOperation(err)
-		}
-	}
-
-	return &andExpr{
-		left:     left,
-		right:    right,
-		funcType: funcType,
-	}, nil
-}
-
-// orExpr - logical OR function.
-type orExpr struct {
-	left     Expr
-	right    Expr
-	funcType Type
-}
-
-// String - returns string representation of this function.
-func (f *orExpr) String() string {
-	return fmt.Sprintf("(%v OR %v)", f.left, f.right)
-}
-
-// Call - evaluates this function for given arg values and returns result as Value.
-func (f *orExpr) Eval(record Record) (*Value, error) {
-	leftValue, err := f.left.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	if f.funcType == aggregateFunction {
-		_, err = f.right.Eval(record)
-		return nil, err
-	}
-
-	if leftValue.Type() != Bool {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to bool", f, leftValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	if leftValue.BoolValue() {
-		return leftValue, nil
-	}
-
-	rightValue, err := f.right.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-	if rightValue.Type() != Bool {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to bool", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return rightValue, nil
-}
-
-// AggregateValue - returns aggregated value.
-func (f *orExpr) AggregateValue() (*Value, error) {
-	if f.funcType != aggregateFunction {
-		err := fmt.Errorf("%v is not aggreate expression", f)
-		return nil, errExternalEvalException(err)
-	}
-
-	leftValue, err := f.left.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-	if leftValue.Type() != Bool {
-		err := fmt.Errorf("%v: left side expression evaluated to %v; not to bool", f, leftValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	if leftValue.BoolValue() {
-		return leftValue, nil
-	}
-
-	rightValue, err := f.right.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-	if rightValue.Type() != Bool {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to bool", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return rightValue, nil
-}
-
-// Type - returns logicalFunction or aggregateFunction type.
-func (f *orExpr) Type() Type {
-	return f.funcType
-}
-
-// ReturnType - returns Bool as return type.
-func (f *orExpr) ReturnType() Type {
-	return Bool
-}
-
-// newOrExpr - creates new OR logical function.
-func newOrExpr(left, right Expr) (*orExpr, error) {
-	if !left.ReturnType().isBoolKind() {
-		err := fmt.Errorf("operator OR: left side expression %v evaluate to %v, not bool", left, left.ReturnType())
-		return nil, errInvalidDataType(err)
-	}
-
-	if !right.ReturnType().isBoolKind() {
-		err := fmt.Errorf("operator OR: right side expression %v evaluate to %v; not bool", right, right.ReturnType())
-		return nil, errInvalidDataType(err)
-	}
-
-	funcType := logicalFunction
-	if left.Type() == aggregateFunction || right.Type() == aggregateFunction {
-		funcType = aggregateFunction
-		if left.Type() == column {
-			err := fmt.Errorf("operator OR: left side expression %v return type %v is incompatible for aggregate evaluation", left, left.Type())
-			return nil, errUnsupportedSQLOperation(err)
-		}
-
-		if right.Type() == column {
-			err := fmt.Errorf("operator OR: right side expression %v return type %v is incompatible for aggregate evaluation", right, right.Type())
-			return nil, errUnsupportedSQLOperation(err)
-		}
-	}
-
-	return &orExpr{
-		left:     left,
-		right:    right,
-		funcType: funcType,
-	}, nil
-}
-
-// notExpr - logical NOT function.
-type notExpr struct {
-	right    Expr
-	funcType Type
-}
-
-// String - returns string representation of this function.
-func (f *notExpr) String() string {
-	return fmt.Sprintf("(%v)", f.right)
-}
-
-// Call - evaluates this function for given arg values and returns result as Value.
-func (f *notExpr) Eval(record Record) (*Value, error) {
-	rightValue, err := f.right.Eval(record)
-	if err != nil {
-		return nil, err
-	}
-
-	if f.funcType == aggregateFunction {
-		return nil, nil
-	}
-
-	if rightValue.Type() != Bool {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to bool", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(!rightValue.BoolValue()), nil
-}
-
-// AggregateValue - returns aggregated value.
-func (f *notExpr) AggregateValue() (*Value, error) {
-	if f.funcType != aggregateFunction {
-		err := fmt.Errorf("%v is not aggreate expression", f)
-		return nil, errExternalEvalException(err)
-	}
-
-	rightValue, err := f.right.AggregateValue()
-	if err != nil {
-		return nil, err
-	}
-	if rightValue.Type() != Bool {
-		err := fmt.Errorf("%v: right side expression evaluated to %v; not to bool", f, rightValue.Type())
-		return nil, errExternalEvalException(err)
-	}
-
-	return NewBool(!rightValue.BoolValue()), nil
-}
-
-// Type - returns logicalFunction or aggregateFunction type.
-func (f *notExpr) Type() Type {
-	return f.funcType
-}
-
-// ReturnType - returns Bool as return type.
-func (f *notExpr) ReturnType() Type {
-	return Bool
-}
-
-// newNotExpr - creates new NOT logical function.
-func newNotExpr(right Expr) (*notExpr, error) {
-	if !right.ReturnType().isBoolKind() {
-		err := fmt.Errorf("operator NOT: right side expression %v evaluate to %v; not bool", right, right.ReturnType())
-		return nil, errInvalidDataType(err)
-	}
-
-	funcType := logicalFunction
-	if right.Type() == aggregateFunction {
-		funcType = aggregateFunction
-	}
-
-	return &notExpr{
-		right:    right,
-		funcType: funcType,
-	}, nil
-}

pkg/s3select/sql/parser.go

@@ -0,0 +1,345 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"strings"
+
+	"github.com/alecthomas/participle"
+	"github.com/alecthomas/participle/lexer"
+)
+
+// Types with custom Capture interface for parsing
+
+// Boolean is a type for a parsed Boolean literal
+type Boolean bool
+
+// Capture interface used by participle
+func (b *Boolean) Capture(values []string) error {
+	*b = strings.ToLower(values[0]) == "true"
+	return nil
+}
+
+// LiteralString is a type for parsed SQL string literals
+type LiteralString string
+
+// Capture interface used by participle
+func (ls *LiteralString) Capture(values []string) error {
+	// Remove enclosing single quote
+	n := len(values[0])
+	r := values[0][1 : n-1]
+	// Translate doubled quotes
+	*ls = LiteralString(strings.Replace(r, "''", "'", -1))
+	return nil
+}
+
+// ObjectKey is a type for parsed strings occurring in key paths
+type ObjectKey struct {
+	Lit *LiteralString `parser:" \"[\" @LitString \"]\""`
+	ID  *Identifier    `parser:"| \".\" @@"`
+}
+
+// QuotedIdentifier is a type for parsed strings that are double
+// quoted.
+type QuotedIdentifier string
+
+// Capture inferface used by participle
+func (qi *QuotedIdentifier) Capture(values []string) error {
+	// Remove enclosing quotes
+	n := len(values[0])
+	r := values[0][1 : n-1]
+
+	// Translate doubled quotes
+	*qi = QuotedIdentifier(strings.Replace(r, `""`, `"`, -1))
+	return nil
+}
+
+// Types representing AST of SQL statement. Only SELECT is supported.
+
+// Select is the top level AST node type
+type Select struct {
+	Expression *SelectExpression `parser:"\"SELECT\" @@"`
+	From       *TableExpression  `parser:"\"FROM\" @@"`
+	Where      *Expression       `parser:"[ \"WHERE\" @@ ]"`
+	Limit      *LitValue         `parser:"[ \"LIMIT\" @@ ]"`
+}
+
+// SelectExpression represents the items requested in the select
+// statement
+type SelectExpression struct {
+	All         bool                 `parser:"  @\"*\""`
+	Expressions []*AliasedExpression `parser:"| @@ { \",\" @@ }"`
+
+	prop qProp
+}
+
+// TableExpression represents the FROM clause
+type TableExpression struct {
+	Table *JSONPath `parser:"@@"`
+	As    string    `parser:"( \"AS\"? @Ident )?"`
+}
+
+// JSONPathElement represents a keypath component
+type JSONPathElement struct {
+	Key            *ObjectKey `parser:"  @@"`                  // ['name'] and .name forms
+	Index          *uint64    `parser:"| \"[\" @Number \"]\""` // [3] form
+	ObjectWildcard bool       `parser:"| @\".*\""`             // .* form
+	ArrayWildcard  bool       `parser:"| @\"[*]\""`            // [*] form
+}
+
+// JSONPath represents a keypath
+type JSONPath struct {
+	BaseKey  *Identifier        `parser:" @@"`
+	PathExpr []*JSONPathElement `parser:"(@@)*"`
+}
+
+// AliasedExpression is an expression that can be optionally named
+type AliasedExpression struct {
+	Expression *Expression `parser:"@@"`
+	As         string      `parser:"[ \"AS\" @Ident ]"`
+}
+
+// Grammar for Expression
+//
+// Expression          → AndCondition ("OR" AndCondition)*
+// AndCondition        → Condition ("AND" Condition)*
+// Condition           → "NOT" Condition | ConditionExpression
+// ConditionExpression → ValueExpression ("=" | "<>" | "<=" | ">=" | "<" | ">") ValueExpression
+//                     | ValueExpression "LIKE" ValueExpression ("ESCAPE" LitString)?
+//                     | ValueExpression ("NOT"? "BETWEEN" ValueExpression "AND" ValueExpression)
+//                     | ValueExpression "IN" "(" Expression ("," Expression)* ")"
+//                     | ValueExpression
+// ValueExpression     → Operand
+//
+// Operand grammar follows below
+
+// Expression represents a logical disjunction of clauses
+type Expression struct {
+	And []*AndCondition `parser:"@@ ( \"OR\" @@ )*"`
+}
+
+// AndCondition represents logical conjunction of clauses
+type AndCondition struct {
+	Condition []*Condition `parser:"@@ ( \"AND\" @@ )*"`
+}
+
+// Condition represents a negation or a condition operand
+type Condition struct {
+	Operand *ConditionOperand `parser:"  @@"`
+	Not     *Condition        `parser:"| \"NOT\" @@"`
+}
+
+// ConditionOperand is a operand followed by an an optional operation
+// expression
+type ConditionOperand struct {
+	Operand      *Operand      `parser:"@@"`
+	ConditionRHS *ConditionRHS `parser:"@@?"`
+}
+
+// ConditionRHS represents the right-hand-side of Compare, Between, In
+// or Like expressions.
+type ConditionRHS struct {
+	Compare *Compare `parser:"  @@"`
+	Between *Between `parser:"| @@"`
+	In      *In      `parser:"| \"IN\" \"(\" @@ \")\""`
+	Like    *Like    `parser:"| @@"`
+}
+
+// Compare represents the RHS of a comparison expression
+type Compare struct {
+	Operator string   `parser:"@( \"<>\" | \"<=\" | \">=\" | \"=\" | \"<\" | \">\" | \"!=\" )"`
+	Operand  *Operand `parser:"  @@"`
+}
+
+// Like represents the RHS of a LIKE expression
+type Like struct {
+	Not        bool     `parser:" @\"NOT\"? "`
+	Pattern    *Operand `parser:" \"LIKE\" @@ "`
+	EscapeChar *Operand `parser:" (\"ESCAPE\" @@)? "`
+}
+
+// Between represents the RHS of a BETWEEN expression
+type Between struct {
+	Not   bool     `parser:" @\"NOT\"? "`
+	Start *Operand `parser:" \"BETWEEN\" @@ "`
+	End   *Operand `parser:" \"AND\" @@ "`
+}
+
+// In represents the RHS of an IN expression
+type In struct {
+	Expressions []*Expression `parser:"@@ ( \",\" @@ )*"`
+}
+
+// Grammar for Operand:
+//
+// operand → multOp ( ("-" | "+") multOp )*
+// multOp  → unary ( ("/" | "*" | "%") unary )*
+// unary   → "-" unary | primary
+// primary → Value | Variable | "(" expression ")"
+//
+
+// An Operand is a single term followed by an optional sequence of
+// terms separated by +/-
+type Operand struct {
+	Left  *MultOp     `parser:"@@"`
+	Right []*OpFactor `parser:"(@@)*"`
+}
+
+// OpFactor represents the right-side of a +/- operation.
+type OpFactor struct {
+	Op    string  `parser:"@(\"+\" | \"-\")"`
+	Right *MultOp `parser:"@@"`
+}
+
+// MultOp represents a single term followed by an optional sequence of
+// terms separated by *, / or % operators.
+type MultOp struct {
+	Left  *UnaryTerm     `parser:"@@"`
+	Right []*OpUnaryTerm `parser:"(@@)*"`
+}
+
+// OpUnaryTerm represents the right side of *, / or % binary operations.
+type OpUnaryTerm struct {
+	Op    string     `parser:"@(\"*\" | \"/\" | \"%\")"`
+	Right *UnaryTerm `parser:"@@"`
+}
+
+// UnaryTerm represents a single negated term or a primary term
+type UnaryTerm struct {
+	Negated *NegatedTerm `parser:"  @@"`
+	Primary *PrimaryTerm `parser:"| @@"`
+}
+
+// NegatedTerm has a leading minus sign.
+type NegatedTerm struct {
+	Term *PrimaryTerm `parser:"\"-\" @@"`
+}
+
+// PrimaryTerm represents a Value, Path expression, a Sub-expression
+// or a function call.
+type PrimaryTerm struct {
+	Value         *LitValue   `parser:"  @@"`
+	JPathExpr     *JSONPath   `parser:"| @@"`
+	SubExpression *Expression `parser:"| \"(\" @@ \")\""`
+	// Include function expressions here.
+	FuncCall *FuncExpr `parser:"| @@"`
+}
+
+// FuncExpr represents a function call
+type FuncExpr struct {
+	SFunc     *SimpleArgFunc `parser:"  @@"`
+	Count     *CountFunc     `parser:"| @@"`
+	Cast      *CastFunc      `parser:"| @@"`
+	Substring *SubstringFunc `parser:"| @@"`
+	Extract   *ExtractFunc   `parser:"| @@"`
+	Trim      *TrimFunc      `parser:"| @@"`
+	DateAdd   *DateAddFunc   `parser:"| @@"`
+	DateDiff  *DateDiffFunc  `parser:"| @@"`
+
+	// Used during evaluation for aggregation funcs
+	aggregate *aggVal
+}
+
+// SimpleArgFunc represents functions with simple expression
+// arguments.
+type SimpleArgFunc struct {
+	FunctionName string `parser:" @(\"AVG\" | \"MAX\" | \"MIN\" | \"SUM\" |  \"COALESCE\" | \"NULLIF\" | \"TO_STRING\" | \"TO_TIMESTAMP\" | \"UTCNOW\" | \"CHAR_LENGTH\" | \"CHARACTER_LENGTH\" | \"LOWER\" | \"UPPER\") "`
+
+	ArgsList []*Expression `parser:"\"(\" (@@ (\",\" @@)*)?\")\""`
+}
+
+// CountFunc represents the COUNT sql function
+type CountFunc struct {
+	StarArg bool        `parser:" \"COUNT\" \"(\" ( @\"*\"?"`
+	ExprArg *Expression `parser:" @@? )! \")\""`
+}
+
+// CastFunc represents CAST sql function
+type CastFunc struct {
+	Expr     *Expression `parser:" \"CAST\" \"(\" @@ "`
+	CastType string      `parser:" \"AS\" @(\"BOOL\" | \"INT\" | \"INTEGER\" | \"STRING\" | \"FLOAT\" | \"DECIMAL\" | \"NUMERIC\" | \"TIMESTAMP\") \")\" "`
+}
+
+// SubstringFunc represents SUBSTRING sql function
+type SubstringFunc struct {
+	Expr *PrimaryTerm `parser:" \"SUBSTRING\" \"(\" @@ "`
+	From *Operand     `parser:" ( \"FROM\" @@ "`
+	For  *Operand     `parser:"   (\"FOR\" @@)? \")\" "`
+	Arg2 *Operand     `parser:" | \",\" @@ "`
+	Arg3 *Operand     `parser:"   (\",\" @@)? \")\" )"`
+}
+
+// ExtractFunc represents EXTRACT sql function
+type ExtractFunc struct {
+	Timeword string       `parser:" \"EXTRACT\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword | \"TIMEZONE_HOUR\":Timeword | \"TIMEZONE_MINUTE\":Timeword ) "`
+	From     *PrimaryTerm `parser:" \"FROM\" @@ \")\" "`
+}
+
+// TrimFunc represents TRIM sql function
+type TrimFunc struct {
+	TrimWhere *string      `parser:" \"TRIM\" \"(\" ( @( \"LEADING\" | \"TRAILING\" | \"BOTH\" ) "`
+	TrimChars *PrimaryTerm `parser:"             @@?  "`
+	TrimFrom  *PrimaryTerm `parser:"             \"FROM\" )? @@ \")\" "`
+}
+
+// DateAddFunc represents the DATE_ADD function
+type DateAddFunc struct {
+	DatePart  string       `parser:" \"DATE_ADD\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword ) \",\""`
+	Quantity  *Operand     `parser:" @@ \",\""`
+	Timestamp *PrimaryTerm `parser:" @@ \")\""`
+}
+
+// DateDiffFunc represents the DATE_DIFF function
+type DateDiffFunc struct {
+	DatePart   string       `parser:" \"DATE_DIFF\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword ) \",\" "`
+	Timestamp1 *PrimaryTerm `parser:" @@ \",\" "`
+	Timestamp2 *PrimaryTerm `parser:" @@ \")\" "`
+}
+
+// LitValue represents a literal value parsed from the sql
+type LitValue struct {
+	Number  *float64       `parser:"(  @Number"`
+	String  *LiteralString `parser:" | @LitString"`
+	Boolean *Boolean       `parser:" | @(\"TRUE\" | \"FALSE\")"`
+	Null    bool           `parser:" | @\"NULL\")"`
+}
+
+// Identifier represents a parsed identifier
+type Identifier struct {
+	Unquoted *string           `parser:"  @Ident"`
+	Quoted   *QuotedIdentifier `parser:"| @QuotIdent"`
+}
+
+var (
+	sqlLexer = lexer.Must(lexer.Regexp(`(\s+)` +
+		`|(?P<Timeword>(?i)\b(?:YEAR|MONTH|DAY|HOUR|MINUTE|SECOND|TIMEZONE_HOUR|TIMEZONE_MINUTE)\b)` +
+		`|(?P<Keyword>(?i)\b(?:SELECT|FROM|TOP|DISTINCT|ALL|WHERE|GROUP|BY|HAVING|UNION|MINUS|EXCEPT|INTERSECT|ORDER|LIMIT|OFFSET|TRUE|FALSE|NULL|IS|NOT|ANY|SOME|BETWEEN|AND|OR|LIKE|ESCAPE|AS|IN|BOOL|INT|INTEGER|STRING|FLOAT|DECIMAL|NUMERIC|TIMESTAMP|AVG|COUNT|MAX|MIN|SUM|COALESCE|NULLIF|CAST|DATE_ADD|DATE_DIFF|EXTRACT|TO_STRING|TO_TIMESTAMP|UTCNOW|CHAR_LENGTH|CHARACTER_LENGTH|LOWER|SUBSTRING|TRIM|UPPER|LEADING|TRAILING|BOTH|FOR)\b)` +
+		`|(?P<Ident>[a-zA-Z_][a-zA-Z0-9_]*)` +
+		`|(?P<QuotIdent>"([^"]*("")?)*")` +
+		`|(?P<Number>\d*\.?\d+([eE][-+]?\d+)?)` +
+		`|(?P<LitString>'([^']*('')?)*')` +
+		`|(?P<Operators><>|!=|<=|>=|\.\*|\[\*\]|[-+*/%,.()=<>\[\]])`,
+	))
+
+	// SQLParser is used to parse SQL statements
+	SQLParser = participle.MustBuild(
+		&Select{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+		participle.CaseInsensitive("Timeword"),
+	)
+)

pkg/s3select/sql/parser_test.go

@@ -0,0 +1,383 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/alecthomas/participle"
+	"github.com/alecthomas/participle/lexer"
+)
+
+func TestJSONPathElement(t *testing.T) {
+	p := participle.MustBuild(
+		&JSONPathElement{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	j := JSONPathElement{}
+	cases := []string{
+		// Key
+		"['name']", ".name", `."name"`,
+
+		// Index
+		"[2]", "[0]", "[100]",
+
+		// Object wilcard
+		".*",
+
+		// array wildcard
+		"[*]",
+	}
+	for i, tc := range cases {
+		err := p.ParseString(tc, &j)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		// repr.Println(j, repr.Indent("  "), repr.OmitEmpty(true))
+	}
+}
+
+func TestJSONPath(t *testing.T) {
+	p := participle.MustBuild(
+		&JSONPath{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	j := JSONPath{}
+	cases := []string{
+		"S3Object",
+		"S3Object.id",
+		"S3Object.book.title",
+		"S3Object.id[1]",
+		"S3Object.id['abc']",
+		"S3Object.id['ab']",
+		"S3Object.words.*.id",
+		"S3Object.words.name[*].val",
+		"S3Object.words.name[*].val[*]",
+		"S3Object.words.name[*].val.*",
+	}
+	for i, tc := range cases {
+		err := p.ParseString(tc, &j)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		// repr.Println(j, repr.Indent("  "), repr.OmitEmpty(true))
+	}
+
+}
+
+func TestIdentifierParsing(t *testing.T) {
+	p := participle.MustBuild(
+		&Identifier{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	id := Identifier{}
+	validCases := []string{
+		"a",
+		"_a",
+		"abc_a",
+		"a2",
+		`"abc"`,
+		`"abc\a""ac"`,
+	}
+	for i, tc := range validCases {
+		err := p.ParseString(tc, &id)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		// repr.Println(id, repr.Indent("  "), repr.OmitEmpty(true))
+	}
+
+	invalidCases := []string{
+		"+a",
+		"-a",
+		"1a",
+		`"ab`,
+		`abc"`,
+		`aa""a`,
+		`"a"a"`,
+	}
+	for i, tc := range invalidCases {
+		err := p.ParseString(tc, &id)
+		if err == nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		// fmt.Println(tc, err)
+	}
+}
+
+func TestLiteralStringParsing(t *testing.T) {
+	var k ObjectKey
+	p := participle.MustBuild(
+		&ObjectKey{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	validCases := []string{
+		"['abc']",
+		"['ab''c']",
+		"['a''b''c']",
+		"['abc-x_1##@(*&(#*))/\\']",
+	}
+	for i, tc := range validCases {
+		err := p.ParseString(tc, &k)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		if string(*k.Lit) == "" {
+			t.Fatalf("Incorrect parse %#v", k)
+		}
+		// repr.Println(k, repr.Indent("  "), repr.OmitEmpty(true))
+	}
+
+	invalidCases := []string{
+		"['abc'']",
+		"['-abc'sc']",
+		"[abc']",
+		"['ac]",
+	}
+	for i, tc := range invalidCases {
+		err := p.ParseString(tc, &k)
+		if err == nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		// fmt.Println(tc, err)
+	}
+}
+
+func TestFunctionParsing(t *testing.T) {
+	var fex FuncExpr
+	p := participle.MustBuild(
+		&FuncExpr{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+		participle.CaseInsensitive("Timeword"),
+	)
+
+	validCases := []string{
+		"count(*)",
+		"sum(2 + s.id)",
+		"sum(t)",
+		"avg(s.id[1])",
+		"coalesce(s.id[1], 2, 2 + 3)",
+
+		"cast(s as string)",
+		"cast(s AS INT)",
+		"cast(s as DECIMAL)",
+		"extract(YEAR from '2018-01-09')",
+		"extract(month from '2018-01-09')",
+
+		"extract(hour from '2018-01-09')",
+		"extract(day from '2018-01-09')",
+		"substring('abcd' from 2 for 2)",
+		"substring('abcd' from 2)",
+		"substring('abcd' , 2 , 2)",
+
+		"substring('abcd' , 22 )",
+		"trim('  aab  ')",
+		"trim(leading from '  aab  ')",
+		"trim(trailing from '  aab  ')",
+		"trim(both from '  aab  ')",
+
+		"trim(both '12' from '  aab  ')",
+		"trim(leading '12' from '  aab  ')",
+		"trim(trailing '12' from '  aab  ')",
+		"count(23)",
+	}
+	for i, tc := range validCases {
+		err := p.ParseString(tc, &fex)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+		// repr.Println(fex, repr.Indent("  "), repr.OmitEmpty(true))
+	}
+}
+
+func TestSqlLexer(t *testing.T) {
+	// s := bytes.NewBuffer([]byte("s.['name'].*.[*].abc.[\"abc\"]"))
+	s := bytes.NewBuffer([]byte("S3Object.words.*.id"))
+	// s := bytes.NewBuffer([]byte("COUNT(Id)"))
+	lex, err := sqlLexer.Lex(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tokens, err := lexer.ConsumeAll(lex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	// for i, t := range tokens {
+	// 	fmt.Printf("%d: %#v\n", i, t)
+	// }
+	if len(tokens) != 7 {
+		t.Fatalf("Expected 7 got %d", len(tokens))
+	}
+}
+
+func TestSelectWhere(t *testing.T) {
+	p := participle.MustBuild(
+		&Select{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	s := Select{}
+	cases := []string{
+		"select * from s3object",
+		"select a, b from s3object s",
+		"select a, b from s3object as s",
+		"select a, b from s3object as s where a = 1",
+		"select a, b from s3object s where a = 1",
+		"select a, b from s3object where a = 1",
+	}
+	for i, tc := range cases {
+		err := p.ParseString(tc, &s)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+
+		// repr.Println(s, repr.Indent("  "), repr.OmitEmpty(true))
+	}
+}
+
+func TestLikeClause(t *testing.T) {
+	p := participle.MustBuild(
+		&Select{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	s := Select{}
+	cases := []string{
+		`select * from s3object where Name like 'abcd'`,
+		`select Name like 'abc' from s3object`,
+		`select * from s3object where Name not like 'abc'`,
+		`select * from s3object where Name like 'abc' escape 't'`,
+		`select * from s3object where Name like 'a\%' escape '?'`,
+		`select * from s3object where Name not like 'abc\' escape '?'`,
+		`select * from s3object where Name like 'a\%' escape LOWER('?')`,
+		`select * from s3object where Name not like LOWER('Bc\') escape '?'`,
+	}
+	for i, tc := range cases {
+		err := p.ParseString(tc, &s)
+		if err != nil {
+			t.Errorf("%d: %v", i, err)
+		}
+	}
+}
+
+func TestBetweenClause(t *testing.T) {
+	p := participle.MustBuild(
+		&Select{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	s := Select{}
+	cases := []string{
+		`select * from s3object where Id between 1 and 2`,
+		`select * from s3object where Id between 1 and 2 and name = 'Ab'`,
+		`select * from s3object where Id not between 1 and 2`,
+		`select * from s3object where Id not between 1 and 2 and name = 'Bc'`,
+	}
+	for i, tc := range cases {
+		err := p.ParseString(tc, &s)
+		if err != nil {
+			t.Errorf("%d: %v", i, err)
+		}
+	}
+}
+
+func TestFromClauseJSONPath(t *testing.T) {
+	p := participle.MustBuild(
+		&Select{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	s := Select{}
+	cases := []string{
+		"select * from s3object",
+		"select * from s3object[*].name",
+		"select * from s3object[*].books[*]",
+		"select * from s3object[*].books[*].name",
+		"select * from s3object where name > 2",
+		"select * from s3object[*].name where name > 2",
+		"select * from s3object[*].books[*] where name > 2",
+		"select * from s3object[*].books[*].name where name > 2",
+		"select * from s3object[*].books[*] s",
+		"select * from s3object[*].books[*].name as s",
+		"select * from s3object s where name > 2",
+		"select * from s3object[*].name as s where name > 2",
+	}
+	for i, tc := range cases {
+		err := p.ParseString(tc, &s)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+
+		// repr.Println(s, repr.Indent("  "), repr.OmitEmpty(true))
+	}
+
+}
+
+func TestSelectParsing(t *testing.T) {
+	p := participle.MustBuild(
+		&Select{},
+		participle.Lexer(sqlLexer),
+		participle.CaseInsensitive("Keyword"),
+	)
+
+	s := Select{}
+	cases := []string{
+		"select * from s3object where name > 2 or value > 1 or word > 2",
+		"select s.word.id + 2 from s3object s",
+		"select 1-2-3 from s3object s limit 1",
+	}
+	for i, tc := range cases {
+		err := p.ParseString(tc, &s)
+		if err != nil {
+			t.Fatalf("%d: %v", i, err)
+		}
+
+		// repr.Println(s, repr.Indent("  "), repr.OmitEmpty(true))
+	}
+}
+
+func TestSqlLexerArithOps(t *testing.T) {
+	s := bytes.NewBuffer([]byte("year from select month hour distinct"))
+	lex, err := sqlLexer.Lex(s)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tokens, err := lexer.ConsumeAll(lex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(tokens) != 7 {
+		t.Errorf("Expected 7 got %d", len(tokens))
+	}
+	// for i, t := range tokens {
+	// 	fmt.Printf("%d: %#v\n", i, t)
+	// }
+}

pkg/s3select/sql/sql.go

@@ -1,529 +0,0 @@
-/*
- * Minio Cloud Storage, (C) 2019 Minio, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package sql
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/xwb1989/sqlparser"
-)
-
-func getColumnName(colName *sqlparser.ColName) string {
-	columnName := colName.Qualifier.Name.String()
-	if qualifier := colName.Qualifier.Qualifier.String(); qualifier != "" {
-		columnName = qualifier + "." + columnName
-	}
-
-	if columnName == "" {
-		columnName = colName.Name.String()
-	} else {
-		columnName = columnName + "." + colName.Name.String()
-	}
-
-	return columnName
-}
-
-func newLiteralExpr(parserExpr sqlparser.Expr, tableAlias string) (Expr, error) {
-	switch parserExpr.(type) {
-	case *sqlparser.NullVal:
-		return newValueExpr(NewNull()), nil
-	case sqlparser.BoolVal:
-		return newValueExpr(NewBool((bool(parserExpr.(sqlparser.BoolVal))))), nil
-	case *sqlparser.SQLVal:
-		sqlValue := parserExpr.(*sqlparser.SQLVal)
-		value, err := NewValue(sqlValue)
-		if err != nil {
-			return nil, err
-		}
-		return newValueExpr(value), nil
-	case *sqlparser.ColName:
-		columnName := getColumnName(parserExpr.(*sqlparser.ColName))
-		if tableAlias != "" {
-			if !strings.HasPrefix(columnName, tableAlias+".") {
-				err := fmt.Errorf("column name %v does not start with table alias %v", columnName, tableAlias)
-				return nil, errInvalidKeyPath(err)
-			}
-			columnName = strings.TrimPrefix(columnName, tableAlias+".")
-		}
-
-		return newColumnExpr(columnName), nil
-	case sqlparser.ValTuple:
-		var valueType Type
-		var values []*Value
-		for i, valExpr := range parserExpr.(sqlparser.ValTuple) {
-			sqlVal, ok := valExpr.(*sqlparser.SQLVal)
-			if !ok {
-				return nil, errParseInvalidTypeParam(fmt.Errorf("value %v in Tuple should be primitive value", i+1))
-			}
-
-			val, err := NewValue(sqlVal)
-			if err != nil {
-				return nil, err
-			}
-
-			if i == 0 {
-				valueType = val.Type()
-			} else if valueType != val.Type() {
-				return nil, errParseInvalidTypeParam(fmt.Errorf("mixed value type is not allowed in Tuple"))
-			}
-
-			values = append(values, val)
-		}
-
-		return newValueExpr(NewArray(values)), nil
-	}
-
-	return nil, nil
-}
-
-func isExprToComparisonExpr(parserExpr *sqlparser.IsExpr, tableAlias string, isSelectExpr bool) (Expr, error) {
-	leftExpr, err := newExpr(parserExpr.Expr, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := newComparisonExpr(ComparisonOperator(parserExpr.Operator), leftExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	if !leftExpr.Type().isBase() {
-		return f, nil
-	}
-
-	value, err := f.Eval(nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return newValueExpr(value), nil
-}
-
-func rangeCondToComparisonFunc(parserExpr *sqlparser.RangeCond, tableAlias string, isSelectExpr bool) (Expr, error) {
-	leftExpr, err := newExpr(parserExpr.Left, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	fromExpr, err := newExpr(parserExpr.From, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	toExpr, err := newExpr(parserExpr.To, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := newComparisonExpr(ComparisonOperator(parserExpr.Operator), leftExpr, fromExpr, toExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	if !leftExpr.Type().isBase() || !fromExpr.Type().isBase() || !toExpr.Type().isBase() {
-		return f, nil
-	}
-
-	value, err := f.Eval(nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return newValueExpr(value), nil
-}
-
-func toComparisonExpr(parserExpr *sqlparser.ComparisonExpr, tableAlias string, isSelectExpr bool) (Expr, error) {
-	leftExpr, err := newExpr(parserExpr.Left, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	rightExpr, err := newExpr(parserExpr.Right, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := newComparisonExpr(ComparisonOperator(parserExpr.Operator), leftExpr, rightExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	if !leftExpr.Type().isBase() || !rightExpr.Type().isBase() {
-		return f, nil
-	}
-
-	value, err := f.Eval(nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return newValueExpr(value), nil
-}
-
-func toArithExpr(parserExpr *sqlparser.BinaryExpr, tableAlias string, isSelectExpr bool) (Expr, error) {
-	leftExpr, err := newExpr(parserExpr.Left, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	rightExpr, err := newExpr(parserExpr.Right, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := newArithExpr(ArithOperator(parserExpr.Operator), leftExpr, rightExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	if !leftExpr.Type().isBase() || !rightExpr.Type().isBase() {
-		return f, nil
-	}
-
-	value, err := f.Eval(nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return newValueExpr(value), nil
-}
-
-func toFuncExpr(parserExpr *sqlparser.FuncExpr, tableAlias string, isSelectExpr bool) (Expr, error) {
-	funcName := strings.ToUpper(parserExpr.Name.String())
-	if !isSelectExpr && isAggregateFuncName(funcName) {
-		return nil, errUnsupportedSQLOperation(fmt.Errorf("%v() must be used in select expression", funcName))
-	}
-	funcs, aggregatedExprFound, err := newSelectExprs(parserExpr.Exprs, tableAlias)
-	if err != nil {
-		return nil, err
-	}
-
-	if aggregatedExprFound {
-		return nil, errIncorrectSQLFunctionArgumentType(fmt.Errorf("%v(): aggregated expression must not be used as argument", funcName))
-	}
-
-	return newFuncExpr(FuncName(funcName), funcs...)
-}
-
-func toAndExpr(parserExpr *sqlparser.AndExpr, tableAlias string, isSelectExpr bool) (Expr, error) {
-	leftExpr, err := newExpr(parserExpr.Left, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	rightExpr, err := newExpr(parserExpr.Right, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := newAndExpr(leftExpr, rightExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	if leftExpr.Type() != Bool || rightExpr.Type() != Bool {
-		return f, nil
-	}
-
-	value, err := f.Eval(nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return newValueExpr(value), nil
-}
-
-func toOrExpr(parserExpr *sqlparser.OrExpr, tableAlias string, isSelectExpr bool) (Expr, error) {
-	leftExpr, err := newExpr(parserExpr.Left, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	rightExpr, err := newExpr(parserExpr.Right, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := newOrExpr(leftExpr, rightExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	if leftExpr.Type() != Bool || rightExpr.Type() != Bool {
-		return f, nil
-	}
-
-	value, err := f.Eval(nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return newValueExpr(value), nil
-}
-
-func toNotExpr(parserExpr *sqlparser.NotExpr, tableAlias string, isSelectExpr bool) (Expr, error) {
-	rightExpr, err := newExpr(parserExpr.Expr, tableAlias, isSelectExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	f, err := newNotExpr(rightExpr)
-	if err != nil {
-		return nil, err
-	}
-
-	if rightExpr.Type() != Bool {
-		return f, nil
-	}
-
-	value, err := f.Eval(nil)
-	if err != nil {
-		return nil, err
-	}
-
-	return newValueExpr(value), nil
-}
-
-func newExpr(parserExpr sqlparser.Expr, tableAlias string, isSelectExpr bool) (Expr, error) {
-	f, err := newLiteralExpr(parserExpr, tableAlias)
-	if err != nil {
-		return nil, err
-	}
-
-	if f != nil {
-		return f, nil
-	}
-
-	switch parserExpr.(type) {
-	case *sqlparser.ParenExpr:
-		return newExpr(parserExpr.(*sqlparser.ParenExpr).Expr, tableAlias, isSelectExpr)
-	case *sqlparser.IsExpr:
-		return isExprToComparisonExpr(parserExpr.(*sqlparser.IsExpr), tableAlias, isSelectExpr)
-	case *sqlparser.RangeCond:
-		return rangeCondToComparisonFunc(parserExpr.(*sqlparser.RangeCond), tableAlias, isSelectExpr)
-	case *sqlparser.ComparisonExpr:
-		return toComparisonExpr(parserExpr.(*sqlparser.ComparisonExpr), tableAlias, isSelectExpr)
-	case *sqlparser.BinaryExpr:
-		return toArithExpr(parserExpr.(*sqlparser.BinaryExpr), tableAlias, isSelectExpr)
-	case *sqlparser.FuncExpr:
-		return toFuncExpr(parserExpr.(*sqlparser.FuncExpr), tableAlias, isSelectExpr)
-	case *sqlparser.AndExpr:
-		return toAndExpr(parserExpr.(*sqlparser.AndExpr), tableAlias, isSelectExpr)
-	case *sqlparser.OrExpr:
-		return toOrExpr(parserExpr.(*sqlparser.OrExpr), tableAlias, isSelectExpr)
-	case *sqlparser.NotExpr:
-		return toNotExpr(parserExpr.(*sqlparser.NotExpr), tableAlias, isSelectExpr)
-	}
-
-	return nil, errParseUnsupportedSyntax(fmt.Errorf("unknown expression type %T; %v", parserExpr, parserExpr))
-}
-
-func newSelectExprs(parserSelectExprs []sqlparser.SelectExpr, tableAlias string) ([]Expr, bool, error) {
-	var funcs []Expr
-	starExprFound := false
-	aggregatedExprFound := false
-
-	for _, selectExpr := range parserSelectExprs {
-		switch selectExpr.(type) {
-		case *sqlparser.AliasedExpr:
-			if starExprFound {
-				return nil, false, errParseAsteriskIsNotAloneInSelectList(nil)
-			}
-
-			aliasedExpr := selectExpr.(*sqlparser.AliasedExpr)
-			f, err := newExpr(aliasedExpr.Expr, tableAlias, true)
-			if err != nil {
-				return nil, false, err
-			}
-
-			if f.Type() == aggregateFunction {
-				if !aggregatedExprFound {
-					aggregatedExprFound = true
-					if len(funcs) > 0 {
-						return nil, false, errParseUnsupportedSyntax(fmt.Errorf("expression must not mixed with aggregated expression"))
-					}
-				}
-			} else if aggregatedExprFound {
-				return nil, false, errParseUnsupportedSyntax(fmt.Errorf("expression must not mixed with aggregated expression"))
-			}
-
-			alias := aliasedExpr.As.String()
-			if alias != "" {
-				f = newAliasExpr(alias, f)
-			}
-
-			funcs = append(funcs, f)
-		case *sqlparser.StarExpr:
-			if starExprFound {
-				err := fmt.Errorf("only single star expression allowed")
-				return nil, false, errParseInvalidContextForWildcardInSelectList(err)
-			}
-			starExprFound = true
-			funcs = append(funcs, newStarExpr())
-		default:
-			return nil, false, errParseUnsupportedSyntax(fmt.Errorf("unknown select expression %v", selectExpr))
-		}
-	}
-
-	return funcs, aggregatedExprFound, nil
-}
-
-// Select - SQL Select statement.
-type Select struct {
-	tableName           string
-	tableAlias          string
-	selectExprs         []Expr
-	aggregatedExprFound bool
-	whereExpr           Expr
-}
-
-// TableAlias - returns table alias name.
-func (statement *Select) TableAlias() string {
-	return statement.tableAlias
-}
-
-// IsSelectAll - returns whether '*' is used in select expression or not.
-func (statement *Select) IsSelectAll() bool {
-	if len(statement.selectExprs) == 1 {
-		_, ok := statement.selectExprs[0].(*starExpr)
-		return ok
-	}
-
-	return false
-}
-
-// IsAggregated - returns whether aggregated functions are used in select expression or not.
-func (statement *Select) IsAggregated() bool {
-	return statement.aggregatedExprFound
-}
-
-// AggregateResult - returns aggregate result as record.
-func (statement *Select) AggregateResult(output Record) error {
-	if !statement.aggregatedExprFound {
-		return nil
-	}
-
-	for i, expr := range statement.selectExprs {
-		value, err := expr.AggregateValue()
-		if err != nil {
-			return err
-		}
-		if value == nil {
-			return errInternalError(fmt.Errorf("%v returns <nil> for AggregateValue()", expr))
-		}
-
-		name := fmt.Sprintf("_%v", i+1)
-		if _, ok := expr.(*aliasExpr); ok {
-			name = expr.(*aliasExpr).alias
-		}
-
-		if err = output.Set(name, value); err != nil {
-			return errInternalError(fmt.Errorf("error occurred to store value %v for %v; %v", value, name, err))
-		}
-	}
-
-	return nil
-}
-
-// Eval - evaluates this Select expressions for given record.
-func (statement *Select) Eval(input, output Record) (Record, error) {
-	if statement.whereExpr != nil {
-		value, err := statement.whereExpr.Eval(input)
-		if err != nil {
-			return nil, err
-		}
-
-		if value == nil || value.valueType != Bool {
-			err = fmt.Errorf("WHERE expression %v returns invalid bool value %v", statement.whereExpr, value)
-			return nil, errInternalError(err)
-		}
-
-		if !value.BoolValue() {
-			return nil, nil
-		}
-	}
-
-	// Call selectExprs
-	for i, expr := range statement.selectExprs {
-		value, err := expr.Eval(input)
-		if err != nil {
-			return nil, err
-		}
-
-		if statement.aggregatedExprFound {
-			continue
-		}
-
-		name := fmt.Sprintf("_%v", i+1)
-		switch expr.(type) {
-		case *starExpr:
-			return value.recordValue(), nil
-		case *aliasExpr:
-			name = expr.(*aliasExpr).alias
-		case *columnExpr:
-			name = expr.(*columnExpr).name
-		}
-
-		if err = output.Set(name, value); err != nil {
-			return nil, errInternalError(fmt.Errorf("error occurred to store value %v for %v; %v", value, name, err))
-		}
-	}
-
-	return output, nil
-}
-
-// NewSelect - creates new Select by parsing sql.
-func NewSelect(sql string) (*Select, error) {
-	stmt, err := sqlparser.Parse(sql)
-	if err != nil {
-		return nil, errUnsupportedSQLStructure(err)
-	}
-
-	selectStmt, ok := stmt.(*sqlparser.Select)
-	if !ok {
-		return nil, errParseUnsupportedSelect(fmt.Errorf("unsupported SQL statement %v", sql))
-	}
-
-	var tableName, tableAlias string
-	for _, fromExpr := range selectStmt.From {
-		tableExpr := fromExpr.(*sqlparser.AliasedTableExpr)
-		tableName = tableExpr.Expr.(sqlparser.TableName).Name.String()
-		tableAlias = tableExpr.As.String()
-	}
-
-	selectExprs, aggregatedExprFound, err := newSelectExprs(selectStmt.SelectExprs, tableAlias)
-	if err != nil {
-		return nil, err
-	}
-
-	var whereExpr Expr
-	if selectStmt.Where != nil {
-		whereExpr, err = newExpr(selectStmt.Where.Expr, tableAlias, false)
-		if err != nil {
-			return nil, err
-		}
-	}
-
-	return &Select{
-		tableName:           tableName,
-		tableAlias:          tableAlias,
-		selectExprs:         selectExprs,
-		aggregatedExprFound: aggregatedExprFound,
-		whereExpr:           whereExpr,
-	}, nil
-}

pkg/s3select/sql/statement.go

@@ -0,0 +1,201 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+)
+
+var (
+	errBadLimitSpecified = errors.New("Limit value must be a positive integer")
+)
+
+// SelectStatement is the top level parsed and analyzed structure
+type SelectStatement struct {
+	selectAST *Select
+
+	// Analysis result of the statement
+	selectQProp qProp
+
+	// Result of parsing the limit clause if one is present
+	// (otherwise -1)
+	limitValue int64
+
+	// Count of rows that have been output.
+	outputCount int64
+}
+
+// ParseSelectStatement - parses a select query from the given string
+// and analyzes it.
+func ParseSelectStatement(s string) (stmt SelectStatement, err error) {
+	var selectAST Select
+	err = SQLParser.ParseString(s, &selectAST)
+	if err != nil {
+		err = errQueryParseFailure(err)
+		return
+	}
+	stmt.selectAST = &selectAST
+
+	// Check the parsed limit value
+	stmt.limitValue, err = parseLimit(selectAST.Limit)
+	if err != nil {
+		err = errQueryAnalysisFailure(err)
+		return
+	}
+
+	// Analyze where clause
+	if selectAST.Where != nil {
+		whereQProp := selectAST.Where.analyze(&selectAST)
+		if whereQProp.err != nil {
+			err = errQueryAnalysisFailure(fmt.Errorf("Where clause error: %v", whereQProp.err))
+			return
+		}
+
+		if whereQProp.isAggregation {
+			err = errQueryAnalysisFailure(errors.New("WHERE clause cannot have an aggregation"))
+			return
+		}
+	}
+
+	// Validate table name
+	tableString := strings.ToLower(selectAST.From.Table.String())
+	if !strings.HasPrefix(tableString, "s3object.") && tableString != "s3object" {
+		err = errBadTableName(errors.New("Table name must be s3object"))
+		return
+	}
+
+	// Analyze main select expression
+	stmt.selectQProp = selectAST.Expression.analyze(&selectAST)
+	err = stmt.selectQProp.err
+	if err != nil {
+		err = errQueryAnalysisFailure(err)
+	}
+	return
+}
+
+func parseLimit(v *LitValue) (int64, error) {
+	switch {
+	case v == nil:
+		return -1, nil
+	case v.Number == nil:
+		return -1, errBadLimitSpecified
+	default:
+		r := int64(*v.Number)
+		if r < 0 {
+			return -1, errBadLimitSpecified
+		}
+		return r, nil
+	}
+}
+
+// IsAggregated returns if the statement involves SQL aggregation
+func (e *SelectStatement) IsAggregated() bool {
+	return e.selectQProp.isAggregation
+}
+
+// AggregateResult - returns the aggregated result after all input
+// records have been processed. Applies only to aggregation queries.
+func (e *SelectStatement) AggregateResult(output Record) error {
+	for i, expr := range e.selectAST.Expression.Expressions {
+		v, err := expr.evalNode(nil)
+		if err != nil {
+			return err
+		}
+		output.Set(fmt.Sprintf("_%d", i+1), v)
+	}
+	return nil
+}
+
+// AggregateRow - aggregates the input record. Applies only to
+// aggregation queries.
+func (e *SelectStatement) AggregateRow(input Record) error {
+	for _, expr := range e.selectAST.Expression.Expressions {
+		err := expr.aggregateRow(input)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Eval - evaluates the Select statement for the given record. It
+// applies only to non-aggregation queries.
+func (e *SelectStatement) Eval(input, output Record) (Record, error) {
+	if whereExpr := e.selectAST.Where; whereExpr != nil {
+		value, err := whereExpr.evalNode(input)
+		if err != nil {
+			return nil, err
+		}
+
+		b, ok := value.ToBool()
+		if !ok {
+			err = fmt.Errorf("WHERE expression did not return bool")
+			return nil, err
+		}
+
+		if !b {
+			// Where clause is not satisfied by the row
+			return nil, nil
+		}
+	}
+
+	if e.selectAST.Expression.All {
+		// Return the input record for `SELECT * FROM
+		// .. WHERE ..`
+
+		// Update count of records output.
+		if e.limitValue > -1 {
+			e.outputCount++
+		}
+
+		return input, nil
+	}
+
+	for i, expr := range e.selectAST.Expression.Expressions {
+		v, err := expr.evalNode(input)
+		if err != nil {
+			return nil, err
+		}
+
+		// Pick output column names
+		if expr.As != "" {
+			output.Set(expr.As, v)
+		} else if comp, ok := getLastKeypathComponent(expr.Expression); ok {
+			output.Set(comp, v)
+		} else {
+			output.Set(fmt.Sprintf("_%d", i+1), v)
+		}
+	}
+
+	// Update count of records output.
+	if e.limitValue > -1 {
+		e.outputCount++
+	}
+
+	return output, nil
+}
+
+// LimitReached - returns true if the number of records output has
+// reached the value of the `LIMIT` clause.
+func (e *SelectStatement) LimitReached() bool {
+	if e.limitValue == -1 {
+		return false
+	}
+	return e.outputCount >= e.limitValue
+}

pkg/s3select/sql/stringfuncs.go

@@ -0,0 +1,188 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"errors"
+	"strings"
+)
+
+var (
+	errMalformedEscapeSequence  = errors.New("Malformed escape sequence in LIKE clause")
+	errInvalidTrimArg           = errors.New("Trim argument is invalid - this should not happen")
+	errInvalidSubstringIndexLen = errors.New("Substring start index or length falls outside the string")
+)
+
+const (
+	percent    rune = '%'
+	underscore rune = '_'
+	runeZero   rune = 0
+)
+
+func evalSQLLike(text, pattern string, escape rune) (match bool, err error) {
+	s := []rune{}
+	prev := runeZero
+	hasLeadingPercent := false
+	patLen := len([]rune(pattern))
+	for i, r := range pattern {
+		if i > 0 && prev == escape {
+			switch r {
+			case percent, escape, underscore:
+				s = append(s, r)
+				prev = r
+				if r == escape {
+					prev = runeZero
+				}
+			default:
+				return false, errMalformedEscapeSequence
+			}
+			continue
+		}
+
+		prev = r
+
+		var ok bool
+		switch r {
+		case percent:
+			if len(s) == 0 {
+				hasLeadingPercent = true
+				continue
+			}
+
+			text, ok = matcher(text, string(s), hasLeadingPercent)
+			if !ok {
+				return false, nil
+			}
+			hasLeadingPercent = true
+			s = []rune{}
+
+			if i == patLen-1 {
+				// Last pattern character is a %, so
+				// we are done.
+				return true, nil
+			}
+
+		case underscore:
+			if len(s) == 0 {
+				text, ok = dropRune(text)
+				if !ok {
+					return false, nil
+				}
+				continue
+			}
+
+			text, ok = matcher(text, string(s), hasLeadingPercent)
+			if !ok {
+				return false, nil
+			}
+			hasLeadingPercent = false
+
+			text, ok = dropRune(text)
+			if !ok {
+				return false, nil
+			}
+			s = []rune{}
+
+		case escape:
+			if i == patLen-1 {
+				return false, errMalformedEscapeSequence
+			}
+			// Otherwise do nothing.
+
+		default:
+			s = append(s, r)
+		}
+
+	}
+	if hasLeadingPercent {
+		return strings.HasSuffix(text, string(s)), nil
+	}
+	return string(s) == text, nil
+}
+
+// matcher - Finds `pat` in `text`, and returns the part remainder of
+// `text`, after the match. If leadingPercent is false, `pat` must be
+// the prefix of `text`, otherwise it must be a substring.
+func matcher(text, pat string, leadingPercent bool) (res string, match bool) {
+	if !leadingPercent {
+		res = strings.TrimPrefix(text, pat)
+		if len(text) == len(res) {
+			return "", false
+		}
+	} else {
+		parts := strings.SplitN(text, pat, 2)
+		if len(parts) == 1 {
+			return "", false
+		}
+		res = parts[1]
+	}
+	return res, true
+}
+
+func dropRune(text string) (res string, ok bool) {
+	r := []rune(text)
+	if len(r) == 0 {
+		return "", false
+	}
+	return string(r[1:]), true
+}
+
+func evalSQLSubstring(s string, startIdx, length int) (res string, err error) {
+	if startIdx <= 0 || startIdx > len(s) {
+		return "", errInvalidSubstringIndexLen
+	}
+	// StartIdx is 1-based in the input
+	startIdx--
+
+	rs := []rune(s)
+	endIdx := len(rs)
+	if length != -1 {
+		if length < 0 || startIdx+length > len(s) {
+			return "", errInvalidSubstringIndexLen
+		}
+		endIdx = startIdx + length
+	}
+
+	return string(rs[startIdx:endIdx]), nil
+}
+
+const (
+	trimLeading  = "LEADING"
+	trimTrailing = "TRAILING"
+	trimBoth     = "BOTH"
+)
+
+func evalSQLTrim(where *string, trimChars, text string) (result string, err error) {
+	cutSet := " "
+	if trimChars != "" {
+		cutSet = trimChars
+	}
+
+	trimFunc := strings.Trim
+	switch {
+	case where == nil:
+	case *where == trimBoth:
+	case *where == trimLeading:
+		trimFunc = strings.TrimLeft
+	case *where == trimTrailing:
+		trimFunc = strings.TrimRight
+	default:
+		return "", errInvalidTrimArg
+	}
+
+	return trimFunc(text, cutSet), nil
+}

pkg/s3select/sql/stringfuncs_test.go

@@ -0,0 +1,107 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"testing"
+)
+
+func TestEvalSQLLike(t *testing.T) {
+	dropCases := []struct {
+		input, resultExpected string
+		matchExpected         bool
+	}{
+		{"", "", false},
+		{"a", "", true},
+		{"ab", "b", true},
+		{"தமிழ்", "மிழ்", true},
+	}
+
+	for i, tc := range dropCases {
+		res, ok := dropRune(tc.input)
+		if res != tc.resultExpected || ok != tc.matchExpected {
+			t.Errorf("DropRune Case %d failed", i)
+		}
+	}
+
+	matcherCases := []struct {
+		iText, iPat        string
+		iHasLeadingPercent bool
+		resultExpected     string
+		matchExpected      bool
+	}{
+		{"abcd", "bcd", false, "", false},
+		{"abcd", "bcd", true, "", true},
+		{"abcd", "abcd", false, "", true},
+		{"abcd", "abcd", true, "", true},
+		{"abcd", "ab", false, "cd", true},
+		{"abcd", "ab", true, "cd", true},
+		{"abcd", "bc", false, "", false},
+		{"abcd", "bc", true, "d", true},
+	}
+
+	for i, tc := range matcherCases {
+		res, ok := matcher(tc.iText, tc.iPat, tc.iHasLeadingPercent)
+		if res != tc.resultExpected || ok != tc.matchExpected {
+			t.Errorf("Matcher Case %d failed", i)
+		}
+	}
+
+	evalCases := []struct {
+		iText, iPat   string
+		iEsc          rune
+		matchExpected bool
+		errExpected   error
+	}{
+		{"abcd", "abc", runeZero, false, nil},
+		{"abcd", "abcd", runeZero, true, nil},
+		{"abcd", "abc_", runeZero, true, nil},
+		{"abcd", "_bdd", runeZero, false, nil},
+		{"abcd", "_b_d", runeZero, true, nil},
+
+		{"abcd", "____", runeZero, true, nil},
+		{"abcd", "____%", runeZero, true, nil},
+		{"abcd", "%____", runeZero, true, nil},
+		{"abcd", "%__", runeZero, true, nil},
+		{"abcd", "____", runeZero, true, nil},
+
+		{"", "_", runeZero, false, nil},
+		{"", "%", runeZero, true, nil},
+		{"abcd", "%%%%%", runeZero, true, nil},
+		{"abcd", "_____", runeZero, false, nil},
+		{"abcd", "%%%%%", runeZero, true, nil},
+
+		{"a%%d", `a\%\%d`, '\\', true, nil},
+		{"a%%d", `a\%d`, '\\', false, nil},
+		{`a%%\d`, `a\%\%\\d`, '\\', true, nil},
+		{`a%%\`, `a\%\%\\`, '\\', true, nil},
+		{`a%__%\`, `a\%\_\_\%\\`, '\\', true, nil},
+
+		{`a%__%\`, `a\%\_\_\%_`, '\\', true, nil},
+		{`a%__%\`, `a\%\_\__`, '\\', false, nil},
+		{`a%__%\`, `a\%\_\_%`, '\\', true, nil},
+		{`a%__%\`, `a?%?_?_?%\`, '?', true, nil},
+	}
+
+	for i, tc := range evalCases {
+		// fmt.Println("Case:", i)
+		res, err := evalSQLLike(tc.iText, tc.iPat, tc.iEsc)
+		if res != tc.matchExpected || err != tc.errExpected {
+			t.Errorf("Eval Case %d failed: %v %v", i, res, err)
+		}
+	}
+}

pkg/s3select/sql/timestampfuncs.go

@@ -0,0 +1,199 @@
+/*
+ * Minio Cloud Storage, (C) 2019 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package sql
+
+import (
+	"time"
+)
+
+const (
+	layoutYear       = "2006T"
+	layoutMonth      = "2006-01T"
+	layoutDay        = "2006-01-02T"
+	layoutMinute     = "2006-01-02T15:04Z07:00"
+	layoutSecond     = "2006-01-02T15:04:05Z07:00"
+	layoutNanosecond = "2006-01-02T15:04:05.999999999Z07:00"
+)
+
+var (
+	tformats = []string{
+		layoutYear,
+		layoutMonth,
+		layoutDay,
+		layoutMinute,
+		layoutSecond,
+		layoutNanosecond,
+	}
+	oneNanoSecond = 1
+)
+
+func parseSQLTimestamp(s string) (t time.Time, err error) {
+	for _, f := range tformats {
+		t, err = time.Parse(f, s)
+		if err == nil {
+			break
+		}
+	}
+	return
+}
+
+// FormatSQLTimestamp - returns the a string representation of the
+// timestamp as used in S3 Select
+func FormatSQLTimestamp(t time.Time) string {
+	_, zoneOffset := t.Zone()
+	hasZone := zoneOffset != 0
+	hasFracSecond := t.Nanosecond() != 0
+	hasSecond := t.Second() != 0
+	hasTime := t.Hour() != 0 || t.Minute() != 0
+	hasDay := t.Day() != 1
+	hasMonth := t.Month() != 1
+
+	switch {
+	case hasFracSecond:
+		return t.Format(layoutNanosecond)
+	case hasSecond:
+		return t.Format(layoutSecond)
+	case hasTime || hasZone:
+		return t.Format(layoutMinute)
+	case hasDay:
+		return t.Format(layoutDay)
+	case hasMonth:
+		return t.Format(layoutMonth)
+	default:
+		return t.Format(layoutYear)
+	}
+}
+
+const (
+	timePartYear           = "YEAR"
+	timePartMonth          = "MONTH"
+	timePartDay            = "DAY"
+	timePartHour           = "HOUR"
+	timePartMinute         = "MINUTE"
+	timePartSecond         = "SECOND"
+	timePartTimezoneHour   = "TIMEZONE_HOUR"
+	timePartTimezoneMinute = "TIMEZONE_MINUTE"
+)
+
+func extract(what string, t time.Time) (v *Value, err error) {
+	switch what {
+	case timePartYear:
+		return FromInt(int64(t.Year())), nil
+	case timePartMonth:
+		return FromInt(int64(t.Month())), nil
+	case timePartDay:
+		return FromInt(int64(t.Day())), nil
+	case timePartHour:
+		return FromInt(int64(t.Hour())), nil
+	case timePartMinute:
+		return FromInt(int64(t.Minute())), nil
+	case timePartSecond:
+		return FromInt(int64(t.Second())), nil
+	case timePartTimezoneHour:
+		_, zoneOffset := t.Zone()
+		return FromInt(int64(zoneOffset / 3600)), nil
+	case timePartTimezoneMinute:
+		_, zoneOffset := t.Zone()
+		return FromInt(int64((zoneOffset % 3600) / 60)), nil
+	default:
+		// This does not happen
+		return nil, errNotImplemented
+	}
+}
+
+func dateAdd(timePart string, qty float64, t time.Time) (*Value, error) {
+	var duration time.Duration
+	switch timePart {
+	case timePartYear:
+		return FromTimestamp(t.AddDate(int(qty), 0, 0)), nil
+	case timePartMonth:
+		return FromTimestamp(t.AddDate(0, int(qty), 0)), nil
+	case timePartDay:
+		return FromTimestamp(t.AddDate(0, 0, int(qty))), nil
+	case timePartHour:
+		duration = time.Duration(qty) * time.Hour
+	case timePartMinute:
+		duration = time.Duration(qty) * time.Minute
+	case timePartSecond:
+		duration = time.Duration(qty) * time.Second
+	default:
+		return nil, errNotImplemented
+	}
+	return FromTimestamp(t.Add(duration)), nil
+}
+
+const (
+	dayInNanoseconds = time.Hour * 24
+)
+
+// dateDiff computes the difference between two times in terms of the
+// `timePart` which can be years, months, days, hours, minutes or
+// seconds. For difference in years, months or days, the time part,
+// including timezone is ignored.
+func dateDiff(timePart string, ts1, ts2 time.Time) (*Value, error) {
+	if ts2.Before(ts1) {
+		v, err := dateDiff(timePart, ts2, ts1)
+		v.negate()
+		return v, err
+	}
+
+	duration := ts2.Sub(ts1)
+	y1, m1, d1 := ts1.Date()
+	y2, m2, d2 := ts2.Date()
+	dy, dm := int64(y2-y1), int64(m2-m1)
+
+	switch timePart {
+	case timePartYear:
+		if m2 > m1 || (m2 == m1 && d2 >= d1) {
+			return FromInt(dy), nil
+		}
+		return FromInt(dy - 1), nil
+	case timePartMonth:
+		months := 12 * dy
+		if m2 >= m1 {
+			months += dm
+		} else {
+			months += 12 + dm
+		}
+		if d2 < d1 {
+			months--
+		}
+		return FromInt(months), nil
+	case timePartDay:
+		// To compute the number of days between two times
+		// using the time package, zero out the time portions
+		// of the timestamps, compute the difference duration
+		// and then divide by the length of a day.
+		d1 := time.Date(y1, m1, d1, 0, 0, 0, 0, time.UTC)
+		d2 := time.Date(y2, m2, d2, 0, 0, 0, 0, time.UTC)
+		diff := d2.Sub(d1)
+		days := diff / dayInNanoseconds
+		return FromInt(int64(days)), nil
+	case timePartHour:
+		hours := duration / time.Hour
+		return FromInt(int64(hours)), nil
+	case timePartMinute:
+		minutes := duration / time.Minute
+		return FromInt(int64(minutes)), nil
+	case timePartSecond:
+		seconds := duration / time.Second
+		return FromInt(int64(seconds)), nil
+	default:
+
+	}
+	return nil, errNotImplemented
+}