astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-07-19 12:10:24 +03:00
Code Issues Packages Projects Releases Wiki Activity

rename all remaining packages to internal/ (#12418)

Browse Source 
This is to ensure that there are no projects
that try to import `minio/minio/pkg` into
their own repo. Any such common packages should
go to `https://github.com/minio/pkg`
This commit is contained in:
Harshavardhana
2021-06-01 14:59:40 -07:00
committed by GitHub
parent bf87c4b1e4
commit 1f262daf6f
540 changed files with 757 additions and 778 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/s3select/csv/args.go
+201
View File
@@ -0,0 +1,201 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package csv
import (
"encoding/xml"
"errors"
"fmt"
"io"
"strings"
"unicode/utf8"
)
const (
none = "none"
use = "use"
defaultRecordDelimiter = "\n"
defaultFieldDelimiter = ","
defaultQuoteCharacter = `"`
defaultQuoteEscapeCharacter = `"`
defaultCommentCharacter = "#"
asneeded = "asneeded"
)
// ReaderArgs - represents elements inside <InputSerialization><CSV> in request XML.
type ReaderArgs struct {
FileHeaderInfo string `xml:"FileHeaderInfo"`
RecordDelimiter string `xml:"RecordDelimiter"`
FieldDelimiter string `xml:"FieldDelimiter"`
QuoteCharacter string `xml:"QuoteCharacter"`
QuoteEscapeCharacter string `xml:"QuoteEscapeCharacter"`
CommentCharacter string `xml:"Comments"`
AllowQuotedRecordDelimiter bool `xml:"AllowQuotedRecordDelimiter"`
unmarshaled bool
}
// IsEmpty - returns whether reader args is empty or not.
func (args *ReaderArgs) IsEmpty() bool {
return !args.unmarshaled
}
// UnmarshalXML - decodes XML data.
func (args *ReaderArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) {
args.FileHeaderInfo = none
args.RecordDelimiter = defaultRecordDelimiter
args.FieldDelimiter = defaultFieldDelimiter
args.QuoteCharacter = defaultQuoteCharacter
args.QuoteEscapeCharacter = defaultQuoteEscapeCharacter
args.CommentCharacter = defaultCommentCharacter
args.AllowQuotedRecordDelimiter = false
for {
// Read tokens from the XML document in a stream.
t, err := d.Token()
if err != nil {
if err == io.EOF {
break
}
return err
}
switch se := t.(type) {
case xml.StartElement:
tagName := se.Name.Local
switch tagName {
case "AllowQuotedRecordDelimiter":
var b bool
if err = d.DecodeElement(&b, &se); err != nil {
return err
}
args.AllowQuotedRecordDelimiter = b
default:
var s string
if err = d.DecodeElement(&s, &se); err != nil {
return err
}
switch tagName {
case "FileHeaderInfo":
args.FileHeaderInfo = strings.ToLower(s)
case "RecordDelimiter":
args.RecordDelimiter = s
case "FieldDelimiter":
args.FieldDelimiter = s
case "QuoteCharacter":
if utf8.RuneCountInString(s) > 1 {
return fmt.Errorf("unsupported QuoteCharacter '%v'", s)
}
args.QuoteCharacter = s
case "QuoteEscapeCharacter":
switch utf8.RuneCountInString(s) {
case 0:
args.QuoteEscapeCharacter = defaultQuoteEscapeCharacter
case 1:
args.QuoteEscapeCharacter = s
default:
return fmt.Errorf("unsupported QuoteEscapeCharacter '%v'", s)
}
case "Comments":
args.CommentCharacter = s
default:
return errors.New("unrecognized option")
}
}
}
}
args.unmarshaled = true
return nil
}
// WriterArgs - represents elements inside <OutputSerialization><CSV/> in request XML.
type WriterArgs struct {
QuoteFields string `xml:"QuoteFields"`
RecordDelimiter string `xml:"RecordDelimiter"`
FieldDelimiter string `xml:"FieldDelimiter"`
QuoteCharacter string `xml:"QuoteCharacter"`
QuoteEscapeCharacter string `xml:"QuoteEscapeCharacter"`
unmarshaled bool
}
// IsEmpty - returns whether writer args is empty or not.
func (args *WriterArgs) IsEmpty() bool {
return !args.unmarshaled
}
// UnmarshalXML - decodes XML data.
func (args *WriterArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
args.QuoteFields = asneeded
args.RecordDelimiter = defaultRecordDelimiter
args.FieldDelimiter = defaultFieldDelimiter
args.QuoteCharacter = defaultQuoteCharacter
args.QuoteEscapeCharacter = defaultQuoteCharacter
for {
// Read tokens from the XML document in a stream.
t, err := d.Token()
if err != nil {
if err == io.EOF {
break
}
return err
}
switch se := t.(type) {
case xml.StartElement:
var s string
if err = d.DecodeElement(&s, &se); err != nil {
return err
}
switch se.Name.Local {
case "QuoteFields":
args.QuoteFields = strings.ToLower(s)
case "RecordDelimiter":
args.RecordDelimiter = s
case "FieldDelimiter":
args.FieldDelimiter = s
case "QuoteCharacter":
switch utf8.RuneCountInString(s) {
case 0:
args.QuoteCharacter = "\x00"
case 1:
args.QuoteCharacter = s
default:
return fmt.Errorf("unsupported QuoteCharacter '%v'", s)
}
case "QuoteEscapeCharacter":
switch utf8.RuneCountInString(s) {
case 0:
args.QuoteEscapeCharacter = defaultQuoteEscapeCharacter
case 1:
args.QuoteEscapeCharacter = s
default:
return fmt.Errorf("unsupported QuoteCharacter '%v'", s)
}
default:
return errors.New("unrecognized option")
}
}
}
args.unmarshaled = true
return nil
}
internal/s3select/csv/errors.go
+65
View File
@@ -0,0 +1,65 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package csv
import "errors"
type s3Error struct {
code string
message string
statusCode int
cause error
}
func (err *s3Error) Cause() error {
return err.cause
}
func (err *s3Error) ErrorCode() string {
return err.code
}
func (err *s3Error) ErrorMessage() string {
return err.message
}
func (err *s3Error) HTTPStatusCode() int {
return err.statusCode
}
func (err *s3Error) Error() string {
return err.message
}
func errCSVParsingError(err error) *s3Error {
return &s3Error{
code: "CSVParsingError",
message: "Encountered an error parsing the CSV file. Check the file and try again.",
statusCode: 400,
cause: err,
}
}
func errInvalidTextEncodingError() *s3Error {
return &s3Error{
code: "InvalidTextEncoding",
message: "UTF-8 encoding is required.",
statusCode: 400,
cause: errors.New("invalid utf8 encoding"),
}
}
internal/s3select/csv/reader.go
+326
View File
@@ -0,0 +1,326 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package csv
import (
"bufio"
"bytes"
"fmt"
"io"
"runtime"
"sync"
"unicode/utf8"
csv "github.com/minio/csvparser"
"github.com/minio/minio/internal/s3select/sql"
)
// Reader - CSV record reader for S3Select.
type Reader struct {
args *ReaderArgs
readCloser io.ReadCloser // raw input
buf *bufio.Reader // input to the splitter
columnNames []string // names of columns
nameIndexMap map[string]int64 // name to column index
current [][]string // current block of results to be returned
recordsRead int // number of records read in current slice
input chan *queueItem // input for workers
queue chan *queueItem // output from workers in order
err error // global error state, only touched by Reader.Read
bufferPool sync.Pool // pool of []byte objects for input
csvDstPool sync.Pool // pool of [][]string used for output
close chan struct{} // used for shutting down the splitter before end of stream
readerWg sync.WaitGroup // used to keep track of async reader.
}
// queueItem is an item in the queue.
type queueItem struct {
input []byte // raw input sent to the worker
dst chan [][]string // result of block decode
err error // any error encountered will be set here
}
// Read - reads single record.
// Once Read is called the previous record should no longer be referenced.
func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
// If we have have any records left, return these before any error.
for len(r.current) <= r.recordsRead {
if r.err != nil {
return nil, r.err
}
// Move to next block
item, ok := <-r.queue
if !ok {
r.err = io.EOF
return nil, r.err
}
//lint:ignore SA6002 Using pointer would allocate more since we would have to copy slice header before taking a pointer.
r.csvDstPool.Put(r.current)
r.current = <-item.dst
r.err = item.err
r.recordsRead = 0
}
csvRecord := r.current[r.recordsRead]
r.recordsRead++
// If no column names are set, use _(index)
if r.columnNames == nil {
r.columnNames = make([]string, len(csvRecord))
for i := range csvRecord {
r.columnNames[i] = fmt.Sprintf("_%v", i+1)
}
}
// If no index map, add that.
if r.nameIndexMap == nil {
r.nameIndexMap = make(map[string]int64)
for i := range r.columnNames {
r.nameIndexMap[r.columnNames[i]] = int64(i)
}
}
dstRec, ok := dst.(*Record)
if !ok {
dstRec = &Record{}
}
dstRec.columnNames = r.columnNames
dstRec.csvRecord = csvRecord
dstRec.nameIndexMap = r.nameIndexMap
return dstRec, nil
}
// Close - closes underlying reader.
func (r *Reader) Close() error {
if r.close != nil {
close(r.close)
r.readerWg.Wait()
r.close = nil
}
r.recordsRead = len(r.current)
if r.err == nil {
r.err = io.EOF
}
return r.readCloser.Close()
}
// nextSplit will attempt to skip a number of bytes and
// return the buffer until the next newline occurs.
// The last block will be sent along with an io.EOF.
func (r *Reader) nextSplit(skip int, dst []byte) ([]byte, error) {
if cap(dst) < skip {
dst = make([]byte, 0, skip+1024)
}
dst = dst[:skip]
if skip > 0 {
n, err := io.ReadFull(r.buf, dst)
if err != nil && err != io.ErrUnexpectedEOF {
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
return dst[:n], err
}
dst = dst[:n]
if err == io.ErrUnexpectedEOF {
return dst, io.EOF
}
}
// Read until next line.
in, err := r.buf.ReadBytes('\n')
dst = append(dst, in...)
return dst, err
}
// csvSplitSize is the size of each block.
// Blocks will read this much and find the first following newline.
// 128KB appears to be a very reasonable default.
const csvSplitSize = 128 << 10
// startReaders will read the header if needed and spin up a parser
// and a number of workers based on GOMAXPROCS.
// If an error is returned no goroutines have been started and r.err will have been set.
func (r *Reader) startReaders(newReader func(io.Reader) *csv.Reader) error {
if r.args.FileHeaderInfo != none {
// Read column names
// Get one line.
b, err := r.nextSplit(0, nil)
if err != nil {
r.err = err
return err
}
if !utf8.Valid(b) {
return errInvalidTextEncodingError()
}
reader := newReader(bytes.NewReader(b))
record, err := reader.Read()
if err != nil {
r.err = err
if err != io.EOF {
r.err = errCSVParsingError(err)
return errCSVParsingError(err)
}
return err
}
if r.args.FileHeaderInfo == use {
// Copy column names since records will be reused.
columns := append(make([]string, 0, len(record)), record...)
r.columnNames = columns
}
}
r.bufferPool.New = func() interface{} {
return make([]byte, csvSplitSize+1024)
}
// Return first block
next, nextErr := r.nextSplit(csvSplitSize, r.bufferPool.Get().([]byte))
// Check if first block is valid.
if !utf8.Valid(next) {
return errInvalidTextEncodingError()
}
// Create queue
r.queue = make(chan *queueItem, runtime.GOMAXPROCS(0))
r.input = make(chan *queueItem, runtime.GOMAXPROCS(0))
r.readerWg.Add(1)
// Start splitter
go func() {
defer close(r.input)
defer close(r.queue)
defer r.readerWg.Done()
for {
q := queueItem{
input: next,
dst: make(chan [][]string, 1),
err: nextErr,
}
select {
case <-r.close:
return
case r.queue <- &q:
}
select {
case <-r.close:
return
case r.input <- &q:
}
if nextErr != nil {
// Exit on any error.
return
}
next, nextErr = r.nextSplit(csvSplitSize, r.bufferPool.Get().([]byte))
}
}()
// Start parsers
for i := 0; i < runtime.GOMAXPROCS(0); i++ {
go func() {
for in := range r.input {
if len(in.input) == 0 {
in.dst <- nil
continue
}
dst, ok := r.csvDstPool.Get().([][]string)
if !ok {
dst = make([][]string, 0, 1000)
}
cr := newReader(bytes.NewBuffer(in.input))
all := dst[:0]
err := func() error {
// Read all records until EOF or another error.
for {
record, err := cr.Read()
if err == io.EOF {
return nil
}
if err != nil {
return errCSVParsingError(err)
}
var recDst []string
if len(dst) > len(all) {
recDst = dst[len(all)]
}
if cap(recDst) < len(record) {
recDst = make([]string, len(record))
}
recDst = recDst[:len(record)]
copy(recDst, record)
all = append(all, recDst)
}
}()
if err != nil {
in.err = err
}
// We don't need the input any more.
//lint:ignore SA6002 Using pointer would allocate more since we would have to copy slice header before taking a pointer.
r.bufferPool.Put(in.input)
in.input = nil
in.dst <- all
}
}()
}
return nil
}
// NewReader - creates new CSV reader using readCloser.
func NewReader(readCloser io.ReadCloser, args *ReaderArgs) (*Reader, error) {
if args == nil || args.IsEmpty() {
panic(fmt.Errorf("empty args passed %v", args))
}
csvIn := io.Reader(readCloser)
if args.RecordDelimiter != "\n" {
csvIn = &recordTransform{
reader: readCloser,
recordDelimiter: []byte(args.RecordDelimiter),
oneByte: make([]byte, len(args.RecordDelimiter)-1),
}
}
r := &Reader{
args: args,
buf: bufio.NewReaderSize(csvIn, csvSplitSize*2),
readCloser: readCloser,
close: make(chan struct{}),
}
// Assume args are validated by ReaderArgs.UnmarshalXML()
newCsvReader := func(r io.Reader) *csv.Reader {
ret := csv.NewReader(r)
ret.Comma = []rune(args.FieldDelimiter)[0]
ret.Comment = []rune(args.CommentCharacter)[0]
ret.Quote = []rune{}
if len([]rune(args.QuoteCharacter)) > 0 {
// Add the first rune of args.QuoteChracter
ret.Quote = append(ret.Quote, []rune(args.QuoteCharacter)[0])
}
ret.QuoteEscape = []rune(args.QuoteEscapeCharacter)[0]
ret.FieldsPerRecord = -1
// If LazyQuotes is true, a quote may appear in an unquoted field and a
// non-doubled quote may appear in a quoted field.
ret.LazyQuotes = true
// We do not trim leading space to keep consistent with s3.
ret.TrimLeadingSpace = false
ret.ReuseRecord = true
return ret
}
return r, r.startReaders(newCsvReader)
}
internal/s3select/csv/reader_contrib_test.go
+650
View File
@@ -0,0 +1,650 @@
/*
* MinIO Object Storage (c) 2021 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package csv
import (
"bytes"
"errors"
"fmt"
"io"
"io/ioutil"
"path/filepath"
"reflect"
"strings"
"testing"
"github.com/klauspost/compress/zip"
"github.com/minio/minio/internal/s3select/sql"
)
func TestRead(t *testing.T) {
cases := []struct {
content string
recordDelimiter string
fieldDelimiter string
}{
{"1,2,3\na,b,c\n", "\n", ","},
{"1,2,3\ta,b,c\t", "\t", ","},
{"1,2,3\r\na,b,c\r\n", "\r\n", ","},
}
for i, c := range cases {
var err error
var record sql.Record
var result bytes.Buffer
r, _ := NewReader(ioutil.NopCloser(strings.NewReader(c.content)), &ReaderArgs{
FileHeaderInfo: none,
RecordDelimiter: c.recordDelimiter,
FieldDelimiter: c.fieldDelimiter,
QuoteCharacter: defaultQuoteCharacter,
QuoteEscapeCharacter: defaultQuoteEscapeCharacter,
CommentCharacter: defaultCommentCharacter,
AllowQuotedRecordDelimiter: false,
unmarshaled: true,
})
for {
record, err = r.Read(record)
if err != nil {
break
}
opts := sql.WriteCSVOpts{
FieldDelimiter: []rune(c.fieldDelimiter)[0],
Quote: '"',
QuoteEscape: '"',
AlwaysQuote: false,
}
record.WriteCSV(&result, opts)
result.Truncate(result.Len() - 1)
result.WriteString(c.recordDelimiter)
}
r.Close()
if err != io.EOF {
t.Fatalf("Case %d failed with %s", i, err)
}
if result.String() != c.content {
t.Errorf("Case %d failed: expected %v result %v", i, c.content, result.String())
}
}
}
type tester interface {
Fatal(...interface{})
}
func openTestFile(t tester, file string) []byte {
f, err := ioutil.ReadFile(filepath.Join("testdata/testdata.zip"))
if err != nil {
t.Fatal(err)
}
z, err := zip.NewReader(bytes.NewReader(f), int64(len(f)))
if err != nil {
t.Fatal(err)
}
for _, f := range z.File {
if f.Name == file {
rc, err := f.Open()
if err != nil {
t.Fatal(err)
}
defer rc.Close()
b, err := ioutil.ReadAll(rc)
if err != nil {
t.Fatal(err)
}
return b
}
}
t.Fatal(file, "not found in testdata/testdata.zip")
return nil
}
func TestReadExtended(t *testing.T) {
cases := []struct {
file string
recordDelimiter string
fieldDelimiter string
header bool
wantColumns []string
wantTenFields string
totalFields int
}{
{
file: "nyc-taxi-data-100k.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
header: true,
wantColumns: []string{"trip_id", "vendor_id", "pickup_datetime", "dropoff_datetime", "store_and_fwd_flag", "rate_code_id", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "passenger_count", "trip_distance", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "ehail_fee", "improvement_surcharge", "total_amount", "payment_type", "trip_type", "pickup", "dropoff", "cab_type", "precipitation", "snow_depth", "snowfall", "max_temp", "min_temp", "wind", "pickup_nyct2010_gid", "pickup_ctlabel", "pickup_borocode", "pickup_boroname", "pickup_ct2010", "pickup_boroct2010", "pickup_cdeligibil", "pickup_ntacode", "pickup_ntaname", "pickup_puma", "dropoff_nyct2010_gid", "dropoff_ctlabel", "dropoff_borocode", "dropoff_boroname", "dropoff_ct2010", "dropoff_boroct2010", "dropoff_cdeligibil", "dropoff_ntacode", "dropoff_ntaname", "dropoff_puma"},
wantTenFields: `3389224,2,2014-03-26 00:26:15,2014-03-26 00:28:38,N,1,-73.950431823730469,40.792251586914063,-73.938949584960937,40.794425964355469,1,0.84,4.5,0.5,0.5,1,0,,,6.5,1,1,75,74,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1828,180,1,Manhattan,018000,1018000,E,MN34,East Harlem North,3804
3389225,2,2014-03-31 09:42:15,2014-03-31 10:01:17,N,1,-73.950340270996094,40.792228698730469,-73.941970825195313,40.842235565185547,1,4.47,17.5,0,0.5,0,0,,,18,2,1,75,244,green,0.16,0.0,0.0,56,36,8.28,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,911,251,1,Manhattan,025100,1025100,E,MN36,Washington Heights South,3801
3389226,2,2014-03-26 17:13:28,2014-03-26 17:19:07,N,1,-73.949493408203125,40.793506622314453,-73.943374633789063,40.786155700683594,1,0.82,5.5,1,0.5,0,0,,,7,1,1,75,75,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1387,164,1,Manhattan,016400,1016400,E,MN33,East Harlem South,3804
3389227,2,2014-03-14 21:07:19,2014-03-14 21:11:41,N,1,-73.950538635253906,40.792228698730469,-73.940811157226563,40.809253692626953,1,1.40,6,0.5,0.5,0,0,,,7,2,1,75,42,green,0.00,0.0,0.0,46,22,5.59,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1184,208,1,Manhattan,020800,1020800,E,MN03,Central Harlem North-Polo Grounds,3803
3389228,1,2014-03-28 13:52:56,2014-03-28 14:29:01,N,1,-73.950569152832031,40.792312622070313,-73.868507385253906,40.688491821289063,2,16.10,46,0,0.5,0,5.33,,,51.83,2,,75,63,green,0.04,0.0,0.0,62,37,5.37,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1544,1182.02,3,Brooklyn,118202,3118202,E,BK83,Cypress Hills-City Line,4008
3389229,2,2014-03-07 09:46:32,2014-03-07 09:55:01,N,1,-73.952301025390625,40.789798736572266,-73.935806274414062,40.794448852539063,1,1.67,8,0,0.5,2,0,,,10.5,1,1,75,74,green,0.00,3.9,0.0,37,26,7.83,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1553,178,1,Manhattan,017800,1017800,E,MN34,East Harlem North,3804
3389230,2,2014-03-17 18:23:05,2014-03-17 18:28:38,N,1,-73.952346801757813,40.789844512939453,-73.946319580078125,40.783851623535156,5,0.95,5.5,1,0.5,0.65,0,,,7.65,1,1,75,263,green,0.00,0.0,0.0,35,23,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,32,156.01,1,Manhattan,015601,1015601,I,MN32,Yorkville,3805
3389231,1,2014-03-19 19:09:36,2014-03-19 19:12:20,N,1,-73.952377319335938,40.789779663085938,-73.947494506835938,40.796474456787109,1,0.50,4,1,0.5,1,0,,,6.5,1,,75,75,green,0.92,0.0,0.0,46,32,7.16,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1401,174.02,1,Manhattan,017402,1017402,E,MN33,East Harlem South,3804
3389232,2,2014-03-20 19:06:28,2014-03-20 19:21:35,N,1,-73.952583312988281,40.789516448974609,-73.985870361328125,40.776973724365234,2,3.04,13,1,0.5,2.8,0,,,17.3,1,1,75,143,green,0.00,0.0,0.0,54,40,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1742,155,1,Manhattan,015500,1015500,I,MN14,Lincoln Square,3806
3389233,2,2014-03-29 09:38:12,2014-03-29 09:44:16,N,1,-73.952728271484375,40.789501190185547,-73.950935363769531,40.775600433349609,1,1.10,6.5,0,0.5,1.3,0,,,8.3,1,1,75,263,green,1.81,0.0,0.0,59,43,10.74,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,2048,138,1,Manhattan,013800,1013800,I,MN32,Yorkville,3805
`,
totalFields: 308*2 + 1,
}, {
file: "nyc-taxi-data-tabs-100k.csv",
recordDelimiter: "\n",
fieldDelimiter: "\t",
header: true,
wantColumns: []string{"trip_id", "vendor_id", "pickup_datetime", "dropoff_datetime", "store_and_fwd_flag", "rate_code_id", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "passenger_count", "trip_distance", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "ehail_fee", "improvement_surcharge", "total_amount", "payment_type", "trip_type", "pickup", "dropoff", "cab_type", "precipitation", "snow_depth", "snowfall", "max_temp", "min_temp", "wind", "pickup_nyct2010_gid", "pickup_ctlabel", "pickup_borocode", "pickup_boroname", "pickup_ct2010", "pickup_boroct2010", "pickup_cdeligibil", "pickup_ntacode", "pickup_ntaname", "pickup_puma", "dropoff_nyct2010_gid", "dropoff_ctlabel", "dropoff_borocode", "dropoff_boroname", "dropoff_ct2010", "dropoff_boroct2010", "dropoff_cdeligibil", "dropoff_ntacode", "dropoff_ntaname", "dropoff_puma"},
wantTenFields: `3389224,2,2014-03-26 00:26:15,2014-03-26 00:28:38,N,1,-73.950431823730469,40.792251586914063,-73.938949584960937,40.794425964355469,1,0.84,4.5,0.5,0.5,1,0,,,6.5,1,1,75,74,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1828,180,1,Manhattan,018000,1018000,E,MN34,East Harlem North,3804
3389225,2,2014-03-31 09:42:15,2014-03-31 10:01:17,N,1,-73.950340270996094,40.792228698730469,-73.941970825195313,40.842235565185547,1,4.47,17.5,0,0.5,0,0,,,18,2,1,75,244,green,0.16,0.0,0.0,56,36,8.28,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,911,251,1,Manhattan,025100,1025100,E,MN36,Washington Heights South,3801
3389226,2,2014-03-26 17:13:28,2014-03-26 17:19:07,N,1,-73.949493408203125,40.793506622314453,-73.943374633789063,40.786155700683594,1,0.82,5.5,1,0.5,0,0,,,7,1,1,75,75,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1387,164,1,Manhattan,016400,1016400,E,MN33,East Harlem South,3804
3389227,2,2014-03-14 21:07:19,2014-03-14 21:11:41,N,1,-73.950538635253906,40.792228698730469,-73.940811157226563,40.809253692626953,1,1.40,6,0.5,0.5,0,0,,,7,2,1,75,42,green,0.00,0.0,0.0,46,22,5.59,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1184,208,1,Manhattan,020800,1020800,E,MN03,Central Harlem North-Polo Grounds,3803
3389228,1,2014-03-28 13:52:56,2014-03-28 14:29:01,N,1,-73.950569152832031,40.792312622070313,-73.868507385253906,40.688491821289063,2,16.10,46,0,0.5,0,5.33,,,51.83,2,,75,63,green,0.04,0.0,0.0,62,37,5.37,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1544,1182.02,3,Brooklyn,118202,3118202,E,BK83,Cypress Hills-City Line,4008
3389229,2,2014-03-07 09:46:32,2014-03-07 09:55:01,N,1,-73.952301025390625,40.789798736572266,-73.935806274414062,40.794448852539063,1,1.67,8,0,0.5,2,0,,,10.5,1,1,75,74,green,0.00,3.9,0.0,37,26,7.83,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1553,178,1,Manhattan,017800,1017800,E,MN34,East Harlem North,3804
3389230,2,2014-03-17 18:23:05,2014-03-17 18:28:38,N,1,-73.952346801757813,40.789844512939453,-73.946319580078125,40.783851623535156,5,0.95,5.5,1,0.5,0.65,0,,,7.65,1,1,75,263,green,0.00,0.0,0.0,35,23,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,32,156.01,1,Manhattan,015601,1015601,I,MN32,Yorkville,3805
3389231,1,2014-03-19 19:09:36,2014-03-19 19:12:20,N,1,-73.952377319335938,40.789779663085938,-73.947494506835938,40.796474456787109,1,0.50,4,1,0.5,1,0,,,6.5,1,,75,75,green,0.92,0.0,0.0,46,32,7.16,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1401,174.02,1,Manhattan,017402,1017402,E,MN33,East Harlem South,3804
3389232,2,2014-03-20 19:06:28,2014-03-20 19:21:35,N,1,-73.952583312988281,40.789516448974609,-73.985870361328125,40.776973724365234,2,3.04,13,1,0.5,2.8,0,,,17.3,1,1,75,143,green,0.00,0.0,0.0,54,40,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1742,155,1,Manhattan,015500,1015500,I,MN14,Lincoln Square,3806
3389233,2,2014-03-29 09:38:12,2014-03-29 09:44:16,N,1,-73.952728271484375,40.789501190185547,-73.950935363769531,40.775600433349609,1,1.10,6.5,0,0.5,1.3,0,,,8.3,1,1,75,263,green,1.81,0.0,0.0,59,43,10.74,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,2048,138,1,Manhattan,013800,1013800,I,MN32,Yorkville,3805
`,
totalFields: 308*2 + 1,
}, {
file: "nyc-taxi-data-100k-single-delim.csv",
recordDelimiter: "^",
fieldDelimiter: ",",
header: true,
wantColumns: []string{"trip_id", "vendor_id", "pickup_datetime", "dropoff_datetime", "store_and_fwd_flag", "rate_code_id", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "passenger_count", "trip_distance", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "ehail_fee", "improvement_surcharge", "total_amount", "payment_type", "trip_type", "pickup", "dropoff", "cab_type", "precipitation", "snow_depth", "snowfall", "max_temp", "min_temp", "wind", "pickup_nyct2010_gid", "pickup_ctlabel", "pickup_borocode", "pickup_boroname", "pickup_ct2010", "pickup_boroct2010", "pickup_cdeligibil", "pickup_ntacode", "pickup_ntaname", "pickup_puma", "dropoff_nyct2010_gid", "dropoff_ctlabel", "dropoff_borocode", "dropoff_boroname", "dropoff_ct2010", "dropoff_boroct2010", "dropoff_cdeligibil", "dropoff_ntacode", "dropoff_ntaname", "dropoff_puma"},
wantTenFields: `3389224,2,2014-03-26 00:26:15,2014-03-26 00:28:38,N,1,-73.950431823730469,40.792251586914063,-73.938949584960937,40.794425964355469,1,0.84,4.5,0.5,0.5,1,0,,,6.5,1,1,75,74,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1828,180,1,Manhattan,018000,1018000,E,MN34,East Harlem North,3804
3389225,2,2014-03-31 09:42:15,2014-03-31 10:01:17,N,1,-73.950340270996094,40.792228698730469,-73.941970825195313,40.842235565185547,1,4.47,17.5,0,0.5,0,0,,,18,2,1,75,244,green,0.16,0.0,0.0,56,36,8.28,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,911,251,1,Manhattan,025100,1025100,E,MN36,Washington Heights South,3801
3389226,2,2014-03-26 17:13:28,2014-03-26 17:19:07,N,1,-73.949493408203125,40.793506622314453,-73.943374633789063,40.786155700683594,1,0.82,5.5,1,0.5,0,0,,,7,1,1,75,75,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1387,164,1,Manhattan,016400,1016400,E,MN33,East Harlem South,3804
3389227,2,2014-03-14 21:07:19,2014-03-14 21:11:41,N,1,-73.950538635253906,40.792228698730469,-73.940811157226563,40.809253692626953,1,1.40,6,0.5,0.5,0,0,,,7,2,1,75,42,green,0.00,0.0,0.0,46,22,5.59,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1184,208,1,Manhattan,020800,1020800,E,MN03,Central Harlem North-Polo Grounds,3803
3389228,1,2014-03-28 13:52:56,2014-03-28 14:29:01,N,1,-73.950569152832031,40.792312622070313,-73.868507385253906,40.688491821289063,2,16.10,46,0,0.5,0,5.33,,,51.83,2,,75,63,green,0.04,0.0,0.0,62,37,5.37,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1544,1182.02,3,Brooklyn,118202,3118202,E,BK83,Cypress Hills-City Line,4008
3389229,2,2014-03-07 09:46:32,2014-03-07 09:55:01,N,1,-73.952301025390625,40.789798736572266,-73.935806274414062,40.794448852539063,1,1.67,8,0,0.5,2,0,,,10.5,1,1,75,74,green,0.00,3.9,0.0,37,26,7.83,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1553,178,1,Manhattan,017800,1017800,E,MN34,East Harlem North,3804
3389230,2,2014-03-17 18:23:05,2014-03-17 18:28:38,N,1,-73.952346801757813,40.789844512939453,-73.946319580078125,40.783851623535156,5,0.95,5.5,1,0.5,0.65,0,,,7.65,1,1,75,263,green,0.00,0.0,0.0,35,23,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,32,156.01,1,Manhattan,015601,1015601,I,MN32,Yorkville,3805
3389231,1,2014-03-19 19:09:36,2014-03-19 19:12:20,N,1,-73.952377319335938,40.789779663085938,-73.947494506835938,40.796474456787109,1,0.50,4,1,0.5,1,0,,,6.5,1,,75,75,green,0.92,0.0,0.0,46,32,7.16,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1401,174.02,1,Manhattan,017402,1017402,E,MN33,East Harlem South,3804
3389232,2,2014-03-20 19:06:28,2014-03-20 19:21:35,N,1,-73.952583312988281,40.789516448974609,-73.985870361328125,40.776973724365234,2,3.04,13,1,0.5,2.8,0,,,17.3,1,1,75,143,green,0.00,0.0,0.0,54,40,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1742,155,1,Manhattan,015500,1015500,I,MN14,Lincoln Square,3806
3389233,2,2014-03-29 09:38:12,2014-03-29 09:44:16,N,1,-73.952728271484375,40.789501190185547,-73.950935363769531,40.775600433349609,1,1.10,6.5,0,0.5,1.3,0,,,8.3,1,1,75,263,green,1.81,0.0,0.0,59,43,10.74,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,2048,138,1,Manhattan,013800,1013800,I,MN32,Yorkville,3805
`,
totalFields: 308*2 + 1,
}, {
file: "nyc-taxi-data-100k-multi-delim.csv",
recordDelimiter: "^Y",
fieldDelimiter: ",",
header: true,
wantColumns: []string{"trip_id", "vendor_id", "pickup_datetime", "dropoff_datetime", "store_and_fwd_flag", "rate_code_id", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "passenger_count", "trip_distance", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "ehail_fee", "improvement_surcharge", "total_amount", "payment_type", "trip_type", "pickup", "dropoff", "cab_type", "precipitation", "snow_depth", "snowfall", "max_temp", "min_temp", "wind", "pickup_nyct2010_gid", "pickup_ctlabel", "pickup_borocode", "pickup_boroname", "pickup_ct2010", "pickup_boroct2010", "pickup_cdeligibil", "pickup_ntacode", "pickup_ntaname", "pickup_puma", "dropoff_nyct2010_gid", "dropoff_ctlabel", "dropoff_borocode", "dropoff_boroname", "dropoff_ct2010", "dropoff_boroct2010", "dropoff_cdeligibil", "dropoff_ntacode", "dropoff_ntaname", "dropoff_puma"},
wantTenFields: `3389224,2,2014-03-26 00:26:15,2014-03-26 00:28:38,N,1,-73.950431823730469,40.792251586914063,-73.938949584960937,40.794425964355469,1,0.84,4.5,0.5,0.5,1,0,,,6.5,1,1,75,74,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1828,180,1,Manhattan,018000,1018000,E,MN34,East Harlem North,3804
3389225,2,2014-03-31 09:42:15,2014-03-31 10:01:17,N,1,-73.950340270996094,40.792228698730469,-73.941970825195313,40.842235565185547,1,4.47,17.5,0,0.5,0,0,,,18,2,1,75,244,green,0.16,0.0,0.0,56,36,8.28,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,911,251,1,Manhattan,025100,1025100,E,MN36,Washington Heights South,3801
3389226,2,2014-03-26 17:13:28,2014-03-26 17:19:07,N,1,-73.949493408203125,40.793506622314453,-73.943374633789063,40.786155700683594,1,0.82,5.5,1,0.5,0,0,,,7,1,1,75,75,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1387,164,1,Manhattan,016400,1016400,E,MN33,East Harlem South,3804
3389227,2,2014-03-14 21:07:19,2014-03-14 21:11:41,N,1,-73.950538635253906,40.792228698730469,-73.940811157226563,40.809253692626953,1,1.40,6,0.5,0.5,0,0,,,7,2,1,75,42,green,0.00,0.0,0.0,46,22,5.59,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1184,208,1,Manhattan,020800,1020800,E,MN03,Central Harlem North-Polo Grounds,3803
3389228,1,2014-03-28 13:52:56,2014-03-28 14:29:01,N,1,-73.950569152832031,40.792312622070313,-73.868507385253906,40.688491821289063,2,16.10,46,0,0.5,0,5.33,,,51.83,2,,75,63,green,0.04,0.0,0.0,62,37,5.37,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1544,1182.02,3,Brooklyn,118202,3118202,E,BK83,Cypress Hills-City Line,4008
3389229,2,2014-03-07 09:46:32,2014-03-07 09:55:01,N,1,-73.952301025390625,40.789798736572266,-73.935806274414062,40.794448852539063,1,1.67,8,0,0.5,2,0,,,10.5,1,1,75,74,green,0.00,3.9,0.0,37,26,7.83,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1553,178,1,Manhattan,017800,1017800,E,MN34,East Harlem North,3804
3389230,2,2014-03-17 18:23:05,2014-03-17 18:28:38,N,1,-73.952346801757813,40.789844512939453,-73.946319580078125,40.783851623535156,5,0.95,5.5,1,0.5,0.65,0,,,7.65,1,1,75,263,green,0.00,0.0,0.0,35,23,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,32,156.01,1,Manhattan,015601,1015601,I,MN32,Yorkville,3805
3389231,1,2014-03-19 19:09:36,2014-03-19 19:12:20,N,1,-73.952377319335938,40.789779663085938,-73.947494506835938,40.796474456787109,1,0.50,4,1,0.5,1,0,,,6.5,1,,75,75,green,0.92,0.0,0.0,46,32,7.16,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1401,174.02,1,Manhattan,017402,1017402,E,MN33,East Harlem South,3804
3389232,2,2014-03-20 19:06:28,2014-03-20 19:21:35,N,1,-73.952583312988281,40.789516448974609,-73.985870361328125,40.776973724365234,2,3.04,13,1,0.5,2.8,0,,,17.3,1,1,75,143,green,0.00,0.0,0.0,54,40,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1742,155,1,Manhattan,015500,1015500,I,MN14,Lincoln Square,3806
3389233,2,2014-03-29 09:38:12,2014-03-29 09:44:16,N,1,-73.952728271484375,40.789501190185547,-73.950935363769531,40.775600433349609,1,1.10,6.5,0,0.5,1.3,0,,,8.3,1,1,75,263,green,1.81,0.0,0.0,59,43,10.74,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,2048,138,1,Manhattan,013800,1013800,I,MN32,Yorkville,3805
`,
totalFields: 308*2 + 1,
}, {
file: "nyc-taxi-data-noheader-100k.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
header: false,
wantColumns: []string{"_1", "_2", "_3", "_4", "_5", "_6", "_7", "_8", "_9", "_10", "_11", "_12", "_13", "_14", "_15", "_16", "_17", "_18", "_19", "_20", "_21", "_22", "_23", "_24", "_25", "_26", "_27", "_28", "_29", "_30", "_31", "_32", "_33", "_34", "_35", "_36", "_37", "_38", "_39", "_40", "_41", "_42", "_43", "_44", "_45", "_46", "_47", "_48", "_49", "_50", "_51"},
wantTenFields: `3389224,2,2014-03-26 00:26:15,2014-03-26 00:28:38,N,1,-73.950431823730469,40.792251586914063,-73.938949584960937,40.794425964355469,1,0.84,4.5,0.5,0.5,1,0,,,6.5,1,1,75,74,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1828,180,1,Manhattan,018000,1018000,E,MN34,East Harlem North,3804
3389225,2,2014-03-31 09:42:15,2014-03-31 10:01:17,N,1,-73.950340270996094,40.792228698730469,-73.941970825195313,40.842235565185547,1,4.47,17.5,0,0.5,0,0,,,18,2,1,75,244,green,0.16,0.0,0.0,56,36,8.28,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,911,251,1,Manhattan,025100,1025100,E,MN36,Washington Heights South,3801
3389226,2,2014-03-26 17:13:28,2014-03-26 17:19:07,N,1,-73.949493408203125,40.793506622314453,-73.943374633789063,40.786155700683594,1,0.82,5.5,1,0.5,0,0,,,7,1,1,75,75,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1387,164,1,Manhattan,016400,1016400,E,MN33,East Harlem South,3804
3389227,2,2014-03-14 21:07:19,2014-03-14 21:11:41,N,1,-73.950538635253906,40.792228698730469,-73.940811157226563,40.809253692626953,1,1.40,6,0.5,0.5,0,0,,,7,2,1,75,42,green,0.00,0.0,0.0,46,22,5.59,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1184,208,1,Manhattan,020800,1020800,E,MN03,Central Harlem North-Polo Grounds,3803
3389228,1,2014-03-28 13:52:56,2014-03-28 14:29:01,N,1,-73.950569152832031,40.792312622070313,-73.868507385253906,40.688491821289063,2,16.10,46,0,0.5,0,5.33,,,51.83,2,,75,63,green,0.04,0.0,0.0,62,37,5.37,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1544,1182.02,3,Brooklyn,118202,3118202,E,BK83,Cypress Hills-City Line,4008
3389229,2,2014-03-07 09:46:32,2014-03-07 09:55:01,N,1,-73.952301025390625,40.789798736572266,-73.935806274414062,40.794448852539063,1,1.67,8,0,0.5,2,0,,,10.5,1,1,75,74,green,0.00,3.9,0.0,37,26,7.83,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1553,178,1,Manhattan,017800,1017800,E,MN34,East Harlem North,3804
3389230,2,2014-03-17 18:23:05,2014-03-17 18:28:38,N,1,-73.952346801757813,40.789844512939453,-73.946319580078125,40.783851623535156,5,0.95,5.5,1,0.5,0.65,0,,,7.65,1,1,75,263,green,0.00,0.0,0.0,35,23,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,32,156.01,1,Manhattan,015601,1015601,I,MN32,Yorkville,3805
3389231,1,2014-03-19 19:09:36,2014-03-19 19:12:20,N,1,-73.952377319335938,40.789779663085938,-73.947494506835938,40.796474456787109,1,0.50,4,1,0.5,1,0,,,6.5,1,,75,75,green,0.92,0.0,0.0,46,32,7.16,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1401,174.02,1,Manhattan,017402,1017402,E,MN33,East Harlem South,3804
3389232,2,2014-03-20 19:06:28,2014-03-20 19:21:35,N,1,-73.952583312988281,40.789516448974609,-73.985870361328125,40.776973724365234,2,3.04,13,1,0.5,2.8,0,,,17.3,1,1,75,143,green,0.00,0.0,0.0,54,40,8.05,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1742,155,1,Manhattan,015500,1015500,I,MN14,Lincoln Square,3806
3389233,2,2014-03-29 09:38:12,2014-03-29 09:44:16,N,1,-73.952728271484375,40.789501190185547,-73.950935363769531,40.775600433349609,1,1.10,6.5,0,0.5,1.3,0,,,8.3,1,1,75,263,green,1.81,0.0,0.0,59,43,10.74,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,2048,138,1,Manhattan,013800,1013800,I,MN32,Yorkville,3805
`,
totalFields: 308 * 2,
},
}
for i, c := range cases {
t.Run(c.file, func(t *testing.T) {
var err error
var record sql.Record
var result bytes.Buffer
input := openTestFile(t, c.file)
// Get above block size.
input = append(input, input...)
args := ReaderArgs{
FileHeaderInfo: use,
RecordDelimiter: c.recordDelimiter,
FieldDelimiter: c.fieldDelimiter,
QuoteCharacter: defaultQuoteCharacter,
QuoteEscapeCharacter: defaultQuoteEscapeCharacter,
CommentCharacter: defaultCommentCharacter,
AllowQuotedRecordDelimiter: false,
unmarshaled: true,
}
if !c.header {
args.FileHeaderInfo = none
}
r, _ := NewReader(ioutil.NopCloser(bytes.NewReader(input)), &args)
fields := 0
for {
record, err = r.Read(record)
if err != nil {
break
}
if fields < 10 {
opts := sql.WriteCSVOpts{
FieldDelimiter: ',',
Quote: '"',
QuoteEscape: '"',
AlwaysQuote: false,
}
// Write with fixed delimiters, newlines.
err := record.WriteCSV(&result, opts)
if err != nil {
t.Error(err)
}
}
fields++
}
r.Close()
if err != io.EOF {
t.Fatalf("Case %d failed with %s", i, err)
}
if !reflect.DeepEqual(r.columnNames, c.wantColumns) {
t.Errorf("Case %d failed: expected %#v, got result %#v", i, c.wantColumns, r.columnNames)
}
if result.String() != c.wantTenFields {
t.Errorf("Case %d failed: expected %v, got result %v", i, c.wantTenFields, result.String())
}
if fields != c.totalFields {
t.Errorf("Case %d failed: expected %v results %v", i, c.totalFields, fields)
}
})
}
}
type errReader struct {
err error
}
func (e errReader) Read(p []byte) (n int, err error) {
return 0, e.err
}
func TestReadFailures(t *testing.T) {
customErr := errors.New("unable to read file :(")
cases := []struct {
file string
recordDelimiter string
fieldDelimiter string
sendErr error
header bool
wantColumns []string
wantFields string
wantErr error
}{
{
file: "truncated-records.csv",
recordDelimiter: "^Y",
fieldDelimiter: ",",
header: true,
wantColumns: []string{"trip_id", "vendor_id", "pickup_datetime", "dropoff_datetime", "store_and_fwd_flag", "rate_code_id", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "passenger_count", "trip_distance", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "ehail_fee", "improvement_surcharge", "total_amount", "payment_type", "trip_type", "pickup", "dropoff", "cab_type", "precipitation", "snow_depth", "snowfall", "max_temp", "min_temp", "wind", "pickup_nyct2010_gid", "pickup_ctlabel", "pickup_borocode", "pickup_boroname", "pickup_ct2010", "pickup_boroct2010", "pickup_cdeligibil", "pickup_ntacode", "pickup_ntaname", "pickup_puma", "dropoff_nyct2010_gid", "dropoff_ctlabel", "dropoff_borocode", "dropoff_boroname", "dropoff_ct2010", "dropoff_boroct2010", "dropoff_cdeligibil", "dropoff_ntacode", "dropoff_ntaname", "dropoff_puma"},
wantFields: `3389224,2,2014-03-26 00:26:15,2014-03-26 00:28:38,N,1,-73.950431823730469,40.792251586914063,-73.938949584960937,40.794425964355469,1,0.84,4.5,0.5,0.5,1,0,,,6.5,1,1,75,74,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1828,180,1,Manhattan,018000,1018000,E,MN34,East Harlem North,3804
3389225,2,2014-03-31 09:42:15,2014-03-31 10:01:17,N,1,-73.950340270996094,40.792228698730469,-73.941970825195313,40.842235565185547,1,4.47,17.5,0,0.5,0,0,,,18,2,1,75,244,green,0.16,0.0,0.0,56,36,8.28,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,911,251,1,Manhattan,025100
`,
wantErr: io.EOF,
},
{
file: "truncated-records.csv",
recordDelimiter: "^Y",
fieldDelimiter: ",",
sendErr: customErr,
header: true,
wantColumns: []string{"trip_id", "vendor_id", "pickup_datetime", "dropoff_datetime", "store_and_fwd_flag", "rate_code_id", "pickup_longitude", "pickup_latitude", "dropoff_longitude", "dropoff_latitude", "passenger_count", "trip_distance", "fare_amount", "extra", "mta_tax", "tip_amount", "tolls_amount", "ehail_fee", "improvement_surcharge", "total_amount", "payment_type", "trip_type", "pickup", "dropoff", "cab_type", "precipitation", "snow_depth", "snowfall", "max_temp", "min_temp", "wind", "pickup_nyct2010_gid", "pickup_ctlabel", "pickup_borocode", "pickup_boroname", "pickup_ct2010", "pickup_boroct2010", "pickup_cdeligibil", "pickup_ntacode", "pickup_ntaname", "pickup_puma", "dropoff_nyct2010_gid", "dropoff_ctlabel", "dropoff_borocode", "dropoff_boroname", "dropoff_ct2010", "dropoff_boroct2010", "dropoff_cdeligibil", "dropoff_ntacode", "dropoff_ntaname", "dropoff_puma"},
wantFields: `3389224,2,2014-03-26 00:26:15,2014-03-26 00:28:38,N,1,-73.950431823730469,40.792251586914063,-73.938949584960937,40.794425964355469,1,0.84,4.5,0.5,0.5,1,0,,,6.5,1,1,75,74,green,0.00,0.0,0.0,36,24,11.86,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,1828,180,1,Manhattan,018000,1018000,E,MN34,East Harlem North,3804
3389225,2,2014-03-31 09:42:15,2014-03-31 10:01:17,N,1,-73.950340270996094,40.792228698730469,-73.941970825195313,40.842235565185547,1,4.47,17.5,0,0.5,0,0,,,18,2,1,75,244,green,0.16,0.0,0.0,56,36,8.28,1267,168,1,Manhattan,016800,1016800,E,MN33,East Harlem South,3804,911,251,1,Manhattan,025100
`,
wantErr: customErr,
},
{
// This works since LazyQuotes is true:
file: "invalid-badbarequote.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `"a ""word""",b` + "\n",
wantErr: io.EOF,
},
{
// This works since LazyQuotes is true:
file: "invalid-baddoubleq.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `"a""""b",c` + "\n",
wantErr: io.EOF,
},
{
// This works since LazyQuotes is true:
file: "invalid-badextraq.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `a word,"b"""` + "\n",
wantErr: io.EOF,
},
{
// This works since LazyQuotes is true:
file: "invalid-badstartline.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `a,"b` + "\n" + `c""d,e` + "\n\"\n",
wantErr: io.EOF,
},
{
// This works since LazyQuotes is true:
file: "invalid-badstartline2.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `a,b` + "\n" + `"d` + "\n\ne\"\n",
wantErr: io.EOF,
},
{
// This works since LazyQuotes is true:
file: "invalid-badtrailingq.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `a word,"b"""` + "\n",
wantErr: io.EOF,
},
{
// This works since LazyQuotes is true:
file: "invalid-crlfquoted.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `"foo""bar"` + "\n",
wantErr: io.EOF,
},
{
// This works since LazyQuotes is true:
file: "invalid-csv.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `"a""""b",c` + "\n",
wantErr: io.EOF,
},
{
// This works since LazyQuotes is true, but output is very weird.
file: "invalid-oddquote.csv",
recordDelimiter: "\n",
fieldDelimiter: ",",
sendErr: nil,
header: true,
wantColumns: []string{"header1", "header2", "header3"},
wantFields: "ok1,ok2,ok3\n" + `""""""",b,c` + "\n\"\n",
wantErr: io.EOF,
},
{
// Test when file ends with a half separator
file: "endswithhalfsep.csv",
recordDelimiter: "%!",
fieldDelimiter: ",",
sendErr: nil,
header: false,
wantColumns: []string{"_1", "_2", "_3"},
wantFields: "a,b,c\na2,b2,c2%\n",
wantErr: io.EOF,
},
}
for i, c := range cases {
t.Run(c.file, func(t *testing.T) {
var err error
var record sql.Record
var result bytes.Buffer
input := openTestFile(t, c.file)
args := ReaderArgs{
FileHeaderInfo: use,
RecordDelimiter: c.recordDelimiter,
FieldDelimiter: c.fieldDelimiter,
QuoteCharacter: defaultQuoteCharacter,
QuoteEscapeCharacter: defaultQuoteEscapeCharacter,
CommentCharacter: defaultCommentCharacter,
AllowQuotedRecordDelimiter: false,
unmarshaled: true,
}
if !c.header {
args.FileHeaderInfo = none
}
inr := io.Reader(bytes.NewReader(input))
if c.sendErr != nil {
inr = io.MultiReader(inr, errReader{c.sendErr})
}
r, _ := NewReader(ioutil.NopCloser(inr), &args)
fields := 0
for {
record, err = r.Read(record)
if err != nil {
break
}
opts := sql.WriteCSVOpts{
FieldDelimiter: ',',
Quote: '"',
QuoteEscape: '"',
AlwaysQuote: false,
}
// Write with fixed delimiters, newlines.
err := record.WriteCSV(&result, opts)
if err != nil {
t.Error(err)
}
fields++
}
r.Close()
if err != c.wantErr {
t.Fatalf("Case %d failed with %s", i, err)
}
if !reflect.DeepEqual(r.columnNames, c.wantColumns) {
t.Errorf("Case %d failed: expected \n%#v, got result \n%#v", i, c.wantColumns, r.columnNames)
}
if result.String() != c.wantFields {
t.Errorf("Case %d failed: expected \n%v\nGot result \n%v", i, c.wantFields, result.String())
}
})
}
}
func BenchmarkReaderBasic(b *testing.B) {
args := ReaderArgs{
FileHeaderInfo: use,
RecordDelimiter: "\n",
FieldDelimiter: ",",
QuoteCharacter: defaultQuoteCharacter,
QuoteEscapeCharacter: defaultQuoteEscapeCharacter,
CommentCharacter: defaultCommentCharacter,
AllowQuotedRecordDelimiter: false,
unmarshaled: true,
}
f := openTestFile(b, "nyc-taxi-data-100k.csv")
r, err := NewReader(ioutil.NopCloser(bytes.NewBuffer(f)), &args)
if err != nil {
b.Fatalf("Reading init failed with %s", err)
}
defer r.Close()
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(int64(len(f)))
var record sql.Record
for i := 0; i < b.N; i++ {
r, err = NewReader(ioutil.NopCloser(bytes.NewBuffer(f)), &args)
if err != nil {
b.Fatalf("Reading init failed with %s", err)
}
for err == nil {
record, err = r.Read(record)
if err != nil && err != io.EOF {
b.Fatalf("Reading failed with %s", err)
}
}
r.Close()
}
}
func BenchmarkReaderHuge(b *testing.B) {
args := ReaderArgs{
FileHeaderInfo: use,
RecordDelimiter: "\n",
FieldDelimiter: ",",
QuoteCharacter: defaultQuoteCharacter,
QuoteEscapeCharacter: defaultQuoteEscapeCharacter,
CommentCharacter: defaultCommentCharacter,
AllowQuotedRecordDelimiter: false,
unmarshaled: true,
}
for n := 0; n < 11; n++ {
f := openTestFile(b, "nyc-taxi-data-100k.csv")
want := 309
for i := 0; i < n; i++ {
f = append(f, f...)
want *= 2
}
b.Run(fmt.Sprint(len(f)/(1<<10), "K"), func(b *testing.B) {
b.ReportAllocs()
b.SetBytes(int64(len(f)))
b.ResetTimer()
var record sql.Record
for i := 0; i < b.N; i++ {
r, err := NewReader(ioutil.NopCloser(bytes.NewBuffer(f)), &args)
if err != nil {
b.Fatalf("Reading init failed with %s", err)
}
got := 0
for err == nil {
record, err = r.Read(record)
if err != nil && err != io.EOF {
b.Fatalf("Reading failed with %s", err)
}
got++
}
r.Close()
if got != want {
b.Errorf("want %d records, got %d", want, got)
}
}
})
}
}
func BenchmarkReaderReplace(b *testing.B) {
args := ReaderArgs{
FileHeaderInfo: use,
RecordDelimiter: "^",
FieldDelimiter: ",",
QuoteCharacter: defaultQuoteCharacter,
QuoteEscapeCharacter: defaultQuoteEscapeCharacter,
CommentCharacter: defaultCommentCharacter,
AllowQuotedRecordDelimiter: false,
unmarshaled: true,
}
f := openTestFile(b, "nyc-taxi-data-100k-single-delim.csv")
r, err := NewReader(ioutil.NopCloser(bytes.NewBuffer(f)), &args)
if err != nil {
b.Fatalf("Reading init failed with %s", err)
}
defer r.Close()
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(int64(len(f)))
var record sql.Record
for i := 0; i < b.N; i++ {
r, err = NewReader(ioutil.NopCloser(bytes.NewBuffer(f)), &args)
if err != nil {
b.Fatalf("Reading init failed with %s", err)
}
for err == nil {
record, err = r.Read(record)
if err != nil && err != io.EOF {
b.Fatalf("Reading failed with %s", err)
}
}
r.Close()
}
}
func BenchmarkReaderReplaceTwo(b *testing.B) {
args := ReaderArgs{
FileHeaderInfo: use,
RecordDelimiter: "^Y",
FieldDelimiter: ",",
QuoteCharacter: defaultQuoteCharacter,
QuoteEscapeCharacter: defaultQuoteEscapeCharacter,
CommentCharacter: defaultCommentCharacter,
AllowQuotedRecordDelimiter: false,
unmarshaled: true,
}
f := openTestFile(b, "nyc-taxi-data-100k-multi-delim.csv")
r, err := NewReader(ioutil.NopCloser(bytes.NewBuffer(f)), &args)
if err != nil {
b.Fatalf("Reading init failed with %s", err)
}
defer r.Close()
b.ReportAllocs()
b.ResetTimer()
b.SetBytes(int64(len(f)))
var record sql.Record
for i := 0; i < b.N; i++ {
r, err = NewReader(ioutil.NopCloser(bytes.NewBuffer(f)), &args)
if err != nil {
b.Fatalf("Reading init failed with %s", err)
}
for err == nil {
record, err = r.Read(record)
if err != nil && err != io.EOF {
b.Fatalf("Reading failed with %s", err)
}
}
r.Close()
}
}
internal/s3select/csv/record.go
+131
View File
@@ -0,0 +1,131 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package csv
import (
"encoding/json"
"errors"
"fmt"
"io"
"github.com/bcicen/jstream"
csv "github.com/minio/csvparser"
"github.com/minio/minio/internal/s3select/sql"
)
// Record - is a CSV record.
type Record struct {
columnNames []string
csvRecord []string
nameIndexMap map[string]int64
}
// Get - gets the value for a column name. CSV fields do not have any
// defined type (other than the default string). So this function
// always returns fields using sql.FromBytes so that the type
// specified/implied by the query can be used, or can be automatically
// converted based on the query.
func (r *Record) Get(name string) (*sql.Value, error) {
index, found := r.nameIndexMap[name]
if !found {
return nil, fmt.Errorf("column %v not found", name)
}
if index >= int64(len(r.csvRecord)) {
// No value found for column 'name', hence return null
// value
return sql.FromNull(), nil
}
return sql.FromBytes([]byte(r.csvRecord[index])), nil
}
// Set - sets the value for a column name.
func (r *Record) Set(name string, value *sql.Value) (sql.Record, error) {
r.columnNames = append(r.columnNames, name)
r.csvRecord = append(r.csvRecord, value.CSVString())
return r, nil
}
// Reset data in record.
func (r *Record) Reset() {
if len(r.columnNames) > 0 {
r.columnNames = r.columnNames[:0]
}
if len(r.csvRecord) > 0 {
r.csvRecord = r.csvRecord[:0]
}
for k := range r.nameIndexMap {
delete(r.nameIndexMap, k)
}
}
// Clone the record.
func (r *Record) Clone(dst sql.Record) sql.Record {
other, ok := dst.(*Record)
if !ok {
other = &Record{}
}
if len(other.columnNames) > 0 {
other.columnNames = other.columnNames[:0]
}
if len(other.csvRecord) > 0 {
other.csvRecord = other.csvRecord[:0]
}
other.columnNames = append(other.columnNames, r.columnNames...)
other.csvRecord = append(other.csvRecord, r.csvRecord...)
return other
}
// WriteCSV - encodes to CSV data.
func (r *Record) WriteCSV(writer io.Writer, opts sql.WriteCSVOpts) error {
w := csv.NewWriter(writer)
w.Comma = opts.FieldDelimiter
w.AlwaysQuote = opts.AlwaysQuote
w.Quote = opts.Quote
w.QuoteEscape = opts.QuoteEscape
if err := w.Write(r.csvRecord); err != nil {
return err
}
w.Flush()
return w.Error()
}
// WriteJSON - encodes to JSON data.
func (r *Record) WriteJSON(writer io.Writer) error {
var kvs jstream.KVS = make([]jstream.KV, len(r.columnNames))
for i := 0; i < len(r.columnNames); i++ {
kvs[i] = jstream.KV{Key: r.columnNames[i], Value: r.csvRecord[i]}
}
return json.NewEncoder(writer).Encode(kvs)
}
// Raw - returns the underlying data with format info.
func (r *Record) Raw() (sql.SelectObjectFormat, interface{}) {
return sql.SelectFmtCSV, r
}
// Replace - is not supported for CSV
func (r *Record) Replace(_ interface{}) error {
return errors.New("Replace is not supported for CSV")
}
// NewRecord - creates new CSV record.
func NewRecord() *Record {
return &Record{}
}
internal/s3select/csv/recordtransform.go
+94
View File
@@ -0,0 +1,94 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package csv
import (
"bytes"
"io"
)
// recordTransform will convert records to always have newline records.
type recordTransform struct {
reader io.Reader
// recordDelimiter can be up to 2 characters.
recordDelimiter []byte
oneByte []byte
useOneByte bool
}
func (rr *recordTransform) Read(p []byte) (n int, err error) {
if rr.useOneByte {
p[0] = rr.oneByte[0]
rr.useOneByte = false
n, err = rr.reader.Read(p[1:])
n++
} else {
n, err = rr.reader.Read(p)
}
if err != nil {
return n, err
}
// Do nothing if record-delimiter is already newline.
if string(rr.recordDelimiter) == "\n" {
return n, nil
}
// Change record delimiters to newline.
if len(rr.recordDelimiter) == 1 {
for idx := 0; idx < len(p); {
i := bytes.Index(p[idx:], rr.recordDelimiter)
if i < 0 {
break
}
idx += i
p[idx] = '\n'
}
return n, nil
}
// 2 characters...
for idx := 0; idx < len(p); {
i := bytes.Index(p[idx:], rr.recordDelimiter)
if i < 0 {
break
}
idx += i
p[idx] = '\n'
p = append(p[:idx+1], p[idx+2:]...)
n--
}
if p[n-1] != rr.recordDelimiter[0] {
return n, nil
}
if _, err = rr.reader.Read(rr.oneByte); err != nil {
return n, err
}
if rr.oneByte[0] == rr.recordDelimiter[1] {
p[n-1] = '\n'
return n, nil
}
rr.useOneByte = true
return n, nil
}
internal/s3select/csv/testdata/testdata.zip
Vendored
BIN
View File
Binary file not shown.
internal/s3select/errors.go
+145
View File
@@ -0,0 +1,145 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package s3select
// SelectError - represents s3 select error specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html#RESTObjectSELECTContent-responses-special-errors.
type SelectError interface {
Cause() error
ErrorCode() string
ErrorMessage() string
HTTPStatusCode() int
Error() string
}
type s3Error struct {
code string
message string
statusCode int
cause error
}
func (err *s3Error) Cause() error {
return err.cause
}
func (err *s3Error) ErrorCode() string {
return err.code
}
func (err *s3Error) ErrorMessage() string {
return err.message
}
func (err *s3Error) HTTPStatusCode() int {
return err.statusCode
}
func (err *s3Error) Error() string {
return err.message
}
func errMalformedXML(err error) *s3Error {
return &s3Error{
code: "MalformedXML",
message: "The XML provided was not well-formed or did not validate against our published schema. Check the service documentation and try again: " + err.Error(),
statusCode: 400,
cause: err,
}
}
func errInvalidCompressionFormat(err error) *s3Error {
return &s3Error{
code: "InvalidCompressionFormat",
message: "The file is not in a supported compression format. Only GZIP and BZIP2 are supported.",
statusCode: 400,
cause: err,
}
}
func errInvalidBZIP2CompressionFormat(err error) *s3Error {
return &s3Error{
code: "InvalidCompressionFormat",
message: "BZIP2 is not applicable to the queried object. Please correct the request and try again.",
statusCode: 400,
cause: err,
}
}
func errInvalidGZIPCompressionFormat(err error) *s3Error {
return &s3Error{
code: "InvalidCompressionFormat",
message: "GZIP is not applicable to the queried object. Please correct the request and try again.",
statusCode: 400,
cause: err,
}
}
func errInvalidDataSource(err error) *s3Error {
return &s3Error{
code: "InvalidDataSource",
message: "Invalid data source type. Only CSV, JSON, and Parquet are supported.",
statusCode: 400,
cause: err,
}
}
func errInvalidRequestParameter(err error) *s3Error {
return &s3Error{
code: "InvalidRequestParameter",
message: "The value of a parameter in SelectRequest element is invalid. Check the service API documentation and try again.",
statusCode: 400,
cause: err,
}
}
func errObjectSerializationConflict(err error) *s3Error {
return &s3Error{
code: "ObjectSerializationConflict",
message: "InputSerialization specifies more than one format (CSV, JSON, or Parquet), or OutputSerialization specifies more than one format (CSV or JSON). InputSerialization and OutputSerialization can only specify one format each.",
statusCode: 400,
cause: err,
}
}
func errInvalidExpressionType(err error) *s3Error {
return &s3Error{
code: "InvalidExpressionType",
message: "The ExpressionType is invalid. Only SQL expressions are supported.",
statusCode: 400,
cause: err,
}
}
func errMissingRequiredParameter(err error) *s3Error {
return &s3Error{
code: "MissingRequiredParameter",
message: "The SelectRequest entity is missing a required parameter. Check the service documentation and try again.",
statusCode: 400,
cause: err,
}
}
func errTruncatedInput(err error) *s3Error {
return &s3Error{
code: "TruncatedInput",
message: "Object decompression failed. Check that the object is properly compressed using the format specified in the request.",
statusCode: 400,
cause: err,
}
}
internal/s3select/genmessage.go
+183
View File
@@ -0,0 +1,183 @@
// +build ignore
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package s3select
import (
"bytes"
"encoding/binary"
"fmt"
"hash/crc32"
)
func genRecordsHeader() {
buf := new(bytes.Buffer)
buf.WriteByte(13)
buf.WriteString(":message-type")
buf.WriteByte(7)
buf.Write([]byte{0, 5})
buf.WriteString("event")
buf.WriteByte(13)
buf.WriteString(":content-type")
buf.WriteByte(7)
buf.Write([]byte{0, 24})
buf.WriteString("application/octet-stream")
buf.WriteByte(11)
buf.WriteString(":event-type")
buf.WriteByte(7)
buf.Write([]byte{0, 7})
buf.WriteString("Records")
fmt.Println(buf.Bytes())
}
// Continuation Message
// ====================
// Header specification
// --------------------
// Continuation messages contain two headers, as follows:
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-cont.png
//
// Payload specification
// ---------------------
// Continuation messages have no payload.
func genContinuationMessage() {
buf := new(bytes.Buffer)
buf.WriteByte(13)
buf.WriteString(":message-type")
buf.WriteByte(7)
buf.Write([]byte{0, 5})
buf.WriteString("event")
buf.WriteByte(11)
buf.WriteString(":event-type")
buf.WriteByte(7)
buf.Write([]byte{0, 4})
buf.WriteString("Cont")
header := buf.Bytes()
headerLength := len(header)
payloadLength := 0
totalLength := totalByteLength(headerLength, payloadLength)
buf = new(bytes.Buffer)
binary.Write(buf, binary.BigEndian, uint32(totalLength))
binary.Write(buf, binary.BigEndian, uint32(headerLength))
prelude := buf.Bytes()
binary.Write(buf, binary.BigEndian, crc32.ChecksumIEEE(prelude))
buf.Write(header)
message := buf.Bytes()
binary.Write(buf, binary.BigEndian, crc32.ChecksumIEEE(message))
fmt.Println(buf.Bytes())
}
func genProgressHeader() {
buf := new(bytes.Buffer)
buf.WriteByte(13)
buf.WriteString(":message-type")
buf.WriteByte(7)
buf.Write([]byte{0, 5})
buf.WriteString("event")
buf.WriteByte(13)
buf.WriteString(":content-type")
buf.WriteByte(7)
buf.Write([]byte{0, 8})
buf.WriteString("text/xml")
buf.WriteByte(11)
buf.WriteString(":event-type")
buf.WriteByte(7)
buf.Write([]byte{0, 8})
buf.WriteString("Progress")
fmt.Println(buf.Bytes())
}
func genStatsHeader() {
buf := new(bytes.Buffer)
buf.WriteByte(13)
buf.WriteString(":message-type")
buf.WriteByte(7)
buf.Write([]byte{0, 5})
buf.WriteString("event")
buf.WriteByte(13)
buf.WriteString(":content-type")
buf.WriteByte(7)
buf.Write([]byte{0, 8})
buf.WriteString("text/xml")
buf.WriteByte(11)
buf.WriteString(":event-type")
buf.WriteByte(7)
buf.Write([]byte{0, 5})
buf.WriteString("Stats")
fmt.Println(buf.Bytes())
}
// End Message
// ===========
// Header specification
// --------------------
// End messages contain two headers, as follows:
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-end.png
//
// Payload specification
// ---------------------
// End messages have no payload.
func genEndMessage() {
buf := new(bytes.Buffer)
buf.WriteByte(13)
buf.WriteString(":message-type")
buf.WriteByte(7)
buf.Write([]byte{0, 5})
buf.WriteString("event")
buf.WriteByte(11)
buf.WriteString(":event-type")
buf.WriteByte(7)
buf.Write([]byte{0, 3})
buf.WriteString("End")
header := buf.Bytes()
headerLength := len(header)
payloadLength := 0
totalLength := totalByteLength(headerLength, payloadLength)
buf = new(bytes.Buffer)
binary.Write(buf, binary.BigEndian, uint32(totalLength))
binary.Write(buf, binary.BigEndian, uint32(headerLength))
prelude := buf.Bytes()
binary.Write(buf, binary.BigEndian, crc32.ChecksumIEEE(prelude))
buf.Write(header)
message := buf.Bytes()
binary.Write(buf, binary.BigEndian, crc32.ChecksumIEEE(message))
fmt.Println(buf.Bytes())
}
internal/s3select/json/args.go
+96
View File
@@ -0,0 +1,96 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package json
import (
"encoding/xml"
"fmt"
"strings"
)
const (
document = "document"
lines = "lines"
defaultRecordDelimiter = "\n"
)
// ReaderArgs - represents elements inside <InputSerialization><JSON/> in request XML.
type ReaderArgs struct {
ContentType string `xml:"Type"`
unmarshaled bool
}
// IsEmpty - returns whether reader args is empty or not.
func (args *ReaderArgs) IsEmpty() bool {
return !args.unmarshaled
}
// UnmarshalXML - decodes XML data.
func (args *ReaderArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type subReaderArgs ReaderArgs
parsedArgs := subReaderArgs{}
if err := d.DecodeElement(&parsedArgs, &start); err != nil {
return err
}
parsedArgs.ContentType = strings.ToLower(parsedArgs.ContentType)
switch parsedArgs.ContentType {
case document, lines:
default:
return errInvalidJSONType(fmt.Errorf("invalid ContentType '%v'", parsedArgs.ContentType))
}
*args = ReaderArgs(parsedArgs)
args.unmarshaled = true
return nil
}
// WriterArgs - represents elements inside <OutputSerialization><JSON/> in request XML.
type WriterArgs struct {
RecordDelimiter string `xml:"RecordDelimiter"`
unmarshaled bool
}
// IsEmpty - returns whether writer args is empty or not.
func (args *WriterArgs) IsEmpty() bool {
return !args.unmarshaled
}
// UnmarshalXML - decodes XML data.
func (args *WriterArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type subWriterArgs WriterArgs
parsedArgs := subWriterArgs{}
if err := d.DecodeElement(&parsedArgs, &start); err != nil {
return err
}
switch len(parsedArgs.RecordDelimiter) {
case 0:
parsedArgs.RecordDelimiter = defaultRecordDelimiter
case 1, 2:
default:
return fmt.Errorf("invalid RecordDelimiter '%v'", parsedArgs.RecordDelimiter)
}
*args = WriterArgs(parsedArgs)
args.unmarshaled = true
return nil
}
internal/s3select/json/errors.go
+63
View File
@@ -0,0 +1,63 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package json
type s3Error struct {
code string
message string
statusCode int
cause error
}
func (err *s3Error) Cause() error {
return err.cause
}
func (err *s3Error) ErrorCode() string {
return err.code
}
func (err *s3Error) ErrorMessage() string {
return err.message
}
func (err *s3Error) HTTPStatusCode() int {
return err.statusCode
}
func (err *s3Error) Error() string {
return err.message
}
func errInvalidJSONType(err error) *s3Error {
return &s3Error{
code: "InvalidJsonType",
message: "The JsonType is invalid. Only DOCUMENT and LINES are supported.",
statusCode: 400,
cause: err,
}
}
func errJSONParsingError(err error) *s3Error {
return &s3Error{
code: "JSONParsingError",
message: "Encountered an error parsing the JSON file. Check the file and try again.",
statusCode: 400,
cause: err,
}
}
internal/s3select/json/preader.go
+228
View File
@@ -0,0 +1,228 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package json
import (
"bufio"
"bytes"
"io"
"runtime"
"sync"
"github.com/bcicen/jstream"
"github.com/minio/minio/internal/s3select/sql"
)
// PReader - JSON record reader for S3Select.
// Operates concurrently on line-delimited JSON.
type PReader struct {
args *ReaderArgs
readCloser io.ReadCloser // raw input
buf *bufio.Reader // input to the splitter
current []jstream.KVS // current block of results to be returned
recordsRead int // number of records read in current slice
input chan *queueItem // input for workers
queue chan *queueItem // output from workers in order
err error // global error state, only touched by Reader.Read
bufferPool sync.Pool // pool of []byte objects for input
kvDstPool sync.Pool // pool of []jstream.KV used for output
close chan struct{} // used for shutting down the splitter before end of stream
readerWg sync.WaitGroup // used to keep track of async reader.
}
// queueItem is an item in the queue.
type queueItem struct {
input []byte // raw input sent to the worker
dst chan []jstream.KVS // result of block decode
err error // any error encountered will be set here
}
// Read - reads single record.
// Once Read is called the previous record should no longer be referenced.
func (r *PReader) Read(dst sql.Record) (sql.Record, error) {
// If we have have any records left, return these before any error.
for len(r.current) <= r.recordsRead {
if r.err != nil {
return nil, r.err
}
// Move to next block
item, ok := <-r.queue
if !ok {
r.err = io.EOF
return nil, r.err
}
//lint:ignore SA6002 Using pointer would allocate more since we would have to copy slice header before taking a pointer.
r.kvDstPool.Put(r.current)
r.current = <-item.dst
r.err = item.err
r.recordsRead = 0
}
kvRecord := r.current[r.recordsRead]
r.recordsRead++
dstRec, ok := dst.(*Record)
if !ok {
dstRec = &Record{}
}
dstRec.KVS = kvRecord
dstRec.SelectFormat = sql.SelectFmtJSON
return dstRec, nil
}
// Close - closes underlying reader.
func (r *PReader) Close() error {
if r.close != nil {
close(r.close)
r.readerWg.Wait()
r.close = nil
}
r.recordsRead = len(r.current)
if r.err == nil {
r.err = io.EOF
}
return r.readCloser.Close()
}
// nextSplit will attempt to skip a number of bytes and
// return the buffer until the next newline occurs.
// The last block will be sent along with an io.EOF.
func (r *PReader) nextSplit(skip int, dst []byte) ([]byte, error) {
if cap(dst) < skip {
dst = make([]byte, 0, skip+1024)
}
dst = dst[:skip]
if skip > 0 {
n, err := io.ReadFull(r.buf, dst)
if err != nil && err != io.ErrUnexpectedEOF {
// If an EOF happens after reading some but not all the bytes,
// ReadFull returns ErrUnexpectedEOF.
return dst[:n], err
}
dst = dst[:n]
if err == io.ErrUnexpectedEOF {
return dst, io.EOF
}
}
// Read until next line.
in, err := r.buf.ReadBytes('\n')
dst = append(dst, in...)
return dst, err
}
// jsonSplitSize is the size of each block.
// Blocks will read this much and find the first following newline.
// 128KB appears to be a very reasonable default.
const jsonSplitSize = 128 << 10
// startReaders will read the header if needed and spin up a parser
// and a number of workers based on GOMAXPROCS.
// If an error is returned no goroutines have been started and r.err will have been set.
func (r *PReader) startReaders() {
r.bufferPool.New = func() interface{} {
return make([]byte, jsonSplitSize+1024)
}
// Create queue
r.queue = make(chan *queueItem, runtime.GOMAXPROCS(0))
r.input = make(chan *queueItem, runtime.GOMAXPROCS(0))
r.readerWg.Add(1)
// Start splitter
go func() {
defer close(r.input)
defer close(r.queue)
defer r.readerWg.Done()
for {
next, err := r.nextSplit(jsonSplitSize, r.bufferPool.Get().([]byte))
q := queueItem{
input: next,
dst: make(chan []jstream.KVS, 1),
err: err,
}
select {
case <-r.close:
return
case r.queue <- &q:
}
select {
case <-r.close:
return
case r.input <- &q:
}
if err != nil {
// Exit on any error.
return
}
}
}()
// Start parsers
for i := 0; i < runtime.GOMAXPROCS(0); i++ {
go func() {
for in := range r.input {
if len(in.input) == 0 {
in.dst <- nil
continue
}
dst, ok := r.kvDstPool.Get().([]jstream.KVS)
if !ok {
dst = make([]jstream.KVS, 0, 1000)
}
d := jstream.NewDecoder(bytes.NewBuffer(in.input), 0).ObjectAsKVS()
stream := d.Stream()
all := dst[:0]
for mv := range stream {
var kvs jstream.KVS
if mv.ValueType == jstream.Object {
// This is a JSON object type (that preserves key
// order)
kvs = mv.Value.(jstream.KVS)
} else {
// To be AWS S3 compatible Select for JSON needs to
// output non-object JSON as single column value
// i.e. a map with `_1` as key and value as the
// non-object.
kvs = jstream.KVS{jstream.KV{Key: "_1", Value: mv.Value}}
}
all = append(all, kvs)
}
// We don't need the input any more.
//lint:ignore SA6002 Using pointer would allocate more since we would have to copy slice header before taking a pointer.
r.bufferPool.Put(in.input)
in.input = nil
in.err = d.Err()
in.dst <- all
}
}()
}
}
// NewPReader - creates new parallel JSON reader using readCloser.
// Should only be used for LINES types.
func NewPReader(readCloser io.ReadCloser, args *ReaderArgs) *PReader {
r := &PReader{
args: args,
buf: bufio.NewReaderSize(readCloser, jsonSplitSize*2),
readCloser: readCloser,
close: make(chan struct{}),
}
r.startReaders()
return r
}
internal/s3select/json/preader_test.go
+107
View File
@@ -0,0 +1,107 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package json
import (
"bytes"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/minio/minio/internal/s3select/sql"
)
func TestNewPReader(t *testing.T) {
files, err := ioutil.ReadDir("testdata")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
t.Run(file.Name(), func(t *testing.T) {
f, err := os.Open(filepath.Join("testdata", file.Name()))
if err != nil {
t.Fatal(err)
}
r := NewPReader(f, &ReaderArgs{})
var record sql.Record
for {
record, err = r.Read(record)
if err != nil {
break
}
}
r.Close()
if err != io.EOF {
t.Fatalf("Reading failed with %s, %s", err, file.Name())
}
})
t.Run(file.Name()+"-close", func(t *testing.T) {
f, err := os.Open(filepath.Join("testdata", file.Name()))
if err != nil {
t.Fatal(err)
}
r := NewPReader(f, &ReaderArgs{})
r.Close()
var record sql.Record
for {
record, err = r.Read(record)
if err != nil {
break
}
}
if err != io.EOF {
t.Fatalf("Reading failed with %s, %s", err, file.Name())
}
})
}
}
func BenchmarkPReader(b *testing.B) {
files, err := ioutil.ReadDir("testdata")
if err != nil {
b.Fatal(err)
}
for _, file := range files {
b.Run(file.Name(), func(b *testing.B) {
f, err := ioutil.ReadFile(filepath.Join("testdata", file.Name()))
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(f)))
b.ReportAllocs()
b.ResetTimer()
var record sql.Record
for i := 0; i < b.N; i++ {
r := NewPReader(ioutil.NopCloser(bytes.NewBuffer(f)), &ReaderArgs{})
for {
record, err = r.Read(record)
if err != nil {
break
}
}
r.Close()
if err != io.EOF {
b.Fatalf("Reading failed with %s, %s", err, file.Name())
}
}
})
}
}
internal/s3select/json/reader.go
+134
View File
@@ -0,0 +1,134 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package json
import (
"errors"
"io"
"sync"
"github.com/minio/minio/internal/s3select/sql"
"github.com/bcicen/jstream"
)
// Reader - JSON record reader for S3Select.
type Reader struct {
args *ReaderArgs
decoder *jstream.Decoder
valueCh chan *jstream.MetaValue
readCloser io.ReadCloser
}
// Read - reads single record.
func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
v, ok := <-r.valueCh
if !ok {
if err := r.decoder.Err(); err != nil {
return nil, errJSONParsingError(err)
}
return nil, io.EOF
}
var kvs jstream.KVS
if v.ValueType == jstream.Object {
// This is a JSON object type (that preserves key
// order)
kvs = v.Value.(jstream.KVS)
} else {
// To be AWS S3 compatible Select for JSON needs to
// output non-object JSON as single column value
// i.e. a map with `_1` as key and value as the
// non-object.
kvs = jstream.KVS{jstream.KV{Key: "_1", Value: v.Value}}
}
dstRec, ok := dst.(*Record)
if !ok {
dstRec = &Record{}
}
dstRec.KVS = kvs
dstRec.SelectFormat = sql.SelectFmtJSON
return dstRec, nil
}
// Close - closes underlying reader.
func (r *Reader) Close() error {
// Close the input.
err := r.readCloser.Close()
for range r.valueCh {
// Drain values so we don't leak a goroutine.
// Since we have closed the input, it should fail rather quickly.
}
return err
}
// NewReader - creates new JSON reader using readCloser.
func NewReader(readCloser io.ReadCloser, args *ReaderArgs) *Reader {
readCloser = &syncReadCloser{rc: readCloser}
d := jstream.NewDecoder(readCloser, 0).ObjectAsKVS()
return &Reader{
args: args,
decoder: d,
valueCh: d.Stream(),
readCloser: readCloser,
}
}
// syncReadCloser will wrap a readcloser and make it safe to call Close
// while reads are running.
// All read errors are also postponed until Close is called and
// io.EOF is returned instead.
type syncReadCloser struct {
rc io.ReadCloser
errMu sync.Mutex
err error
}
func (pr *syncReadCloser) Read(p []byte) (n int, err error) {
// This ensures that Close will block until Read has completed.
// This allows another goroutine to close the reader.
pr.errMu.Lock()
defer pr.errMu.Unlock()
if pr.err != nil {
return 0, io.EOF
}
n, pr.err = pr.rc.Read(p)
if pr.err != nil {
// Translate any error into io.EOF, so we don't crash:
// https://github.com/bcicen/jstream/blob/master/scanner.go#L48
return n, io.EOF
}
return n, nil
}
var errClosed = errors.New("read after close")
func (pr *syncReadCloser) Close() error {
pr.errMu.Lock()
defer pr.errMu.Unlock()
if pr.err == errClosed {
return nil
}
if pr.err != nil {
return pr.err
}
pr.err = errClosed
return pr.rc.Close()
}
internal/s3select/json/reader_test.go
+107
View File
@@ -0,0 +1,107 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package json
import (
"bytes"
"io"
"io/ioutil"
"os"
"path/filepath"
"testing"
"github.com/minio/minio/internal/s3select/sql"
)
func TestNewReader(t *testing.T) {
files, err := ioutil.ReadDir("testdata")
if err != nil {
t.Fatal(err)
}
for _, file := range files {
t.Run(file.Name(), func(t *testing.T) {
f, err := os.Open(filepath.Join("testdata", file.Name()))
if err != nil {
t.Fatal(err)
}
r := NewReader(f, &ReaderArgs{})
var record sql.Record
for {
record, err = r.Read(record)
if err != nil {
break
}
}
r.Close()
if err != io.EOF {
t.Fatalf("Reading failed with %s, %s", err, file.Name())
}
})
t.Run(file.Name()+"-close", func(t *testing.T) {
f, err := os.Open(filepath.Join("testdata", file.Name()))
if err != nil {
t.Fatal(err)
}
r := NewReader(f, &ReaderArgs{})
r.Close()
var record sql.Record
for {
record, err = r.Read(record)
if err != nil {
break
}
}
if err != io.EOF {
t.Fatalf("Reading failed with %s, %s", err, file.Name())
}
})
}
}
func BenchmarkReader(b *testing.B) {
files, err := ioutil.ReadDir("testdata")
if err != nil {
b.Fatal(err)
}
for _, file := range files {
b.Run(file.Name(), func(b *testing.B) {
f, err := ioutil.ReadFile(filepath.Join("testdata", file.Name()))
if err != nil {
b.Fatal(err)
}
b.SetBytes(int64(len(f)))
b.ReportAllocs()
b.ResetTimer()
var record sql.Record
for i := 0; i < b.N; i++ {
r := NewReader(ioutil.NopCloser(bytes.NewBuffer(f)), &ReaderArgs{})
for {
record, err = r.Read(record)
if err != nil {
break
}
}
r.Close()
if err != io.EOF {
b.Fatalf("Reading failed with %s, %s", err, file.Name())
}
}
})
}
}
internal/s3select/json/record.go
+206
View File
@@ -0,0 +1,206 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package json
import (
"encoding/json"
"errors"
"fmt"
"io"
"math"
"strconv"
"strings"
"github.com/bcicen/jstream"
csv "github.com/minio/csvparser"
"github.com/minio/minio/internal/s3select/sql"
)
// RawJSON is a byte-slice that contains valid JSON
type RawJSON []byte
// MarshalJSON instance for []byte that assumes that byte-slice is
// already serialized JSON
func (b RawJSON) MarshalJSON() ([]byte, error) {
return b, nil
}
// Record - is JSON record.
type Record struct {
// Used in Set(), Marshal*()
KVS jstream.KVS
SelectFormat sql.SelectObjectFormat
}
// Get - gets the value for a column name.
func (r *Record) Get(name string) (*sql.Value, error) {
// Get is implemented directly in the sql package.
return nil, errors.New("not implemented here")
}
// Reset the record.
func (r *Record) Reset() {
if len(r.KVS) > 0 {
r.KVS = r.KVS[:0]
}
}
// Clone the record and if possible use the destination provided.
func (r *Record) Clone(dst sql.Record) sql.Record {
other, ok := dst.(*Record)
if !ok {
other = &Record{}
}
if len(other.KVS) > 0 {
other.KVS = other.KVS[:0]
}
other.KVS = append(other.KVS, r.KVS...)
return other
}
// Set - sets the value for a column name.
func (r *Record) Set(name string, value *sql.Value) (sql.Record, error) {
var v interface{}
if b, ok := value.ToBool(); ok {
v = b
} else if f, ok := value.ToFloat(); ok {
v = f
} else if i, ok := value.ToInt(); ok {
v = i
} else if t, ok := value.ToTimestamp(); ok {
v = sql.FormatSQLTimestamp(t)
} else if s, ok := value.ToString(); ok {
v = s
} else if value.IsNull() {
v = nil
} else if b, ok := value.ToBytes(); ok {
// This can either be raw json or a CSV value.
// Only treat objects and arrays as JSON.
if len(b) > 0 && (b[0] == '{' || b[0] == '[') {
v = RawJSON(b)
} else {
v = string(b)
}
} else if arr, ok := value.ToArray(); ok {
v = arr
} else {
return nil, fmt.Errorf("unsupported sql value %v and type %v", value, value.GetTypeString())
}
name = strings.Replace(name, "*", "__ALL__", -1)
r.KVS = append(r.KVS, jstream.KV{Key: name, Value: v})
return r, nil
}
// WriteCSV - encodes to CSV data.
func (r *Record) WriteCSV(writer io.Writer, opts sql.WriteCSVOpts) error {
var csvRecord []string
for _, kv := range r.KVS {
var columnValue string
switch val := kv.Value.(type) {
case float64:
columnValue = jsonFloat(val)
case string:
columnValue = val
case bool, int64:
columnValue = fmt.Sprintf("%v", val)
case nil:
columnValue = ""
case RawJSON:
columnValue = string([]byte(val))
case []interface{}:
b, err := json.Marshal(val)
if err != nil {
return err
}
columnValue = string(b)
default:
return fmt.Errorf("Cannot marshal unhandled type: %T", kv.Value)
}
csvRecord = append(csvRecord, columnValue)
}
w := csv.NewWriter(writer)
w.Comma = opts.FieldDelimiter
w.Quote = opts.Quote
w.AlwaysQuote = opts.AlwaysQuote
w.QuoteEscape = opts.QuoteEscape
if err := w.Write(csvRecord); err != nil {
return err
}
w.Flush()
return w.Error()
}
// Raw - returns the underlying representation.
func (r *Record) Raw() (sql.SelectObjectFormat, interface{}) {
return r.SelectFormat, r.KVS
}
// WriteJSON - encodes to JSON data.
func (r *Record) WriteJSON(writer io.Writer) error {
return json.NewEncoder(writer).Encode(r.KVS)
}
// Replace the underlying buffer of json data.
func (r *Record) Replace(k interface{}) error {
v, ok := k.(jstream.KVS)
if !ok {
return fmt.Errorf("cannot replace internal data in json record with type %T", k)
}
r.KVS = v
return nil
}
// NewRecord - creates new empty JSON record.
func NewRecord(f sql.SelectObjectFormat) *Record {
return &Record{
KVS: jstream.KVS{},
SelectFormat: f,
}
}
// jsonFloat converts a float to string similar to Go stdlib formats json floats.
func jsonFloat(f float64) string {
var tmp [32]byte
dst := tmp[:0]
// Convert as if by ES6 number to string conversion.
// This matches most other JSON generators.
// See golang.org/issue/6384 and golang.org/issue/14135.
// Like fmt %g, but the exponent cutoffs are different
// and exponents themselves are not padded to two digits.
abs := math.Abs(f)
fmt := byte('f')
if abs != 0 {
if abs < 1e-6 || abs >= 1e21 {
fmt = 'e'
}
}
dst = strconv.AppendFloat(dst, f, fmt, -1, 64)
if fmt == 'e' {
// clean up e-09 to e-9
n := len(dst)
if n >= 4 && dst[n-4] == 'e' && dst[n-3] == '-' && dst[n-2] == '0' {
dst[n-2] = dst[n-1]
dst = dst[:n-1]
}
}
return string(dst)
}
internal/s3select/json/testdata/10.json
Vendored
+12
View File
@@ -0,0 +1,12 @@
[
{
"key_1": "value",
"key_2": "value"
}
]
[
{
"key_1": "value2",
"key_2": "value3"
}
]
internal/s3select/json/testdata/11.json
Vendored
+8
View File
@@ -0,0 +1,8 @@
"a"
1
3.145
["a"]
{}
{
"a": 1
}
internal/s3select/json/testdata/12.json
Vendored
+5
View File
@@ -0,0 +1,5 @@
{
"a": 1
}{
"b": 2
}
internal/s3select/json/testdata/2.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"text": "hello world\\n2nd line"}
internal/s3select/json/testdata/3.json
Vendored
+1
View File
@@ -0,0 +1 @@
{"hello":"wor{l}d"}
internal/s3select/json/testdata/4.json
Vendored
+26
View File
@@ -0,0 +1,26 @@
{
"id": "0001",
"type": "donut",
"name": "Cake",
"ppu": 0.55,
"batters":
{
"batter":
[
{ "id": "1001", "type": "Regular" },
{ "id": "1002", "type": "Chocolate" },
{ "id": "1003", "type": "Blueberry" },
{ "id": "1004", "type": "Devil's Food" }
]
},
"topping":
[
{ "id": "5001", "type": "None" },
{ "id": "5002", "type": "Glazed" },
{ "id": "5005", "type": "Sugar" },
{ "id": "5007", "type": "Powdered Sugar" },
{ "id": "5006", "type": "Chocolate with Sprinkles" },
{ "id": "5003", "type": "Chocolate" },
{ "id": "5004", "type": "Maple" }
]
}
internal/s3select/json/testdata/5.json
Vendored
+5
View File
@@ -0,0 +1,5 @@
{
"foo": {
"bar": "baz"
}
}
internal/s3select/json/testdata/6.json
Vendored
+1
View File
@@ -0,0 +1 @@
{ "name": "John", "age":28, "hobby": { "name": "chess", "type": "boardgame" }}
internal/s3select/json/testdata/7.json
Vendored
+3
View File
@@ -0,0 +1,3 @@
{"name":"Michael", "age": 31}
{"name":"Andy", "age": 30}
{"name":"Justin", "age": 19}
internal/s3select/json/testdata/8.json
Vendored
+2
View File
@@ -0,0 +1,2 @@
{"a":"}"
}
internal/s3select/json/testdata/9.json
Vendored
+6
View File
@@ -0,0 +1,6 @@
[
{
"key_1": "value",
"key_2": "value"
}
]
internal/s3select/message.go
+433
View File
@@ -0,0 +1,433 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package s3select
import (
"bytes"
"encoding/binary"
"fmt"
"hash/crc32"
"net/http"
"strconv"
"sync/atomic"
"time"
)
// A message is in the format specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-frame-overview.png
// hence the calculation is made accordingly.
func totalByteLength(headerLength, payloadLength int) int {
return 4 + 4 + 4 + headerLength + payloadLength + 4
}
func genMessage(header, payload []byte) []byte {
headerLength := len(header)
payloadLength := len(payload)
totalLength := totalByteLength(headerLength, payloadLength)
buf := new(bytes.Buffer)
binary.Write(buf, binary.BigEndian, uint32(totalLength))
binary.Write(buf, binary.BigEndian, uint32(headerLength))
prelude := buf.Bytes()
binary.Write(buf, binary.BigEndian, crc32.ChecksumIEEE(prelude))
buf.Write(header)
if payload != nil {
buf.Write(payload)
}
message := buf.Bytes()
binary.Write(buf, binary.BigEndian, crc32.ChecksumIEEE(message))
return buf.Bytes()
}
// Refer genRecordsHeader().
var recordsHeader = []byte{
13, ':', 'm', 'e', 's', 's', 'a', 'g', 'e', '-', 't', 'y', 'p', 'e', 7, 0, 5, 'e', 'v', 'e', 'n', 't',
13, ':', 'c', 'o', 'n', 't', 'e', 'n', 't', '-', 't', 'y', 'p', 'e', 7, 0, 24, 'a', 'p', 'p', 'l', 'i', 'c', 'a', 't', 'i', 'o', 'n', '/', 'o', 'c', 't', 'e', 't', '-', 's', 't', 'r', 'e', 'a', 'm',
11, ':', 'e', 'v', 'e', 'n', 't', '-', 't', 'y', 'p', 'e', 7, 0, 7, 'R', 'e', 'c', 'o', 'r', 'd', 's',
}
const (
// Chosen for compatibility with AWS JAVA SDK
// It has a a buffer size of 128K:
// https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/internal/eventstreaming/MessageDecoder.java#L26
// but we must make sure there is always space to add 256 bytes:
// https://github.com/aws/aws-sdk-java/blob/master/aws-java-sdk-s3/src/main/java/com/amazonaws/services/s3/model/SelectObjectContentEventStream.java#L197
maxRecordMessageLength = (128 << 10) - 256
)
var (
bufLength = payloadLenForMsgLen(maxRecordMessageLength)
)
// newRecordsMessage - creates new Records Message which can contain a single record, partial records,
// or multiple records. Depending on the size of the result, a response can contain one or more of these messages.
//
// Header specification
// Records messages contain three headers, as follows:
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-record.png
//
// Payload specification
// Records message payloads can contain a single record, partial records, or multiple records.
func newRecordsMessage(payload []byte) []byte {
return genMessage(recordsHeader, payload)
}
// payloadLenForMsgLen computes the length of the payload in a record
// message given the total length of the message.
func payloadLenForMsgLen(messageLength int) int {
headerLength := len(recordsHeader)
payloadLength := messageLength - 4 - 4 - 4 - headerLength - 4
return payloadLength
}
// continuationMessage - S3 periodically sends this message to keep the TCP connection open.
// These messages appear in responses at random. The client must detect the message type and process accordingly.
//
// Header specification:
// Continuation messages contain two headers, as follows:
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-cont.png
//
// Payload specification:
// Continuation messages have no payload.
var continuationMessage = []byte{
0, 0, 0, 57, // total byte-length.
0, 0, 0, 41, // headers byte-length.
139, 161, 157, 242, // prelude crc.
13, ':', 'm', 'e', 's', 's', 'a', 'g', 'e', '-', 't', 'y', 'p', 'e', 7, 0, 5, 'e', 'v', 'e', 'n', 't', // headers.
11, ':', 'e', 'v', 'e', 'n', 't', '-', 't', 'y', 'p', 'e', 7, 0, 4, 'C', 'o', 'n', 't', // headers.
156, 134, 74, 13, // message crc.
}
// Refer genProgressHeader().
var progressHeader = []byte{
13, ':', 'm', 'e', 's', 's', 'a', 'g', 'e', '-', 't', 'y', 'p', 'e', 7, 0, 5, 'e', 'v', 'e', 'n', 't',
13, ':', 'c', 'o', 'n', 't', 'e', 'n', 't', '-', 't', 'y', 'p', 'e', 7, 0, 8, 't', 'e', 'x', 't', '/', 'x', 'm', 'l',
11, ':', 'e', 'v', 'e', 'n', 't', '-', 't', 'y', 'p', 'e', 7, 0, 8, 'P', 'r', 'o', 'g', 'r', 'e', 's', 's',
}
// newProgressMessage - creates new Progress Message. S3 periodically sends this message, if requested.
// It contains information about the progress of a query that has started but has not yet completed.
//
// Header specification:
// Progress messages contain three headers, as follows:
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-progress.png
//
// Payload specification:
// Progress message payload is an XML document containing information about the progress of a request.
// * BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
// * BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
// * BytesReturned => Current number of bytes of records payload data returned by S3.
//
// For uncompressed files, BytesScanned and BytesProcessed are equal.
//
// Example:
//
// <?xml version="1.0" encoding="UTF-8"?>
// <Progress>
// <BytesScanned>512</BytesScanned>
// <BytesProcessed>1024</BytesProcessed>
// <BytesReturned>1024</BytesReturned>
// </Progress>
//
func newProgressMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
payload := []byte(`<?xml version="1.0" encoding="UTF-8"?><Progress><BytesScanned>` +
strconv.FormatInt(bytesScanned, 10) + `</BytesScanned><BytesProcessed>` +
strconv.FormatInt(bytesProcessed, 10) + `</BytesProcessed><BytesReturned>` +
strconv.FormatInt(bytesReturned, 10) + `</BytesReturned></Stats>`)
return genMessage(progressHeader, payload)
}
// Refer genStatsHeader().
var statsHeader = []byte{
13, ':', 'm', 'e', 's', 's', 'a', 'g', 'e', '-', 't', 'y', 'p', 'e', 7, 0, 5, 'e', 'v', 'e', 'n', 't',
13, ':', 'c', 'o', 'n', 't', 'e', 'n', 't', '-', 't', 'y', 'p', 'e', 7, 0, 8, 't', 'e', 'x', 't', '/', 'x', 'm', 'l',
11, ':', 'e', 'v', 'e', 'n', 't', '-', 't', 'y', 'p', 'e', 7, 0, 5, 'S', 't', 'a', 't', 's',
}
// newStatsMessage - creates new Stats Message. S3 sends this message at the end of the request.
// It contains statistics about the query.
//
// Header specification:
// Stats messages contain three headers, as follows:
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-stats.png
//
// Payload specification:
// Stats message payload is an XML document containing information about a request's stats when processing is complete.
// * BytesScanned => Number of bytes that have been processed before being uncompressed (if the file is compressed).
// * BytesProcessed => Number of bytes that have been processed after being uncompressed (if the file is compressed).
// * BytesReturned => Total number of bytes of records payload data returned by S3.
//
// For uncompressed files, BytesScanned and BytesProcessed are equal.
//
// Example:
//
// <?xml version="1.0" encoding="UTF-8"?>
// <Stats>
// <BytesScanned>512</BytesScanned>
// <BytesProcessed>1024</BytesProcessed>
// <BytesReturned>1024</BytesReturned>
// </Stats>
func newStatsMessage(bytesScanned, bytesProcessed, bytesReturned int64) []byte {
payload := []byte(`<?xml version="1.0" encoding="UTF-8"?><Stats><BytesScanned>` +
strconv.FormatInt(bytesScanned, 10) + `</BytesScanned><BytesProcessed>` +
strconv.FormatInt(bytesProcessed, 10) + `</BytesProcessed><BytesReturned>` +
strconv.FormatInt(bytesReturned, 10) + `</BytesReturned></Stats>`)
return genMessage(statsHeader, payload)
}
// endMessage - indicates that the request is complete, and no more messages will be sent.
// You should not assume that the request is complete until the client receives an End message.
//
// Header specification:
// End messages contain two headers, as follows:
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-end.png
//
// Payload specification:
// End messages have no payload.
var endMessage = []byte{
0, 0, 0, 56, // total byte-length.
0, 0, 0, 40, // headers byte-length.
193, 198, 132, 212, // prelude crc.
13, ':', 'm', 'e', 's', 's', 'a', 'g', 'e', '-', 't', 'y', 'p', 'e', 7, 0, 5, 'e', 'v', 'e', 'n', 't', // headers.
11, ':', 'e', 'v', 'e', 'n', 't', '-', 't', 'y', 'p', 'e', 7, 0, 3, 'E', 'n', 'd', // headers.
207, 151, 211, 146, // message crc.
}
// newErrorMessage - creates new Request Level Error Message. S3 sends this message if the request failed for any reason.
// It contains the error code and error message for the failure. If S3 sends a RequestLevelError message,
// it doesn't send an End message.
//
// Header specification:
// Request-level error messages contain three headers, as follows:
// https://docs.aws.amazon.com/AmazonS3/latest/API/images/s3select-frame-diagram-error.png
//
// Payload specification:
// Request-level error messages have no payload.
func newErrorMessage(errorCode, errorMessage []byte) []byte {
buf := new(bytes.Buffer)
buf.Write([]byte{13, ':', 'm', 'e', 's', 's', 'a', 'g', 'e', '-', 't', 'y', 'p', 'e', 7, 0, 5, 'e', 'r', 'r', 'o', 'r'})
buf.Write([]byte{14, ':', 'e', 'r', 'r', 'o', 'r', '-', 'm', 'e', 's', 's', 'a', 'g', 'e', 7})
binary.Write(buf, binary.BigEndian, uint16(len(errorMessage)))
buf.Write(errorMessage)
buf.Write([]byte{11, ':', 'e', 'r', 'r', 'o', 'r', '-', 'c', 'o', 'd', 'e', 7})
binary.Write(buf, binary.BigEndian, uint16(len(errorCode)))
buf.Write(errorCode)
return genMessage(buf.Bytes(), nil)
}
// NewErrorMessage - creates new Request Level Error Message specified in
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html.
func NewErrorMessage(errorCode, errorMessage string) []byte {
return newErrorMessage([]byte(errorCode), []byte(errorMessage))
}
type messageWriter struct {
writer http.ResponseWriter
getProgressFunc func() (int64, int64)
bytesReturned int64
payloadBuffer []byte
payloadBufferIndex int
payloadCh chan *bytes.Buffer
finBytesScanned, finBytesProcessed int64
errCh chan []byte
doneCh chan struct{}
}
func (writer *messageWriter) write(data []byte) bool {
if _, err := writer.writer.Write(data); err != nil {
return false
}
writer.writer.(http.Flusher).Flush()
return true
}
func (writer *messageWriter) start() {
keepAliveTicker := time.NewTicker(1 * time.Second)
var progressTicker *time.Ticker
var progressTickerC <-chan time.Time
if writer.getProgressFunc != nil {
progressTicker = time.NewTicker(1 * time.Minute)
progressTickerC = progressTicker.C
}
recordStagingTicker := time.NewTicker(500 * time.Millisecond)
// Exit conditions:
//
// 1. If a writer.write() returns false, select loop below exits and
// closes `doneCh` to indicate to caller to also exit.
//
// 2. If caller (Evaluate()) has an error, it sends an error
// message and waits for this go-routine to quit in
// FinishWithError()
//
// 3. If caller is done, it waits for this go-routine to exit
// in Finish()
quitFlag := false
for !quitFlag {
select {
case data := <-writer.errCh:
quitFlag = true
// Flush collected records before sending error message
if !writer.flushRecords() {
break
}
writer.write(data)
case payload, ok := <-writer.payloadCh:
if !ok {
// payloadCh is closed by caller to
// indicate finish with success
quitFlag = true
if !writer.flushRecords() {
break
}
// Write Stats message, then End message
bytesReturned := atomic.LoadInt64(&writer.bytesReturned)
if !writer.write(newStatsMessage(writer.finBytesScanned, writer.finBytesProcessed, bytesReturned)) {
break
}
writer.write(endMessage)
} else {
for payload.Len() > 0 {
copiedLen := copy(writer.payloadBuffer[writer.payloadBufferIndex:], payload.Bytes())
writer.payloadBufferIndex += copiedLen
payload.Next(copiedLen)
// If buffer is filled, flush it now!
freeSpace := bufLength - writer.payloadBufferIndex
if freeSpace == 0 {
if !writer.flushRecords() {
quitFlag = true
break
}
}
}
bufPool.Put(payload)
}
case <-recordStagingTicker.C:
if !writer.flushRecords() {
quitFlag = true
}
case <-keepAliveTicker.C:
if !writer.write(continuationMessage) {
quitFlag = true
}
case <-progressTickerC:
bytesScanned, bytesProcessed := writer.getProgressFunc()
bytesReturned := atomic.LoadInt64(&writer.bytesReturned)
if !writer.write(newProgressMessage(bytesScanned, bytesProcessed, bytesReturned)) {
quitFlag = true
}
}
}
close(writer.doneCh)
recordStagingTicker.Stop()
keepAliveTicker.Stop()
if progressTicker != nil {
progressTicker.Stop()
}
// Whatever drain the payloadCh to prevent from memory leaking.
for len(writer.payloadCh) > 0 {
payload := <-writer.payloadCh
bufPool.Put(payload)
}
}
// Sends a single whole record.
func (writer *messageWriter) SendRecord(payload *bytes.Buffer) error {
select {
case writer.payloadCh <- payload:
return nil
case <-writer.doneCh:
return fmt.Errorf("messageWriter is done")
}
}
func (writer *messageWriter) flushRecords() bool {
if writer.payloadBufferIndex == 0 {
return true
}
result := writer.write(newRecordsMessage(writer.payloadBuffer[0:writer.payloadBufferIndex]))
if result {
atomic.AddInt64(&writer.bytesReturned, int64(writer.payloadBufferIndex))
writer.payloadBufferIndex = 0
}
return result
}
// Finish is the last call to the message writer - it sends any
// remaining record payload, then sends statistics and finally the end
// message.
func (writer *messageWriter) Finish(bytesScanned, bytesProcessed int64) error {
select {
case <-writer.doneCh:
return fmt.Errorf("messageWriter is done")
default:
writer.finBytesScanned = bytesScanned
writer.finBytesProcessed = bytesProcessed
close(writer.payloadCh)
// Wait until the `start` go-routine is done.
<-writer.doneCh
return nil
}
}
func (writer *messageWriter) FinishWithError(errorCode, errorMessage string) error {
select {
case <-writer.doneCh:
return fmt.Errorf("messageWriter is done")
case writer.errCh <- newErrorMessage([]byte(errorCode), []byte(errorMessage)):
// Wait until the `start` go-routine is done.
<-writer.doneCh
return nil
}
}
// newMessageWriter creates a message writer that writes to the HTTP
// response writer
func newMessageWriter(w http.ResponseWriter, getProgressFunc func() (bytesScanned, bytesProcessed int64)) *messageWriter {
writer := &messageWriter{
writer: w,
getProgressFunc: getProgressFunc,
payloadBuffer: make([]byte, bufLength),
payloadCh: make(chan *bytes.Buffer, 1),
errCh: make(chan []byte),
doneCh: make(chan struct{}),
}
go writer.start()
return writer
}
internal/s3select/parquet/args.go
+43
View File
@@ -0,0 +1,43 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package parquet
import "encoding/xml"
// ReaderArgs - represents elements inside <InputSerialization><Parquet/> in request XML.
type ReaderArgs struct {
unmarshaled bool
}
// IsEmpty - returns whether reader args is empty or not.
func (args *ReaderArgs) IsEmpty() bool {
return !args.unmarshaled
}
// UnmarshalXML - decodes XML data.
func (args *ReaderArgs) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type subReaderArgs ReaderArgs
parsedArgs := subReaderArgs{}
if err := d.DecodeElement(&parsedArgs, &start); err != nil {
return err
}
args.unmarshaled = true
return nil
}
internal/s3select/parquet/errors.go
+54
View File
@@ -0,0 +1,54 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package parquet
type s3Error struct {
code string
message string
statusCode int
cause error
}
func (err *s3Error) Cause() error {
return err.cause
}
func (err *s3Error) ErrorCode() string {
return err.code
}
func (err *s3Error) ErrorMessage() string {
return err.message
}
func (err *s3Error) HTTPStatusCode() int {
return err.statusCode
}
func (err *s3Error) Error() string {
return err.message
}
func errParquetParsingError(err error) *s3Error {
return &s3Error{
code: "ParquetParsingError",
message: "Error parsing Parquet file. Please check the file and try again.",
statusCode: 400,
cause: err,
}
}
internal/s3select/parquet/reader.go
+138
View File
@@ -0,0 +1,138 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package parquet
import (
"fmt"
"io"
"time"
"github.com/bcicen/jstream"
jsonfmt "github.com/minio/minio/internal/s3select/json"
"github.com/minio/minio/internal/s3select/sql"
parquetgo "github.com/minio/parquet-go"
parquetgen "github.com/minio/parquet-go/gen-go/parquet"
)
// Reader - Parquet record reader for S3Select.
type Reader struct {
args *ReaderArgs
reader *parquetgo.Reader
}
// Read - reads single record.
func (r *Reader) Read(dst sql.Record) (rec sql.Record, rerr error) {
defer func() {
if rec := recover(); rec != nil {
rerr = fmt.Errorf("panic reading parquet record: %v", rec)
}
}()
parquetRecord, err := r.reader.Read()
if err != nil {
if err != io.EOF {
return nil, errParquetParsingError(err)
}
return nil, err
}
kvs := jstream.KVS{}
f := func(name string, v parquetgo.Value) bool {
if v.Value == nil {
kvs = append(kvs, jstream.KV{Key: name, Value: nil})
return true
}
var value interface{}
switch v.Type {
case parquetgen.Type_BOOLEAN:
value = v.Value.(bool)
case parquetgen.Type_INT32:
value = int64(v.Value.(int32))
if v.Schema != nil && v.Schema.ConvertedType != nil {
switch *v.Schema.ConvertedType {
case parquetgen.ConvertedType_DATE:
value = sql.FormatSQLTimestamp(time.Unix(60*60*24*int64(v.Value.(int32)), 0).UTC())
}
}
case parquetgen.Type_INT64:
value = v.Value.(int64)
if v.Schema != nil && v.Schema.ConvertedType != nil {
switch *v.Schema.ConvertedType {
// Only UTC supported, add one NS to never be exactly midnight.
case parquetgen.ConvertedType_TIMESTAMP_MILLIS:
value = sql.FormatSQLTimestamp(time.Unix(0, 0).Add(time.Duration(v.Value.(int64)) * time.Millisecond).UTC())
case parquetgen.ConvertedType_TIMESTAMP_MICROS:
value = sql.FormatSQLTimestamp(time.Unix(0, 0).Add(time.Duration(v.Value.(int64)) * time.Microsecond).UTC())
}
}
case parquetgen.Type_FLOAT:
value = float64(v.Value.(float32))
case parquetgen.Type_DOUBLE:
value = v.Value.(float64)
case parquetgen.Type_INT96, parquetgen.Type_BYTE_ARRAY, parquetgen.Type_FIXED_LEN_BYTE_ARRAY:
value = string(v.Value.([]byte))
default:
rerr = errParquetParsingError(nil)
return false
}
kvs = append(kvs, jstream.KV{Key: name, Value: value})
return true
}
// Apply our range
parquetRecord.Range(f)
// Reuse destination if we can.
dstRec, ok := dst.(*jsonfmt.Record)
if !ok {
dstRec = &jsonfmt.Record{}
}
dstRec.SelectFormat = sql.SelectFmtParquet
dstRec.KVS = kvs
return dstRec, nil
}
// Close - closes underlying readers.
func (r *Reader) Close() error {
return r.reader.Close()
}
// NewReader - creates new Parquet reader using readerFunc callback.
func NewReader(getReaderFunc func(offset, length int64) (io.ReadCloser, error), args *ReaderArgs) (r *Reader, err error) {
defer func() {
if rec := recover(); rec != nil {
err = fmt.Errorf("panic reading parquet header: %v", rec)
}
}()
reader, err := parquetgo.NewReader(getReaderFunc, nil)
if err != nil {
if err != io.EOF {
return nil, errParquetParsingError(err)
}
return nil, err
}
return &Reader{
args: args,
reader: reader,
}, nil
}
internal/s3select/progress.go
+128
View File
@@ -0,0 +1,128 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package s3select
import (
"compress/bzip2"
"errors"
"fmt"
"io"
"sync"
"sync/atomic"
gzip "github.com/klauspost/pgzip"
)
type countUpReader struct {
reader io.Reader
bytesRead int64
}
func (r *countUpReader) Read(p []byte) (n int, err error) {
n, err = r.reader.Read(p)
atomic.AddInt64(&r.bytesRead, int64(n))
return n, err
}
func (r *countUpReader) BytesRead() int64 {
if r == nil {
return 0
}
return atomic.LoadInt64(&r.bytesRead)
}
func newCountUpReader(reader io.Reader) *countUpReader {
return &countUpReader{
reader: reader,
}
}
type progressReader struct {
rc io.ReadCloser
scannedReader *countUpReader
processedReader *countUpReader
closedMu sync.Mutex
gzr *gzip.Reader
closed bool
}
func (pr *progressReader) Read(p []byte) (n int, err error) {
// This ensures that Close will block until Read has completed.
// This allows another goroutine to close the reader.
pr.closedMu.Lock()
defer pr.closedMu.Unlock()
if pr.closed {
return 0, errors.New("progressReader: read after Close")
}
return pr.processedReader.Read(p)
}
func (pr *progressReader) Close() error {
pr.closedMu.Lock()
defer pr.closedMu.Unlock()
if pr.closed {
return nil
}
pr.closed = true
if pr.gzr != nil {
pr.gzr.Close()
}
return pr.rc.Close()
}
func (pr *progressReader) Stats() (bytesScanned, bytesProcessed int64) {
if pr == nil {
return 0, 0
}
return pr.scannedReader.BytesRead(), pr.processedReader.BytesRead()
}
func newProgressReader(rc io.ReadCloser, compType CompressionType) (*progressReader, error) {
if rc == nil {
return nil, errors.New("newProgressReader: nil reader provided")
}
scannedReader := newCountUpReader(rc)
pr := progressReader{
rc: rc,
scannedReader: scannedReader,
}
var err error
var r io.Reader
switch compType {
case noneType:
r = scannedReader
case gzipType:
pr.gzr, err = gzip.NewReader(scannedReader)
if err != nil {
if errors.Is(err, gzip.ErrHeader) || errors.Is(err, gzip.ErrChecksum) {
return nil, errInvalidGZIPCompressionFormat(err)
}
return nil, errTruncatedInput(err)
}
r = pr.gzr
case bzip2Type:
r = bzip2.NewReader(scannedReader)
default:
return nil, errInvalidCompressionFormat(fmt.Errorf("unknown compression type '%v'", compType))
}
pr.processedReader = newCountUpReader(r)
return &pr, nil
}
internal/s3select/select.go
+559
View File
@@ -0,0 +1,559 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package s3select
import (
"bufio"
"bytes"
"compress/bzip2"
"encoding/xml"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"os"
"strings"
"sync"
"github.com/minio/minio/internal/s3select/csv"
"github.com/minio/minio/internal/s3select/json"
"github.com/minio/minio/internal/s3select/parquet"
"github.com/minio/minio/internal/s3select/simdj"
"github.com/minio/minio/internal/s3select/sql"
"github.com/minio/simdjson-go"
)
type recordReader interface {
// Read a record.
// dst is optional but will be used if valid.
Read(dst sql.Record) (sql.Record, error)
Close() error
}
const (
csvFormat = "csv"
jsonFormat = "json"
parquetFormat = "parquet"
)
// CompressionType - represents value inside <CompressionType/> in request XML.
type CompressionType string
const (
noneType CompressionType = "none"
gzipType CompressionType = "gzip"
bzip2Type CompressionType = "bzip2"
)
const (
maxRecordSize = 1 << 20 // 1 MiB
)
var bufPool = sync.Pool{
New: func() interface{} {
// make a buffer with a reasonable capacity.
return bytes.NewBuffer(make([]byte, 0, maxRecordSize))
},
}
var bufioWriterPool = sync.Pool{
New: func() interface{} {
// ioutil.Discard is just used to create the writer. Actual destination
// writer is set later by Reset() before using it.
return bufio.NewWriter(ioutil.Discard)
},
}
// UnmarshalXML - decodes XML data.
func (c *CompressionType) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
var s string
if err := d.DecodeElement(&s, &start); err != nil {
return errMalformedXML(err)
}
parsedType := CompressionType(strings.ToLower(s))
if s == "" {
parsedType = noneType
}
switch parsedType {
case noneType, gzipType, bzip2Type:
default:
return errInvalidCompressionFormat(fmt.Errorf("invalid compression format '%v'", s))
}
*c = parsedType
return nil
}
// InputSerialization - represents elements inside <InputSerialization/> in request XML.
type InputSerialization struct {
CompressionType CompressionType `xml:"CompressionType"`
CSVArgs csv.ReaderArgs `xml:"CSV"`
JSONArgs json.ReaderArgs `xml:"JSON"`
ParquetArgs parquet.ReaderArgs `xml:"Parquet"`
unmarshaled bool
format string
}
// IsEmpty - returns whether input serialization is empty or not.
func (input *InputSerialization) IsEmpty() bool {
return !input.unmarshaled
}
// UnmarshalXML - decodes XML data.
func (input *InputSerialization) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type subInputSerialization InputSerialization
parsedInput := subInputSerialization{}
if err := d.DecodeElement(&parsedInput, &start); err != nil {
return errMalformedXML(err)
}
// If no compression is specified, set to noneType
if parsedInput.CompressionType == CompressionType("") {
parsedInput.CompressionType = noneType
}
found := 0
if !parsedInput.CSVArgs.IsEmpty() {
parsedInput.format = csvFormat
found++
}
if !parsedInput.JSONArgs.IsEmpty() {
parsedInput.format = jsonFormat
found++
}
if !parsedInput.ParquetArgs.IsEmpty() {
if parsedInput.CompressionType != "" && parsedInput.CompressionType != noneType {
return errInvalidRequestParameter(fmt.Errorf("CompressionType must be NONE for Parquet format"))
}
parsedInput.format = parquetFormat
found++
}
if found != 1 {
return errInvalidDataSource(nil)
}
*input = InputSerialization(parsedInput)
input.unmarshaled = true
return nil
}
// OutputSerialization - represents elements inside <OutputSerialization/> in request XML.
type OutputSerialization struct {
CSVArgs csv.WriterArgs `xml:"CSV"`
JSONArgs json.WriterArgs `xml:"JSON"`
unmarshaled bool
format string
}
// IsEmpty - returns whether output serialization is empty or not.
func (output *OutputSerialization) IsEmpty() bool {
return !output.unmarshaled
}
// UnmarshalXML - decodes XML data.
func (output *OutputSerialization) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// Make subtype to avoid recursive UnmarshalXML().
type subOutputSerialization OutputSerialization
parsedOutput := subOutputSerialization{}
if err := d.DecodeElement(&parsedOutput, &start); err != nil {
return errMalformedXML(err)
}
found := 0
if !parsedOutput.CSVArgs.IsEmpty() {
parsedOutput.format = csvFormat
found++
}
if !parsedOutput.JSONArgs.IsEmpty() {
parsedOutput.format = jsonFormat
found++
}
if found != 1 {
return errObjectSerializationConflict(fmt.Errorf("either CSV or JSON should be present in OutputSerialization"))
}
*output = OutputSerialization(parsedOutput)
output.unmarshaled = true
return nil
}
// RequestProgress - represents elements inside <RequestProgress/> in request XML.
type RequestProgress struct {
Enabled bool `xml:"Enabled"`
}
// S3Select - filters the contents on a simple structured query language (SQL) statement. It
// represents elements inside <SelectRequest/> in request XML specified in detail at
// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectSELECTContent.html.
type S3Select struct {
XMLName xml.Name `xml:"SelectRequest"`
Expression string `xml:"Expression"`
ExpressionType string `xml:"ExpressionType"`
Input InputSerialization `xml:"InputSerialization"`
Output OutputSerialization `xml:"OutputSerialization"`
Progress RequestProgress `xml:"RequestProgress"`
statement *sql.SelectStatement
progressReader *progressReader
recordReader recordReader
close func() error
}
var (
legacyXMLName = "SelectObjectContentRequest"
)
// UnmarshalXML - decodes XML data.
func (s3Select *S3Select) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error {
// S3 also supports the older SelectObjectContentRequest tag,
// though it is no longer found in documentation. This is
// checked and renamed below to allow older clients to also
// work.
if start.Name.Local == legacyXMLName {
start.Name = xml.Name{Space: "", Local: "SelectRequest"}
}
// Make subtype to avoid recursive UnmarshalXML().
type subS3Select S3Select
parsedS3Select := subS3Select{}
if err := d.DecodeElement(&parsedS3Select, &start); err != nil {
if _, ok := err.(*s3Error); ok {
return err
}
return errMalformedXML(err)
}
parsedS3Select.ExpressionType = strings.ToLower(parsedS3Select.ExpressionType)
if parsedS3Select.ExpressionType != "sql" {
return errInvalidExpressionType(fmt.Errorf("invalid expression type '%v'", parsedS3Select.ExpressionType))
}
if parsedS3Select.Input.IsEmpty() {
return errMissingRequiredParameter(fmt.Errorf("InputSerialization must be provided"))
}
if parsedS3Select.Output.IsEmpty() {
return errMissingRequiredParameter(fmt.Errorf("OutputSerialization must be provided"))
}
statement, err := sql.ParseSelectStatement(parsedS3Select.Expression)
if err != nil {
return err
}
parsedS3Select.statement = &statement
*s3Select = S3Select(parsedS3Select)
return nil
}
func (s3Select *S3Select) outputRecord() sql.Record {
switch s3Select.Output.format {
case csvFormat:
return csv.NewRecord()
case jsonFormat:
return json.NewRecord(sql.SelectFmtJSON)
}
panic(fmt.Errorf("unknown output format '%v'", s3Select.Output.format))
}
func (s3Select *S3Select) getProgress() (bytesScanned, bytesProcessed int64) {
if s3Select.progressReader != nil {
return s3Select.progressReader.Stats()
}
return -1, -1
}
// Open - opens S3 object by using callback for SQL selection query.
// Currently CSV, JSON and Apache Parquet formats are supported.
func (s3Select *S3Select) Open(getReader func(offset, length int64) (io.ReadCloser, error)) error {
switch s3Select.Input.format {
case csvFormat:
rc, err := getReader(0, -1)
if err != nil {
return err
}
s3Select.progressReader, err = newProgressReader(rc, s3Select.Input.CompressionType)
if err != nil {
rc.Close()
return err
}
s3Select.recordReader, err = csv.NewReader(s3Select.progressReader, &s3Select.Input.CSVArgs)
if err != nil {
rc.Close()
var stErr bzip2.StructuralError
if errors.As(err, &stErr) {
return errInvalidBZIP2CompressionFormat(err)
}
return err
}
s3Select.close = rc.Close
return nil
case jsonFormat:
rc, err := getReader(0, -1)
if err != nil {
return err
}
s3Select.progressReader, err = newProgressReader(rc, s3Select.Input.CompressionType)
if err != nil {
rc.Close()
return err
}
if strings.EqualFold(s3Select.Input.JSONArgs.ContentType, "lines") {
if simdjson.SupportedCPU() {
s3Select.recordReader = simdj.NewReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
} else {
s3Select.recordReader = json.NewPReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
}
} else {
s3Select.recordReader = json.NewReader(s3Select.progressReader, &s3Select.Input.JSONArgs)
}
s3Select.close = rc.Close
return nil
case parquetFormat:
if !strings.EqualFold(os.Getenv("MINIO_API_SELECT_PARQUET"), "on") {
return errors.New("parquet format parsing not enabled on server")
}
var err error
s3Select.recordReader, err = parquet.NewReader(getReader, &s3Select.Input.ParquetArgs)
return err
}
panic(fmt.Errorf("unknown input format '%v'", s3Select.Input.format))
}
func (s3Select *S3Select) marshal(buf *bytes.Buffer, record sql.Record) error {
switch s3Select.Output.format {
case csvFormat:
// Use bufio Writer to prevent csv.Writer from allocating a new buffer.
bufioWriter := bufioWriterPool.Get().(*bufio.Writer)
defer func() {
bufioWriter.Reset(ioutil.Discard)
bufioWriterPool.Put(bufioWriter)
}()
bufioWriter.Reset(buf)
opts := sql.WriteCSVOpts{
FieldDelimiter: []rune(s3Select.Output.CSVArgs.FieldDelimiter)[0],
Quote: []rune(s3Select.Output.CSVArgs.QuoteCharacter)[0],
QuoteEscape: []rune(s3Select.Output.CSVArgs.QuoteEscapeCharacter)[0],
AlwaysQuote: strings.ToLower(s3Select.Output.CSVArgs.QuoteFields) == "always",
}
err := record.WriteCSV(bufioWriter, opts)
if err != nil {
return err
}
err = bufioWriter.Flush()
if err != nil {
return err
}
if buf.Bytes()[buf.Len()-1] == '\n' {
buf.Truncate(buf.Len() - 1)
}
buf.WriteString(s3Select.Output.CSVArgs.RecordDelimiter)
return nil
case jsonFormat:
err := record.WriteJSON(buf)
if err != nil {
return err
}
// Trim trailing newline from non-simd output
if buf.Bytes()[buf.Len()-1] == '\n' {
buf.Truncate(buf.Len() - 1)
}
buf.WriteString(s3Select.Output.JSONArgs.RecordDelimiter)
return nil
}
panic(fmt.Errorf("unknown output format '%v'", s3Select.Output.format))
}
// Evaluate - filters and sends records read from opened reader as per select statement to http response writer.
func (s3Select *S3Select) Evaluate(w http.ResponseWriter) {
defer func() {
if s3Select.close != nil {
s3Select.close()
}
}()
getProgressFunc := s3Select.getProgress
if !s3Select.Progress.Enabled {
getProgressFunc = nil
}
writer := newMessageWriter(w, getProgressFunc)
var outputQueue []sql.Record
// Create queue based on the type.
if s3Select.statement.IsAggregated() {
outputQueue = make([]sql.Record, 0, 1)
} else {
outputQueue = make([]sql.Record, 0, 100)
}
var err error
sendRecord := func() bool {
buf := bufPool.Get().(*bytes.Buffer)
buf.Reset()
for _, outputRecord := range outputQueue {
if outputRecord == nil {
continue
}
before := buf.Len()
if err = s3Select.marshal(buf, outputRecord); err != nil {
bufPool.Put(buf)
return false
}
if buf.Len()-before > maxRecordSize {
writer.FinishWithError("OverMaxRecordSize", "The length of a record in the input or result is greater than maxCharsPerRecord of 1 MB.")
bufPool.Put(buf)
return false
}
}
if err = writer.SendRecord(buf); err != nil {
// FIXME: log this error.
err = nil
bufPool.Put(buf)
return false
}
outputQueue = outputQueue[:0]
return true
}
var rec sql.Record
OuterLoop:
for {
if s3Select.statement.LimitReached() {
if !sendRecord() {
break
}
if err = writer.Finish(s3Select.getProgress()); err != nil {
// FIXME: log this error.
err = nil
}
break
}
if rec, err = s3Select.recordReader.Read(rec); err != nil {
if err != io.EOF {
break
}
if s3Select.statement.IsAggregated() {
outputRecord := s3Select.outputRecord()
if err = s3Select.statement.AggregateResult(outputRecord); err != nil {
break
}
outputQueue = append(outputQueue, outputRecord)
}
if !sendRecord() {
break
}
if err = writer.Finish(s3Select.getProgress()); err != nil {
// FIXME: log this error.
err = nil
}
break
}
var inputRecords []*sql.Record
if inputRecords, err = s3Select.statement.EvalFrom(s3Select.Input.format, rec); err != nil {
break
}
for _, inputRecord := range inputRecords {
if s3Select.statement.IsAggregated() {
if err = s3Select.statement.AggregateRow(*inputRecord); err != nil {
break OuterLoop
}
} else {
var outputRecord sql.Record
// We will attempt to reuse the records in the table.
// The type of these should not change.
// The queue should always have at least one entry left for this to work.
outputQueue = outputQueue[:len(outputQueue)+1]
if t := outputQueue[len(outputQueue)-1]; t != nil {
// If the output record is already set, we reuse it.
outputRecord = t
outputRecord.Reset()
} else {
// Create new one
outputRecord = s3Select.outputRecord()
outputQueue[len(outputQueue)-1] = outputRecord
}
outputRecord, err = s3Select.statement.Eval(*inputRecord, outputRecord)
if outputRecord == nil || err != nil {
// This should not be written.
// Remove it from the queue.
outputQueue = outputQueue[:len(outputQueue)-1]
if err != nil {
break OuterLoop
}
continue
}
outputQueue[len(outputQueue)-1] = outputRecord
if len(outputQueue) < cap(outputQueue) {
continue
}
if !sendRecord() {
break OuterLoop
}
}
}
}
if err != nil {
_ = writer.FinishWithError("InternalError", err.Error())
}
}
// Close - closes opened S3 object.
func (s3Select *S3Select) Close() error {
return s3Select.recordReader.Close()
}
// NewS3Select - creates new S3Select by given request XML reader.
func NewS3Select(r io.Reader) (*S3Select, error) {
s3Select := &S3Select{}
if err := xml.NewDecoder(r).Decode(s3Select); err != nil {
return nil, err
}
return s3Select, nil
}
internal/s3select/select_benchmark_test.go
+198
View File
@@ -0,0 +1,198 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package s3select
import (
"bytes"
"encoding/csv"
"io"
"io/ioutil"
"math/rand"
"net/http"
"strconv"
"testing"
"time"
humanize "github.com/dustin/go-humanize"
)
const charset = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
func newRandString(length int) string {
randSrc := rand.New(rand.NewSource(time.Now().UnixNano()))
b := make([]byte, length)
for i := range b {
b[i] = charset[randSrc.Intn(len(charset))]
}
return string(b)
}
func genSampleCSVData(count int) []byte {
buf := &bytes.Buffer{}
csvWriter := csv.NewWriter(buf)
csvWriter.Write([]string{"id", "name", "age", "city"})
for i := 0; i < count; i++ {
csvWriter.Write([]string{
strconv.Itoa(i),
newRandString(10),
newRandString(5),
newRandString(10),
})
}
csvWriter.Flush()
return buf.Bytes()
}
type nullResponseWriter struct {
}
func (w *nullResponseWriter) Header() http.Header {
return nil
}
func (w *nullResponseWriter) Write(p []byte) (int, error) {
return len(p), nil
}
func (w *nullResponseWriter) WriteHeader(statusCode int) {
}
func (w *nullResponseWriter) Flush() {
}
func benchmarkSelect(b *testing.B, count int, query string) {
var requestXML = []byte(`
<?xml version="1.0" encoding="UTF-8"?>
<SelectObjectContentRequest>
<Expression>` + query + `</Expression>
<ExpressionType>SQL</ExpressionType>
<InputSerialization>
<CompressionType>NONE</CompressionType>
<CSV>
<FileHeaderInfo>USE</FileHeaderInfo>
</CSV>
</InputSerialization>
<OutputSerialization>
<CSV>
</CSV>
</OutputSerialization>
<RequestProgress>
<Enabled>FALSE</Enabled>
</RequestProgress>
</SelectObjectContentRequest>
`)
csvData := genSampleCSVData(count)
b.ResetTimer()
b.ReportAllocs()
b.SetBytes(int64(count))
b.RunParallel(func(pb *testing.PB) {
for pb.Next() {
s3Select, err := NewS3Select(bytes.NewReader(requestXML))
if err != nil {
b.Fatal(err)
}
if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
return ioutil.NopCloser(bytes.NewReader(csvData)), nil
}); err != nil {
b.Fatal(err)
}
s3Select.Evaluate(&nullResponseWriter{})
s3Select.Close()
}
})
}
func benchmarkSelectAll(b *testing.B, count int) {
benchmarkSelect(b, count, "select * from S3Object")
}
// BenchmarkSelectAll_100K - benchmark * function with 100k records.
func BenchmarkSelectAll_100K(b *testing.B) {
benchmarkSelectAll(b, 100*humanize.KiByte)
}
// BenchmarkSelectAll_1M - benchmark * function with 1m records.
func BenchmarkSelectAll_1M(b *testing.B) {
benchmarkSelectAll(b, 1*humanize.MiByte)
}
// BenchmarkSelectAll_2M - benchmark * function with 2m records.
func BenchmarkSelectAll_2M(b *testing.B) {
benchmarkSelectAll(b, 2*humanize.MiByte)
}
// BenchmarkSelectAll_10M - benchmark * function with 10m records.
func BenchmarkSelectAll_10M(b *testing.B) {
benchmarkSelectAll(b, 10*humanize.MiByte)
}
func benchmarkSingleCol(b *testing.B, count int) {
benchmarkSelect(b, count, "select id from S3Object")
}
// BenchmarkSingleRow_100K - benchmark SELECT column function with 100k records.
func BenchmarkSingleCol_100K(b *testing.B) {
benchmarkSingleCol(b, 1e5)
}
// BenchmarkSelectAll_1M - benchmark * function with 1m records.
func BenchmarkSingleCol_1M(b *testing.B) {
benchmarkSingleCol(b, 1e6)
}
// BenchmarkSelectAll_2M - benchmark * function with 2m records.
func BenchmarkSingleCol_2M(b *testing.B) {
benchmarkSingleCol(b, 2e6)
}
// BenchmarkSelectAll_10M - benchmark * function with 10m records.
func BenchmarkSingleCol_10M(b *testing.B) {
benchmarkSingleCol(b, 1e7)
}
func benchmarkAggregateCount(b *testing.B, count int) {
benchmarkSelect(b, count, "select count(*) from S3Object")
}
// BenchmarkAggregateCount_100K - benchmark count(*) function with 100k records.
func BenchmarkAggregateCount_100K(b *testing.B) {
benchmarkAggregateCount(b, 100*humanize.KiByte)
}
// BenchmarkAggregateCount_1M - benchmark count(*) function with 1m records.
func BenchmarkAggregateCount_1M(b *testing.B) {
benchmarkAggregateCount(b, 1*humanize.MiByte)
}
// BenchmarkAggregateCount_2M - benchmark count(*) function with 2m records.
func BenchmarkAggregateCount_2M(b *testing.B) {
benchmarkAggregateCount(b, 2*humanize.MiByte)
}
// BenchmarkAggregateCount_10M - benchmark count(*) function with 10m records.
func BenchmarkAggregateCount_10M(b *testing.B) {
benchmarkAggregateCount(b, 10*humanize.MiByte)
}
internal/s3select/select_test.go
+1515
View File
File diff suppressed because it is too large Load Diff
internal/s3select/simdj/errors.go
+56
View File
@@ -0,0 +1,56 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package simdj
import "fmt"
type s3Error struct {
code string
message string
statusCode int
cause error
}
func (err *s3Error) Cause() error {
return err.cause
}
func (err *s3Error) ErrorCode() string {
return err.code
}
func (err *s3Error) ErrorMessage() string {
return err.message
}
func (err *s3Error) HTTPStatusCode() int {
return err.statusCode
}
func (err *s3Error) Error() string {
return err.message
}
func errJSONParsingError(err error) *s3Error {
return &s3Error{
code: "JSONParsingError",
message: fmt.Sprintf("Encountered an error parsing the JSON file: %v. Check the file and try again.", err),
statusCode: 400,
cause: err,
}
}
internal/s3select/simdj/reader.go
+188
View File
@@ -0,0 +1,188 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package simdj
import (
"fmt"
"io"
"sync"
"github.com/minio/minio/internal/s3select/json"
"github.com/minio/minio/internal/s3select/sql"
"github.com/minio/simdjson-go"
)
// Reader - JSON record reader for S3Select.
type Reader struct {
args *json.ReaderArgs
input chan simdjson.Stream
decoded chan simdjson.Object
// err will only be returned after decoded has been closed.
err *error
readCloser io.ReadCloser
exitReader chan struct{}
readerWg sync.WaitGroup
}
// Read - reads single record.
func (r *Reader) Read(dst sql.Record) (sql.Record, error) {
v, ok := <-r.decoded
if !ok {
if r.err != nil && *r.err != nil {
return nil, errJSONParsingError(*r.err)
}
return nil, io.EOF
}
dstRec, ok := dst.(*Record)
if !ok {
dstRec = &Record{}
}
dstRec.object = v
return dstRec, nil
}
// Close - closes underlying reader.
func (r *Reader) Close() error {
// Close the input.
// Potentially racy if the stream decoder is still reading.
if r.readCloser != nil {
r.readCloser.Close()
}
if r.exitReader != nil {
close(r.exitReader)
r.readerWg.Wait()
r.exitReader = nil
r.input = nil
}
return nil
}
// startReader will start a reader that accepts input from r.input.
// Input should be root -> object input. Each root indicates a record.
// If r.input is closed, it is assumed that no more input will come.
// When this function returns r.readerWg will be decremented and r.decoded will be closed.
// On errors, r.err will be set. This should only be accessed after r.decoded has been closed.
func (r *Reader) startReader() {
defer r.readerWg.Done()
defer close(r.decoded)
var tmpObj simdjson.Object
for {
var in simdjson.Stream
select {
case in = <-r.input:
case <-r.exitReader:
return
}
if in.Error != nil && in.Error != io.EOF {
r.err = &in.Error
return
}
if in.Value == nil {
if in.Error == io.EOF {
return
}
continue
}
i := in.Value.Iter()
readloop:
for {
var next simdjson.Iter
typ, err := i.AdvanceIter(&next)
if err != nil {
r.err = &err
return
}
switch typ {
case simdjson.TypeNone:
break readloop
case simdjson.TypeRoot:
typ, obj, err := next.Root(nil)
if err != nil {
r.err = &err
return
}
if typ != simdjson.TypeObject {
if typ == simdjson.TypeNone {
continue
}
err = fmt.Errorf("unexpected json type below root :%v", typ)
r.err = &err
return
}
o, err := obj.Object(&tmpObj)
if err != nil {
r.err = &err
return
}
select {
case <-r.exitReader:
return
case r.decoded <- *o:
}
default:
err = fmt.Errorf("unexpected root json type:%v", typ)
r.err = &err
return
}
}
if in.Error == io.EOF {
return
}
}
}
// NewReader - creates new JSON reader using readCloser.
func NewReader(readCloser io.ReadCloser, args *json.ReaderArgs) *Reader {
r := Reader{
args: args,
readCloser: readCloser,
decoded: make(chan simdjson.Object, 1000),
input: make(chan simdjson.Stream, 2),
exitReader: make(chan struct{}),
}
simdjson.ParseNDStream(readCloser, r.input, nil)
r.readerWg.Add(1)
go r.startReader()
return &r
}
// NewElementReader - creates new JSON reader using readCloser.
func NewElementReader(ch chan simdjson.Object, err *error, args *json.ReaderArgs) *Reader {
return &Reader{
args: args,
decoded: ch,
err: err,
readCloser: nil,
}
}
// NewTapeReaderChan will start a reader that will read input from the provided channel.
func NewTapeReaderChan(pj chan simdjson.Stream, args *json.ReaderArgs) *Reader {
r := Reader{
args: args,
decoded: make(chan simdjson.Object, 1000),
input: pj,
exitReader: make(chan struct{}),
}
r.readerWg.Add(1)
go r.startReader()
return &r
}
internal/s3select/simdj/reader_amd64_test.go
+177
View File
@@ -0,0 +1,177 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package simdj
import (
"bytes"
"io"
"io/ioutil"
"path/filepath"
"testing"
"github.com/klauspost/compress/zstd"
"github.com/minio/minio/internal/s3select/json"
"github.com/minio/minio/internal/s3select/sql"
"github.com/minio/simdjson-go"
)
type tester interface {
Fatal(args ...interface{})
}
func loadCompressed(t tester, file string) (js []byte) {
dec, err := zstd.NewReader(nil)
if err != nil {
t.Fatal(err)
}
defer dec.Close()
js, err = ioutil.ReadFile(filepath.Join("testdata", file+".json.zst"))
if err != nil {
t.Fatal(err)
}
js, err = dec.DecodeAll(js, nil)
if err != nil {
t.Fatal(err)
}
return js
}
var testCases = []struct {
name string
array bool
}{
{
name: "parking-citations-10",
},
}
func TestNDJSON(t *testing.T) {
if !simdjson.SupportedCPU() {
t.Skip("Unsupported cpu")
}
for _, tt := range testCases {
t.Run(tt.name, func(t *testing.T) {
ref := loadCompressed(t, tt.name)
var err error
dst := make(chan simdjson.Object, 100)
dec := NewElementReader(dst, &err, &json.ReaderArgs{ContentType: "json"})
pj, err := simdjson.ParseND(ref, nil)
if err != nil {
t.Fatal(err)
}
i := pj.Iter()
cpy := i
b, err := cpy.MarshalJSON()
if err != nil {
t.Fatal(err)
}
if false {
t.Log(string(b))
}
//_ = ioutil.WriteFile(filepath.Join("testdata", tt.name+".json"), b, os.ModePerm)
parser:
for {
var next simdjson.Iter
typ, err := i.AdvanceIter(&next)
if err != nil {
t.Fatal(err)
}
switch typ {
case simdjson.TypeNone:
close(dst)
break parser
case simdjson.TypeRoot:
typ, obj, err := next.Root(nil)
if err != nil {
t.Fatal(err)
}
if typ != simdjson.TypeObject {
if typ == simdjson.TypeNone {
close(dst)
break parser
}
t.Fatal("Unexpected type:", typ.String())
}
o, err := obj.Object(nil)
if err != nil {
t.Fatal(err)
}
dst <- *o
default:
t.Fatal("unexpected type:", typ.String())
}
}
refDec := json.NewReader(ioutil.NopCloser(bytes.NewBuffer(ref)), &json.ReaderArgs{ContentType: "json"})
for {
rec, err := dec.Read(nil)
if err == io.EOF {
break
}
if err != nil {
t.Error(err)
}
want, err := refDec.Read(nil)
if err != nil {
t.Error(err)
}
var gotB, wantB bytes.Buffer
opts := sql.WriteCSVOpts{
FieldDelimiter: ',',
Quote: '"',
QuoteEscape: '"',
AlwaysQuote: false,
}
err = rec.WriteCSV(&gotB, opts)
if err != nil {
t.Error(err)
}
err = want.WriteCSV(&wantB, opts)
if err != nil {
t.Error(err)
}
if !bytes.Equal(gotB.Bytes(), wantB.Bytes()) {
t.Errorf("CSV output mismatch.\nwant: %s(%x)\ngot: %s(%x)", wantB.String(), wantB.Bytes(), gotB.String(), gotB.Bytes())
}
gotB.Reset()
wantB.Reset()
err = rec.WriteJSON(&gotB)
if err != nil {
t.Error(err)
}
err = want.WriteJSON(&wantB)
if err != nil {
t.Error(err)
}
// truncate newline from 'want'
wantB.Truncate(wantB.Len() - 1)
if !bytes.Equal(gotB.Bytes(), wantB.Bytes()) {
t.Errorf("JSON output mismatch.\nwant: %s\ngot: %s", wantB.String(), gotB.String())
}
}
})
}
}
internal/s3select/simdj/record.go
+228
View File
@@ -0,0 +1,228 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package simdj
import (
"fmt"
"io"
"github.com/bcicen/jstream"
csv "github.com/minio/csvparser"
"github.com/minio/minio/internal/s3select/json"
"github.com/minio/minio/internal/s3select/sql"
"github.com/minio/simdjson-go"
)
// Record - is JSON record.
type Record struct {
// object
object simdjson.Object
}
// Get - gets the value for a column name.
func (r *Record) Get(name string) (*sql.Value, error) {
elem := r.object.FindKey(name, nil)
if elem == nil {
return nil, nil
}
return iterToValue(elem.Iter)
}
func iterToValue(iter simdjson.Iter) (*sql.Value, error) {
switch iter.Type() {
case simdjson.TypeString:
v, err := iter.String()
if err != nil {
return nil, err
}
return sql.FromString(v), nil
case simdjson.TypeFloat:
v, err := iter.Float()
if err != nil {
return nil, err
}
return sql.FromFloat(v), nil
case simdjson.TypeInt:
v, err := iter.Int()
if err != nil {
return nil, err
}
return sql.FromInt(v), nil
case simdjson.TypeUint:
v, err := iter.Int()
if err != nil {
// Can't fit into int, convert to float.
v, err := iter.Float()
return sql.FromFloat(v), err
}
return sql.FromInt(v), nil
case simdjson.TypeBool:
v, err := iter.Bool()
if err != nil {
return nil, err
}
return sql.FromBool(v), nil
case simdjson.TypeNull:
return sql.FromNull(), nil
case simdjson.TypeObject, simdjson.TypeArray:
b, err := iter.MarshalJSON()
return sql.FromBytes(b), err
}
return nil, fmt.Errorf("iterToValue: unknown JSON type: %s", iter.Type().String())
}
// Reset the record.
func (r *Record) Reset() {
r.object = simdjson.Object{}
}
// Clone the record and if possible use the destination provided.
func (r *Record) Clone(dst sql.Record) sql.Record {
other, ok := dst.(*Record)
if !ok {
other = &Record{}
}
other.object = r.object
return other
}
// CloneTo clones the record to a json Record.
// Values are only unmashaled on object level.
func (r *Record) CloneTo(dst *json.Record) (sql.Record, error) {
if dst == nil {
dst = &json.Record{SelectFormat: sql.SelectFmtJSON}
}
dst.Reset()
elems, err := r.object.Parse(nil)
if err != nil {
return nil, err
}
if cap(dst.KVS) < len(elems.Elements) {
dst.KVS = make(jstream.KVS, 0, len(elems.Elements))
}
for _, elem := range elems.Elements {
v, err := sql.IterToValue(elem.Iter)
if err != nil {
v, err = elem.Iter.Interface()
if err != nil {
panic(err)
}
}
dst.KVS = append(dst.KVS, jstream.KV{
Key: elem.Name,
Value: v,
})
}
return dst, nil
}
// Set - sets the value for a column name.
func (r *Record) Set(name string, value *sql.Value) (sql.Record, error) {
dst, err := r.CloneTo(nil)
if err != nil {
return nil, err
}
return dst.Set(name, value)
}
// WriteCSV - encodes to CSV data.
func (r *Record) WriteCSV(writer io.Writer, opts sql.WriteCSVOpts) error {
csvRecord := make([]string, 0, 10)
var tmp simdjson.Iter
obj := r.object
allElems:
for {
_, typ, err := obj.NextElement(&tmp)
if err != nil {
return err
}
var columnValue string
switch typ {
case simdjson.TypeNull, simdjson.TypeFloat, simdjson.TypeUint, simdjson.TypeInt, simdjson.TypeBool, simdjson.TypeString:
val, err := tmp.StringCvt()
if err != nil {
return err
}
columnValue = val
case simdjson.TypeObject, simdjson.TypeArray:
b, err := tmp.MarshalJSON()
if err != nil {
return err
}
columnValue = string(b)
case simdjson.TypeNone:
break allElems
default:
return fmt.Errorf("cannot marshal unhandled type: %s", typ.String())
}
csvRecord = append(csvRecord, columnValue)
}
w := csv.NewWriter(writer)
w.Comma = opts.FieldDelimiter
w.Quote = opts.Quote
w.QuoteEscape = opts.QuoteEscape
w.AlwaysQuote = opts.AlwaysQuote
if err := w.Write(csvRecord); err != nil {
return err
}
w.Flush()
return w.Error()
}
// Raw - returns the underlying representation.
func (r *Record) Raw() (sql.SelectObjectFormat, interface{}) {
return sql.SelectFmtSIMDJSON, r.object
}
// WriteJSON - encodes to JSON data.
func (r *Record) WriteJSON(writer io.Writer) error {
o := r.object
elems, err := o.Parse(nil)
if err != nil {
return err
}
b, err := elems.MarshalJSON()
if err != nil {
return err
}
n, err := writer.Write(b)
if err != nil {
return err
}
if n != len(b) {
return io.ErrShortWrite
}
return nil
}
// Replace the underlying buffer of json data.
func (r *Record) Replace(k interface{}) error {
v, ok := k.(simdjson.Object)
if !ok {
return fmt.Errorf("cannot replace internal data in simd json record with type %T", k)
}
r.object = v
return nil
}
// NewRecord - creates new empty JSON record.
func NewRecord(f sql.SelectObjectFormat, obj simdjson.Object) *Record {
return &Record{
object: obj,
}
}
internal/s3select/simdj/testdata/parking-citations-10.json.zst
Vendored
BIN
View File
Binary file not shown.
internal/s3select/sql/aggregation.go
+331
View File
@@ -0,0 +1,331 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"errors"
"fmt"
)
// Aggregation Function name constants
const (
aggFnAvg FuncName = "AVG"
aggFnCount FuncName = "COUNT"
aggFnMax FuncName = "MAX"
aggFnMin FuncName = "MIN"
aggFnSum FuncName = "SUM"
)
var (
errNonNumericArg = func(fnStr FuncName) error {
return fmt.Errorf("%s() requires a numeric argument", fnStr)
}
errInvalidAggregation = errors.New("Invalid aggregation seen")
)
type aggVal struct {
runningSum *Value
runningCount int64
runningMax, runningMin *Value
// Stores if at least one record has been seen
seen bool
}
func newAggVal(fn FuncName) *aggVal {
switch fn {
case aggFnAvg, aggFnSum:
return &aggVal{runningSum: FromFloat(0)}
case aggFnMin:
return &aggVal{runningMin: FromInt(0)}
case aggFnMax:
return &aggVal{runningMax: FromInt(0)}
default:
return &aggVal{}
}
}
// evalAggregationNode - performs partial computation using the
// current row and stores the result.
//
// On success, it returns (nil, nil).
func (e *FuncExpr) evalAggregationNode(r Record, tableAlias string) error {
// It is assumed that this function is called only when
// `e` is an aggregation function.
var val *Value
var err error
funcName := e.getFunctionName()
if aggFnCount == funcName {
if e.Count.StarArg {
// Handle COUNT(*)
e.aggregate.runningCount++
return nil
}
val, err = e.Count.ExprArg.evalNode(r, tableAlias)
if err != nil {
return err
}
} else {
// Evaluate the (only) argument
val, err = e.SFunc.ArgsList[0].evalNode(r, tableAlias)
if err != nil {
return err
}
}
if val.IsNull() {
// E.g. the column or field does not exist in the
// record - in all such cases the aggregation is not
// updated.
return nil
}
argVal := val
if funcName != aggFnCount {
// All aggregation functions, except COUNT require a
// numeric argument.
// Here, we diverge from Amazon S3 behavior by
// inferring untyped values are numbers.
if !argVal.isNumeric() {
if i, ok := argVal.bytesToInt(); ok {
argVal.setInt(i)
} else if f, ok := argVal.bytesToFloat(); ok {
argVal.setFloat(f)
} else {
return errNonNumericArg(funcName)
}
}
}
// Mark that we have seen one non-null value.
isFirstRow := false
if !e.aggregate.seen {
e.aggregate.seen = true
isFirstRow = true
}
switch funcName {
case aggFnCount:
// For all non-null values, the count is incremented.
e.aggregate.runningCount++
case aggFnAvg, aggFnSum:
e.aggregate.runningCount++
// Convert to float.
f, ok := argVal.ToFloat()
if !ok {
return fmt.Errorf("Could not convert value %v (%s) to a number", argVal.value, argVal.GetTypeString())
}
argVal.setFloat(f)
err = e.aggregate.runningSum.arithOp(opPlus, argVal)
case aggFnMin:
err = e.aggregate.runningMin.minmax(argVal, false, isFirstRow)
case aggFnMax:
err = e.aggregate.runningMax.minmax(argVal, true, isFirstRow)
default:
err = errInvalidAggregation
}
return err
}
func (e *AliasedExpression) aggregateRow(r Record, tableAlias string) error {
return e.Expression.aggregateRow(r, tableAlias)
}
func (e *Expression) aggregateRow(r Record, tableAlias string) error {
for _, ex := range e.And {
err := ex.aggregateRow(r, tableAlias)
if err != nil {
return err
}
}
return nil
}
func (e *ListExpr) aggregateRow(r Record, tableAlias string) error {
for _, ex := range e.Elements {
err := ex.aggregateRow(r, tableAlias)
if err != nil {
return err
}
}
return nil
}
func (e *AndCondition) aggregateRow(r Record, tableAlias string) error {
for _, ex := range e.Condition {
err := ex.aggregateRow(r, tableAlias)
if err != nil {
return err
}
}
return nil
}
func (e *Condition) aggregateRow(r Record, tableAlias string) error {
if e.Operand != nil {
return e.Operand.aggregateRow(r, tableAlias)
}
return e.Not.aggregateRow(r, tableAlias)
}
func (e *ConditionOperand) aggregateRow(r Record, tableAlias string) error {
err := e.Operand.aggregateRow(r, tableAlias)
if err != nil {
return err
}
if e.ConditionRHS == nil {
return nil
}
switch {
case e.ConditionRHS.Compare != nil:
return e.ConditionRHS.Compare.Operand.aggregateRow(r, tableAlias)
case e.ConditionRHS.Between != nil:
err = e.ConditionRHS.Between.Start.aggregateRow(r, tableAlias)
if err != nil {
return err
}
return e.ConditionRHS.Between.End.aggregateRow(r, tableAlias)
case e.ConditionRHS.In != nil:
elt := e.ConditionRHS.In.ListExpression
err = elt.aggregateRow(r, tableAlias)
if err != nil {
return err
}
return nil
case e.ConditionRHS.Like != nil:
err = e.ConditionRHS.Like.Pattern.aggregateRow(r, tableAlias)
if err != nil {
return err
}
return e.ConditionRHS.Like.EscapeChar.aggregateRow(r, tableAlias)
default:
return errInvalidASTNode
}
}
func (e *Operand) aggregateRow(r Record, tableAlias string) error {
err := e.Left.aggregateRow(r, tableAlias)
if err != nil {
return err
}
for _, rt := range e.Right {
err = rt.Right.aggregateRow(r, tableAlias)
if err != nil {
return err
}
}
return nil
}
func (e *MultOp) aggregateRow(r Record, tableAlias string) error {
err := e.Left.aggregateRow(r, tableAlias)
if err != nil {
return err
}
for _, rt := range e.Right {
err = rt.Right.aggregateRow(r, tableAlias)
if err != nil {
return err
}
}
return nil
}
func (e *UnaryTerm) aggregateRow(r Record, tableAlias string) error {
if e.Negated != nil {
return e.Negated.Term.aggregateRow(r, tableAlias)
}
return e.Primary.aggregateRow(r, tableAlias)
}
func (e *PrimaryTerm) aggregateRow(r Record, tableAlias string) error {
switch {
case e.ListExpr != nil:
return e.ListExpr.aggregateRow(r, tableAlias)
case e.SubExpression != nil:
return e.SubExpression.aggregateRow(r, tableAlias)
case e.FuncCall != nil:
return e.FuncCall.aggregateRow(r, tableAlias)
}
return nil
}
func (e *FuncExpr) aggregateRow(r Record, tableAlias string) error {
switch e.getFunctionName() {
case aggFnAvg, aggFnSum, aggFnMax, aggFnMin, aggFnCount:
return e.evalAggregationNode(r, tableAlias)
default:
// TODO: traverse arguments and call aggregateRow on
// them if they could be an ancestor of an
// aggregation.
}
return nil
}
// getAggregate() implementation for each AST node follows. This is
// called after calling aggregateRow() on each input row, to calculate
// the final aggregate result.
func (e *FuncExpr) getAggregate() (*Value, error) {
switch e.getFunctionName() {
case aggFnCount:
return FromInt(e.aggregate.runningCount), nil
case aggFnAvg:
if e.aggregate.runningCount == 0 {
// No rows were seen by AVG.
return FromNull(), nil
}
err := e.aggregate.runningSum.arithOp(opDivide, FromInt(e.aggregate.runningCount))
return e.aggregate.runningSum, err
case aggFnMin:
if !e.aggregate.seen {
// No rows were seen by MIN
return FromNull(), nil
}
return e.aggregate.runningMin, nil
case aggFnMax:
if !e.aggregate.seen {
// No rows were seen by MAX
return FromNull(), nil
}
return e.aggregate.runningMax, nil
case aggFnSum:
// TODO: check if returning 0 when no rows were seen
// by SUM is expected behavior.
return e.aggregate.runningSum, nil
default:
// TODO:
}
return nil, errInvalidAggregation
}
internal/s3select/sql/analysis.go
+324
View File
@@ -0,0 +1,324 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"errors"
"fmt"
"strings"
)
// Query analysis - The query is analyzed to determine if it involves
// aggregation.
//
// Aggregation functions - An expression that involves aggregation of
// rows in some manner. Requires all input rows to be processed,
// before a result is returned.
//
// Row function - An expression that depends on a value in the
// row. They have an output for each input row.
//
// Some types of a queries are not valid. For example, an aggregation
// function combined with a row function is meaningless ("AVG(s.Age) +
// s.Salary"). Analysis determines if such a scenario exists so an
// error can be returned.
var (
// Fatal error for query processing.
errNestedAggregation = errors.New("Cannot nest aggregations")
errFunctionNotImplemented = errors.New("Function is not yet implemented")
errUnexpectedInvalidNode = errors.New("Unexpected node value")
errInvalidKeypath = errors.New("A provided keypath is invalid")
)
// qProp contains analysis info about an SQL term.
type qProp struct {
isAggregation, isRowFunc bool
err error
}
// `combine` combines a pair of `qProp`s, so that errors are
// propagated correctly, and checks that an aggregation is not being
// combined with a row-function term.
func (p *qProp) combine(q qProp) {
switch {
case p.err != nil:
// Do nothing
case q.err != nil:
p.err = q.err
default:
p.isAggregation = p.isAggregation || q.isAggregation
p.isRowFunc = p.isRowFunc || q.isRowFunc
if p.isAggregation && p.isRowFunc {
p.err = errNestedAggregation
}
}
}
func (e *SelectExpression) analyze(s *Select) (result qProp) {
if e.All {
return qProp{isRowFunc: true}
}
for _, ex := range e.Expressions {
result.combine(ex.analyze(s))
}
return
}
func (e *AliasedExpression) analyze(s *Select) qProp {
return e.Expression.analyze(s)
}
func (e *Expression) analyze(s *Select) (result qProp) {
for _, ac := range e.And {
result.combine(ac.analyze(s))
}
return
}
func (e *AndCondition) analyze(s *Select) (result qProp) {
for _, ac := range e.Condition {
result.combine(ac.analyze(s))
}
return
}
func (e *Condition) analyze(s *Select) (result qProp) {
if e.Operand != nil {
result = e.Operand.analyze(s)
} else {
result = e.Not.analyze(s)
}
return
}
func (e *ListExpr) analyze(s *Select) (result qProp) {
for _, ac := range e.Elements {
result.combine(ac.analyze(s))
}
return
}
func (e *ConditionOperand) analyze(s *Select) (result qProp) {
if e.ConditionRHS == nil {
result = e.Operand.analyze(s)
} else {
result.combine(e.Operand.analyze(s))
result.combine(e.ConditionRHS.analyze(s))
}
return
}
func (e *ConditionRHS) analyze(s *Select) (result qProp) {
switch {
case e.Compare != nil:
result = e.Compare.Operand.analyze(s)
case e.Between != nil:
result.combine(e.Between.Start.analyze(s))
result.combine(e.Between.End.analyze(s))
case e.In != nil:
result.combine(e.In.ListExpression.analyze(s))
case e.Like != nil:
result.combine(e.Like.Pattern.analyze(s))
if e.Like.EscapeChar != nil {
result.combine(e.Like.EscapeChar.analyze(s))
}
default:
result = qProp{err: errUnexpectedInvalidNode}
}
return
}
func (e *Operand) analyze(s *Select) (result qProp) {
result.combine(e.Left.analyze(s))
for _, r := range e.Right {
result.combine(r.Right.analyze(s))
}
return
}
func (e *MultOp) analyze(s *Select) (result qProp) {
result.combine(e.Left.analyze(s))
for _, r := range e.Right {
result.combine(r.Right.analyze(s))
}
return
}
func (e *UnaryTerm) analyze(s *Select) (result qProp) {
if e.Negated != nil {
result = e.Negated.Term.analyze(s)
} else {
result = e.Primary.analyze(s)
}
return
}
func (e *PrimaryTerm) analyze(s *Select) (result qProp) {
switch {
case e.Value != nil:
result = qProp{}
case e.JPathExpr != nil:
// Check if the path expression is valid
if len(e.JPathExpr.PathExpr) > 0 {
if e.JPathExpr.BaseKey.String() != s.From.As && strings.ToLower(e.JPathExpr.BaseKey.String()) != baseTableName {
result = qProp{err: errInvalidKeypath}
return
}
}
result = qProp{isRowFunc: true}
case e.ListExpr != nil:
result = e.ListExpr.analyze(s)
case e.SubExpression != nil:
result = e.SubExpression.analyze(s)
case e.FuncCall != nil:
result = e.FuncCall.analyze(s)
default:
result = qProp{err: errUnexpectedInvalidNode}
}
return
}
func (e *FuncExpr) analyze(s *Select) (result qProp) {
funcName := e.getFunctionName()
switch funcName {
case sqlFnCast:
return e.Cast.Expr.analyze(s)
case sqlFnExtract:
return e.Extract.From.analyze(s)
case sqlFnDateAdd:
result.combine(e.DateAdd.Quantity.analyze(s))
result.combine(e.DateAdd.Timestamp.analyze(s))
return result
case sqlFnDateDiff:
result.combine(e.DateDiff.Timestamp1.analyze(s))
result.combine(e.DateDiff.Timestamp2.analyze(s))
return result
// Handle aggregation function calls
case aggFnAvg, aggFnMax, aggFnMin, aggFnSum, aggFnCount:
// Initialize accumulator
e.aggregate = newAggVal(funcName)
var exprA qProp
if funcName == aggFnCount {
if e.Count.StarArg {
return qProp{isAggregation: true}
}
exprA = e.Count.ExprArg.analyze(s)
} else {
if len(e.SFunc.ArgsList) != 1 {
return qProp{err: fmt.Errorf("%s takes exactly one argument", funcName)}
}
exprA = e.SFunc.ArgsList[0].analyze(s)
}
if exprA.err != nil {
return exprA
}
if exprA.isAggregation {
return qProp{err: errNestedAggregation}
}
return qProp{isAggregation: true}
case sqlFnCoalesce:
if len(e.SFunc.ArgsList) == 0 {
return qProp{err: fmt.Errorf("%s needs at least one argument", string(funcName))}
}
for _, arg := range e.SFunc.ArgsList {
result.combine(arg.analyze(s))
}
return result
case sqlFnNullIf:
if len(e.SFunc.ArgsList) != 2 {
return qProp{err: fmt.Errorf("%s needs exactly 2 arguments", string(funcName))}
}
for _, arg := range e.SFunc.ArgsList {
result.combine(arg.analyze(s))
}
return result
case sqlFnCharLength, sqlFnCharacterLength:
if len(e.SFunc.ArgsList) != 1 {
return qProp{err: fmt.Errorf("%s needs exactly 2 arguments", string(funcName))}
}
for _, arg := range e.SFunc.ArgsList {
result.combine(arg.analyze(s))
}
return result
case sqlFnLower, sqlFnUpper:
if len(e.SFunc.ArgsList) != 1 {
return qProp{err: fmt.Errorf("%s needs exactly 2 arguments", string(funcName))}
}
for _, arg := range e.SFunc.ArgsList {
result.combine(arg.analyze(s))
}
return result
case sqlFnTrim:
if e.Trim.TrimChars != nil {
result.combine(e.Trim.TrimChars.analyze(s))
}
if e.Trim.TrimFrom != nil {
result.combine(e.Trim.TrimFrom.analyze(s))
}
return result
case sqlFnSubstring:
errVal := fmt.Errorf("Invalid argument(s) to %s", string(funcName))
result.combine(e.Substring.Expr.analyze(s))
switch {
case e.Substring.From != nil:
result.combine(e.Substring.From.analyze(s))
if e.Substring.For != nil {
result.combine(e.Substring.Expr.analyze(s))
}
case e.Substring.Arg2 != nil:
result.combine(e.Substring.Arg2.analyze(s))
if e.Substring.Arg3 != nil {
result.combine(e.Substring.Arg3.analyze(s))
}
default:
result.err = errVal
}
return result
case sqlFnUTCNow:
if len(e.SFunc.ArgsList) != 0 {
result.err = fmt.Errorf("%s() takes no arguments", string(funcName))
}
return result
}
// TODO: implement other functions
return qProp{err: errFunctionNotImplemented}
}
internal/s3select/sql/errors.go
+110
View File
@@ -0,0 +1,110 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import "fmt"
type s3Error struct {
code string
message string
statusCode int
cause error
}
func (err *s3Error) Cause() error {
return err.cause
}
func (err *s3Error) ErrorCode() string {
return err.code
}
func (err *s3Error) ErrorMessage() string {
return err.message
}
func (err *s3Error) HTTPStatusCode() int {
return err.statusCode
}
func (err *s3Error) Error() string {
return err.message
}
func errInvalidDataType(err error) *s3Error {
return &s3Error{
code: "InvalidDataType",
message: "The SQL expression contains an invalid data type.",
statusCode: 400,
cause: err,
}
}
func errIncorrectSQLFunctionArgumentType(err error) *s3Error {
return &s3Error{
code: "IncorrectSqlFunctionArgumentType",
message: "Incorrect type of arguments in function call.",
statusCode: 400,
cause: err,
}
}
func errLikeInvalidInputs(err error) *s3Error {
return &s3Error{
code: "LikeInvalidInputs",
message: "Invalid argument given to the LIKE clause in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errQueryParseFailure(err error) *s3Error {
return &s3Error{
code: "ParseSelectFailure",
message: err.Error(),
statusCode: 400,
cause: err,
}
}
func errQueryAnalysisFailure(err error) *s3Error {
return &s3Error{
code: "InvalidQuery",
message: err.Error(),
statusCode: 400,
cause: err,
}
}
func errBadTableName(err error) *s3Error {
return &s3Error{
code: "BadTableName",
message: fmt.Sprintf("The table name is not supported: %v", err),
statusCode: 400,
cause: err,
}
}
func errDataSource(err error) *s3Error {
return &s3Error{
code: "DataSourcePathUnsupported",
message: fmt.Sprintf("Data source: %v", err),
statusCode: 400,
cause: err,
}
}
internal/s3select/sql/evaluate.go
+491
View File
@@ -0,0 +1,491 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"encoding/json"
"errors"
"fmt"
"math"
"github.com/bcicen/jstream"
"github.com/minio/simdjson-go"
)
var (
errInvalidASTNode = errors.New("invalid AST Node")
errExpectedBool = errors.New("expected bool")
errLikeNonStrArg = errors.New("LIKE clause requires string arguments")
errLikeInvalidEscape = errors.New("LIKE clause has invalid ESCAPE character")
errNotImplemented = errors.New("not implemented")
)
// AST Node Evaluation functions
//
// During evaluation, the query is known to be valid, as analysis is
// complete. The only errors possible are due to value type
// mismatches, etc.
//
// If an aggregation node is present as a descendant (when
// e.prop.isAggregation is true), we call evalNode on all child nodes,
// check for errors, but do not perform any combining of the results
// of child nodes. The final result row is returned after all rows are
// processed, and the `getAggregate` function is called.
func (e *AliasedExpression) evalNode(r Record, tableAlias string) (*Value, error) {
return e.Expression.evalNode(r, tableAlias)
}
func (e *Expression) evalNode(r Record, tableAlias string) (*Value, error) {
if len(e.And) == 1 {
// In this case, result is not required to be boolean
// type.
return e.And[0].evalNode(r, tableAlias)
}
// Compute OR of conditions
result := false
for _, ex := range e.And {
res, err := ex.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
b, ok := res.ToBool()
if !ok {
return nil, errExpectedBool
}
result = result || b
}
return FromBool(result), nil
}
func (e *AndCondition) evalNode(r Record, tableAlias string) (*Value, error) {
if len(e.Condition) == 1 {
// In this case, result does not have to be boolean
return e.Condition[0].evalNode(r, tableAlias)
}
// Compute AND of conditions
result := true
for _, ex := range e.Condition {
res, err := ex.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
b, ok := res.ToBool()
if !ok {
return nil, errExpectedBool
}
result = result && b
}
return FromBool(result), nil
}
func (e *Condition) evalNode(r Record, tableAlias string) (*Value, error) {
if e.Operand != nil {
// In this case, result does not have to be boolean
return e.Operand.evalNode(r, tableAlias)
}
// Compute NOT of condition
res, err := e.Not.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
b, ok := res.ToBool()
if !ok {
return nil, errExpectedBool
}
return FromBool(!b), nil
}
func (e *ConditionOperand) evalNode(r Record, tableAlias string) (*Value, error) {
opVal, opErr := e.Operand.evalNode(r, tableAlias)
if opErr != nil || e.ConditionRHS == nil {
return opVal, opErr
}
// Need to evaluate the ConditionRHS
switch {
case e.ConditionRHS.Compare != nil:
cmpRight, cmpRErr := e.ConditionRHS.Compare.Operand.evalNode(r, tableAlias)
if cmpRErr != nil {
return nil, cmpRErr
}
b, err := opVal.compareOp(e.ConditionRHS.Compare.Operator, cmpRight)
return FromBool(b), err
case e.ConditionRHS.Between != nil:
return e.ConditionRHS.Between.evalBetweenNode(r, opVal, tableAlias)
case e.ConditionRHS.Like != nil:
return e.ConditionRHS.Like.evalLikeNode(r, opVal, tableAlias)
case e.ConditionRHS.In != nil:
return e.ConditionRHS.In.evalInNode(r, opVal, tableAlias)
default:
return nil, errInvalidASTNode
}
}
func (e *Between) evalBetweenNode(r Record, arg *Value, tableAlias string) (*Value, error) {
stVal, stErr := e.Start.evalNode(r, tableAlias)
if stErr != nil {
return nil, stErr
}
endVal, endErr := e.End.evalNode(r, tableAlias)
if endErr != nil {
return nil, endErr
}
part1, err1 := stVal.compareOp(opLte, arg)
if err1 != nil {
return nil, err1
}
part2, err2 := arg.compareOp(opLte, endVal)
if err2 != nil {
return nil, err2
}
result := part1 && part2
if e.Not {
result = !result
}
return FromBool(result), nil
}
func (e *Like) evalLikeNode(r Record, arg *Value, tableAlias string) (*Value, error) {
inferTypeAsString(arg)
s, ok := arg.ToString()
if !ok {
err := errLikeNonStrArg
return nil, errLikeInvalidInputs(err)
}
pattern, err1 := e.Pattern.evalNode(r, tableAlias)
if err1 != nil {
return nil, err1
}
// Infer pattern as string (in case it is untyped)
inferTypeAsString(pattern)
patternStr, ok := pattern.ToString()
if !ok {
err := errLikeNonStrArg
return nil, errLikeInvalidInputs(err)
}
escape := runeZero
if e.EscapeChar != nil {
escapeVal, err2 := e.EscapeChar.evalNode(r, tableAlias)
if err2 != nil {
return nil, err2
}
inferTypeAsString(escapeVal)
escapeStr, ok := escapeVal.ToString()
if !ok {
err := errLikeNonStrArg
return nil, errLikeInvalidInputs(err)
}
if len([]rune(escapeStr)) > 1 {
err := errLikeInvalidEscape
return nil, errLikeInvalidInputs(err)
}
}
matchResult, err := evalSQLLike(s, patternStr, escape)
if err != nil {
return nil, err
}
if e.Not {
matchResult = !matchResult
}
return FromBool(matchResult), nil
}
func (e *ListExpr) evalNode(r Record, tableAlias string) (*Value, error) {
res := make([]Value, len(e.Elements))
if len(e.Elements) == 1 {
// If length 1, treat as single value.
return e.Elements[0].evalNode(r, tableAlias)
}
for i, elt := range e.Elements {
v, err := elt.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
res[i] = *v
}
return FromArray(res), nil
}
const floatCmpTolerance = 0.000001
func (e *In) evalInNode(r Record, lhs *Value, tableAlias string) (*Value, error) {
// Compare two values in terms of in-ness.
var cmp func(a, b Value) bool
cmp = func(a, b Value) bool {
// Convert if needed.
inferTypesForCmp(&a, &b)
if a.Equals(b) {
return true
}
// If elements, compare each.
aA, aOK := a.ToArray()
bA, bOK := b.ToArray()
if aOK && bOK {
if len(aA) != len(bA) {
return false
}
for i := range aA {
if !cmp(aA[i], bA[i]) {
return false
}
}
return true
}
// Try as numbers
aF, aOK := a.ToFloat()
bF, bOK := b.ToFloat()
diff := math.Abs(aF - bF)
return aOK && bOK && diff < floatCmpTolerance
}
var rhs Value
if elt := e.ListExpression; elt != nil {
eltVal, err := elt.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
rhs = *eltVal
}
// If RHS is array compare each element.
if arr, ok := rhs.ToArray(); ok {
for _, element := range arr {
// If we have an array we are on the wrong level.
if cmp(element, *lhs) {
return FromBool(true), nil
}
}
return FromBool(false), nil
}
return FromBool(cmp(rhs, *lhs)), nil
}
func (e *Operand) evalNode(r Record, tableAlias string) (*Value, error) {
lval, lerr := e.Left.evalNode(r, tableAlias)
if lerr != nil || len(e.Right) == 0 {
return lval, lerr
}
// Process remaining child nodes - result must be
// numeric. This AST node is for terms separated by + or -
// symbols.
for _, rightTerm := range e.Right {
op := rightTerm.Op
rval, rerr := rightTerm.Right.evalNode(r, tableAlias)
if rerr != nil {
return nil, rerr
}
err := lval.arithOp(op, rval)
if err != nil {
return nil, err
}
}
return lval, nil
}
func (e *MultOp) evalNode(r Record, tableAlias string) (*Value, error) {
lval, lerr := e.Left.evalNode(r, tableAlias)
if lerr != nil || len(e.Right) == 0 {
return lval, lerr
}
// Process other child nodes - result must be numeric. This
// AST node is for terms separated by *, / or % symbols.
for _, rightTerm := range e.Right {
op := rightTerm.Op
rval, rerr := rightTerm.Right.evalNode(r, tableAlias)
if rerr != nil {
return nil, rerr
}
err := lval.arithOp(op, rval)
if err != nil {
return nil, err
}
}
return lval, nil
}
func (e *UnaryTerm) evalNode(r Record, tableAlias string) (*Value, error) {
if e.Negated == nil {
return e.Primary.evalNode(r, tableAlias)
}
v, err := e.Negated.Term.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
inferTypeForArithOp(v)
v.negate()
if v.isNumeric() {
return v, nil
}
return nil, errArithMismatchedTypes
}
func (e *JSONPath) evalNode(r Record, tableAlias string) (*Value, error) {
alias := tableAlias
if tableAlias == "" {
alias = baseTableName
}
pathExpr := e.StripTableAlias(alias)
_, rawVal := r.Raw()
switch rowVal := rawVal.(type) {
case jstream.KVS, simdjson.Object:
if len(pathExpr) == 0 {
pathExpr = []*JSONPathElement{{Key: &ObjectKey{ID: e.BaseKey}}}
}
result, _, err := jsonpathEval(pathExpr, rowVal)
if err != nil {
return nil, err
}
return jsonToValue(result)
default:
if pathExpr[len(pathExpr)-1].Key == nil {
return nil, errInvalidKeypath
}
return r.Get(pathExpr[len(pathExpr)-1].Key.keyString())
}
}
// jsonToValue will convert the json value to an internal value.
func jsonToValue(result interface{}) (*Value, error) {
switch rval := result.(type) {
case string:
return FromString(rval), nil
case float64:
return FromFloat(rval), nil
case int64:
return FromInt(rval), nil
case uint64:
if rval <= math.MaxInt64 {
return FromInt(int64(rval)), nil
}
return FromFloat(float64(rval)), nil
case bool:
return FromBool(rval), nil
case jstream.KVS:
bs, err := json.Marshal(result)
if err != nil {
return nil, err
}
return FromBytes(bs), nil
case []interface{}:
dst := make([]Value, len(rval))
for i := range rval {
v, err := jsonToValue(rval[i])
if err != nil {
return nil, err
}
dst[i] = *v
}
return FromArray(dst), nil
case simdjson.Object:
o := rval
elems, err := o.Parse(nil)
if err != nil {
return nil, err
}
bs, err := elems.MarshalJSON()
if err != nil {
return nil, err
}
return FromBytes(bs), nil
case []Value:
return FromArray(rval), nil
case nil:
return FromNull(), nil
}
return nil, fmt.Errorf("Unhandled value type: %T", result)
}
func (e *PrimaryTerm) evalNode(r Record, tableAlias string) (res *Value, err error) {
switch {
case e.Value != nil:
return e.Value.evalNode(r)
case e.JPathExpr != nil:
return e.JPathExpr.evalNode(r, tableAlias)
case e.ListExpr != nil:
return e.ListExpr.evalNode(r, tableAlias)
case e.SubExpression != nil:
return e.SubExpression.evalNode(r, tableAlias)
case e.FuncCall != nil:
return e.FuncCall.evalNode(r, tableAlias)
}
return nil, errInvalidASTNode
}
func (e *FuncExpr) evalNode(r Record, tableAlias string) (res *Value, err error) {
switch e.getFunctionName() {
case aggFnCount, aggFnAvg, aggFnMax, aggFnMin, aggFnSum:
return e.getAggregate()
default:
return e.evalSQLFnNode(r, tableAlias)
}
}
// evalNode on a literal value is independent of the node being an
// aggregation or a row function - it always returns a value.
func (e *LitValue) evalNode(_ Record) (res *Value, err error) {
switch {
case e.Int != nil:
if *e.Int < math.MaxInt64 && *e.Int > math.MinInt64 {
return FromInt(int64(*e.Int)), nil
}
return FromFloat(*e.Int), nil
case e.Float != nil:
return FromFloat(*e.Float), nil
case e.String != nil:
return FromString(string(*e.String)), nil
case e.Boolean != nil:
return FromBool(bool(*e.Boolean)), nil
}
return FromNull(), nil
}
internal/s3select/sql/funceval.go
+569
View File
@@ -0,0 +1,569 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"errors"
"fmt"
"strconv"
"strings"
"time"
)
// FuncName - SQL function name.
type FuncName string
// SQL Function name constants
const (
// Conditionals
sqlFnCoalesce FuncName = "COALESCE"
sqlFnNullIf FuncName = "NULLIF"
// Conversion
sqlFnCast FuncName = "CAST"
// Date and time
sqlFnDateAdd FuncName = "DATE_ADD"
sqlFnDateDiff FuncName = "DATE_DIFF"
sqlFnExtract FuncName = "EXTRACT"
sqlFnToString FuncName = "TO_STRING"
sqlFnToTimestamp FuncName = "TO_TIMESTAMP"
sqlFnUTCNow FuncName = "UTCNOW"
// String
sqlFnCharLength FuncName = "CHAR_LENGTH"
sqlFnCharacterLength FuncName = "CHARACTER_LENGTH"
sqlFnLower FuncName = "LOWER"
sqlFnSubstring FuncName = "SUBSTRING"
sqlFnTrim FuncName = "TRIM"
sqlFnUpper FuncName = "UPPER"
)
var (
errUnimplementedCast = errors.New("This cast not yet implemented")
errNonStringTrimArg = errors.New("TRIM() received a non-string argument")
errNonTimestampArg = errors.New("Expected a timestamp argument")
)
func (e *FuncExpr) getFunctionName() FuncName {
switch {
case e.SFunc != nil:
return FuncName(strings.ToUpper(e.SFunc.FunctionName))
case e.Count != nil:
return aggFnCount
case e.Cast != nil:
return sqlFnCast
case e.Substring != nil:
return sqlFnSubstring
case e.Extract != nil:
return sqlFnExtract
case e.Trim != nil:
return sqlFnTrim
case e.DateAdd != nil:
return sqlFnDateAdd
case e.DateDiff != nil:
return sqlFnDateDiff
default:
return ""
}
}
// evalSQLFnNode assumes that the FuncExpr is not an aggregation
// function.
func (e *FuncExpr) evalSQLFnNode(r Record, tableAlias string) (res *Value, err error) {
// Handle functions that have phrase arguments
switch e.getFunctionName() {
case sqlFnCast:
expr := e.Cast.Expr
res, err = expr.castTo(r, strings.ToUpper(e.Cast.CastType), tableAlias)
return
case sqlFnSubstring:
return handleSQLSubstring(r, e.Substring, tableAlias)
case sqlFnExtract:
return handleSQLExtract(r, e.Extract, tableAlias)
case sqlFnTrim:
return handleSQLTrim(r, e.Trim, tableAlias)
case sqlFnDateAdd:
return handleDateAdd(r, e.DateAdd, tableAlias)
case sqlFnDateDiff:
return handleDateDiff(r, e.DateDiff, tableAlias)
}
// For all simple argument functions, we evaluate the arguments here
argVals := make([]*Value, len(e.SFunc.ArgsList))
for i, arg := range e.SFunc.ArgsList {
argVals[i], err = arg.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
}
switch e.getFunctionName() {
case sqlFnCoalesce:
return coalesce(argVals)
case sqlFnNullIf:
return nullif(argVals[0], argVals[1])
case sqlFnCharLength, sqlFnCharacterLength:
return charlen(argVals[0])
case sqlFnLower:
return lowerCase(argVals[0])
case sqlFnUpper:
return upperCase(argVals[0])
case sqlFnUTCNow:
return handleUTCNow()
case sqlFnToString, sqlFnToTimestamp:
// TODO: implement
fallthrough
default:
return nil, errNotImplemented
}
}
func coalesce(args []*Value) (res *Value, err error) {
for _, arg := range args {
if arg.IsNull() {
continue
}
return arg, nil
}
return FromNull(), nil
}
func nullif(v1, v2 *Value) (res *Value, err error) {
// Handle Null cases
if v1.IsNull() || v2.IsNull() {
return v1, nil
}
err = inferTypesForCmp(v1, v2)
if err != nil {
return nil, err
}
atleastOneNumeric := v1.isNumeric() || v2.isNumeric()
bothNumeric := v1.isNumeric() && v2.isNumeric()
if atleastOneNumeric || !bothNumeric {
return v1, nil
}
if v1.SameTypeAs(*v2) {
return v1, nil
}
cmpResult, cmpErr := v1.compareOp(opEq, v2)
if cmpErr != nil {
return nil, cmpErr
}
if cmpResult {
return FromNull(), nil
}
return v1, nil
}
func charlen(v *Value) (*Value, error) {
inferTypeAsString(v)
s, ok := v.ToString()
if !ok {
err := fmt.Errorf("%s/%s expects a string argument", sqlFnCharLength, sqlFnCharacterLength)
return nil, errIncorrectSQLFunctionArgumentType(err)
}
return FromInt(int64(len([]rune(s)))), nil
}
func lowerCase(v *Value) (*Value, error) {
inferTypeAsString(v)
s, ok := v.ToString()
if !ok {
err := fmt.Errorf("%s expects a string argument", sqlFnLower)
return nil, errIncorrectSQLFunctionArgumentType(err)
}
return FromString(strings.ToLower(s)), nil
}
func upperCase(v *Value) (*Value, error) {
inferTypeAsString(v)
s, ok := v.ToString()
if !ok {
err := fmt.Errorf("%s expects a string argument", sqlFnUpper)
return nil, errIncorrectSQLFunctionArgumentType(err)
}
return FromString(strings.ToUpper(s)), nil
}
func handleDateAdd(r Record, d *DateAddFunc, tableAlias string) (*Value, error) {
q, err := d.Quantity.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
inferTypeForArithOp(q)
qty, ok := q.ToFloat()
if !ok {
return nil, fmt.Errorf("QUANTITY must be a numeric argument to %s()", sqlFnDateAdd)
}
ts, err := d.Timestamp.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
if err = inferTypeAsTimestamp(ts); err != nil {
return nil, err
}
t, ok := ts.ToTimestamp()
if !ok {
return nil, fmt.Errorf("%s() expects a timestamp argument", sqlFnDateAdd)
}
return dateAdd(strings.ToUpper(d.DatePart), qty, t)
}
func handleDateDiff(r Record, d *DateDiffFunc, tableAlias string) (*Value, error) {
tval1, err := d.Timestamp1.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
if err = inferTypeAsTimestamp(tval1); err != nil {
return nil, err
}
ts1, ok := tval1.ToTimestamp()
if !ok {
return nil, fmt.Errorf("%s() expects two timestamp arguments", sqlFnDateDiff)
}
tval2, err := d.Timestamp2.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
if err = inferTypeAsTimestamp(tval2); err != nil {
return nil, err
}
ts2, ok := tval2.ToTimestamp()
if !ok {
return nil, fmt.Errorf("%s() expects two timestamp arguments", sqlFnDateDiff)
}
return dateDiff(strings.ToUpper(d.DatePart), ts1, ts2)
}
func handleUTCNow() (*Value, error) {
return FromTimestamp(time.Now().UTC()), nil
}
func handleSQLSubstring(r Record, e *SubstringFunc, tableAlias string) (val *Value, err error) {
// Both forms `SUBSTRING('abc' FROM 2 FOR 1)` and
// SUBSTRING('abc', 2, 1) are supported.
// Evaluate the string argument
v1, err := e.Expr.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
inferTypeAsString(v1)
s, ok := v1.ToString()
if !ok {
err := fmt.Errorf("Incorrect argument type passed to %s", sqlFnSubstring)
return nil, errIncorrectSQLFunctionArgumentType(err)
}
// Assemble other arguments
arg2, arg3 := e.From, e.For
// Check if the second form of substring is being used
if e.From == nil {
arg2, arg3 = e.Arg2, e.Arg3
}
// Evaluate the FROM argument
v2, err := arg2.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
inferTypeForArithOp(v2)
startIdx, ok := v2.ToInt()
if !ok {
err := fmt.Errorf("Incorrect type for start index argument in %s", sqlFnSubstring)
return nil, errIncorrectSQLFunctionArgumentType(err)
}
length := -1
// Evaluate the optional FOR argument
if arg3 != nil {
v3, err := arg3.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
inferTypeForArithOp(v3)
lenInt, ok := v3.ToInt()
if !ok {
err := fmt.Errorf("Incorrect type for length argument in %s", sqlFnSubstring)
return nil, errIncorrectSQLFunctionArgumentType(err)
}
length = int(lenInt)
if length < 0 {
err := fmt.Errorf("Negative length argument in %s", sqlFnSubstring)
return nil, errIncorrectSQLFunctionArgumentType(err)
}
}
res, err := evalSQLSubstring(s, int(startIdx), length)
return FromString(res), err
}
func handleSQLTrim(r Record, e *TrimFunc, tableAlias string) (res *Value, err error) {
chars := ""
ok := false
if e.TrimChars != nil {
charsV, cerr := e.TrimChars.evalNode(r, tableAlias)
if cerr != nil {
return nil, cerr
}
inferTypeAsString(charsV)
chars, ok = charsV.ToString()
if !ok {
return nil, errNonStringTrimArg
}
}
fromV, ferr := e.TrimFrom.evalNode(r, tableAlias)
if ferr != nil {
return nil, ferr
}
inferTypeAsString(fromV)
from, ok := fromV.ToString()
if !ok {
return nil, errNonStringTrimArg
}
result, terr := evalSQLTrim(e.TrimWhere, chars, from)
if terr != nil {
return nil, terr
}
return FromString(result), nil
}
func handleSQLExtract(r Record, e *ExtractFunc, tableAlias string) (res *Value, err error) {
timeVal, verr := e.From.evalNode(r, tableAlias)
if verr != nil {
return nil, verr
}
if err = inferTypeAsTimestamp(timeVal); err != nil {
return nil, err
}
t, ok := timeVal.ToTimestamp()
if !ok {
return nil, errNonTimestampArg
}
return extract(strings.ToUpper(e.Timeword), t)
}
func errUnsupportedCast(fromType, toType string) error {
return fmt.Errorf("Cannot cast from %v to %v", fromType, toType)
}
func errCastFailure(msg string) error {
return fmt.Errorf("Error casting: %s", msg)
}
// Allowed cast types
const (
castBool = "BOOL"
castInt = "INT"
castInteger = "INTEGER"
castString = "STRING"
castFloat = "FLOAT"
castDecimal = "DECIMAL"
castNumeric = "NUMERIC"
castTimestamp = "TIMESTAMP"
)
func (e *Expression) castTo(r Record, castType string, tableAlias string) (res *Value, err error) {
v, err := e.evalNode(r, tableAlias)
if err != nil {
return nil, err
}
switch castType {
case castInt, castInteger:
i, err := intCast(v)
return FromInt(i), err
case castFloat:
f, err := floatCast(v)
return FromFloat(f), err
case castString:
s, err := stringCast(v)
return FromString(s), err
case castTimestamp:
t, err := timestampCast(v)
return FromTimestamp(t), err
case castBool:
b, err := boolCast(v)
return FromBool(b), err
case castDecimal, castNumeric:
fallthrough
default:
return nil, errUnimplementedCast
}
}
func intCast(v *Value) (int64, error) {
// This conversion truncates floating point numbers to
// integer.
strToInt := func(s string) (int64, bool) {
i, errI := strconv.ParseInt(s, 10, 64)
if errI == nil {
return i, true
}
f, errF := strconv.ParseFloat(s, 64)
if errF == nil {
return int64(f), true
}
return 0, false
}
switch x := v.value.(type) {
case float64:
// Truncate fractional part
return int64(x), nil
case int64:
return x, nil
case string:
// Parse as number, truncate floating point if
// needed.
// String might contain trimming spaces, which
// needs to be trimmed.
res, ok := strToInt(strings.TrimSpace(x))
if !ok {
return 0, errCastFailure("could not parse as int")
}
return res, nil
case []byte:
// Parse as number, truncate floating point if
// needed.
// String might contain trimming spaces, which
// needs to be trimmed.
res, ok := strToInt(strings.TrimSpace(string(x)))
if !ok {
return 0, errCastFailure("could not parse as int")
}
return res, nil
default:
return 0, errUnsupportedCast(v.GetTypeString(), castInt)
}
}
func floatCast(v *Value) (float64, error) {
switch x := v.value.(type) {
case float64:
return x, nil
case int64:
return float64(x), nil
case string:
f, err := strconv.ParseFloat(strings.TrimSpace(x), 64)
if err != nil {
return 0, errCastFailure("could not parse as float")
}
return f, nil
case []byte:
f, err := strconv.ParseFloat(strings.TrimSpace(string(x)), 64)
if err != nil {
return 0, errCastFailure("could not parse as float")
}
return f, nil
default:
return 0, errUnsupportedCast(v.GetTypeString(), castFloat)
}
}
func stringCast(v *Value) (string, error) {
switch x := v.value.(type) {
case float64:
return fmt.Sprintf("%v", x), nil
case int64:
return fmt.Sprintf("%v", x), nil
case string:
return x, nil
case []byte:
return string(x), nil
case bool:
return fmt.Sprintf("%v", x), nil
case nil:
// FIXME: verify this case is correct
return "NULL", nil
}
// This does not happen
return "", errCastFailure(fmt.Sprintf("cannot cast %v to string type", v.GetTypeString()))
}
func timestampCast(v *Value) (t time.Time, _ error) {
switch x := v.value.(type) {
case string:
return parseSQLTimestamp(x)
case []byte:
return parseSQLTimestamp(string(x))
case time.Time:
return x, nil
default:
return t, errCastFailure(fmt.Sprintf("cannot cast %v to Timestamp type", v.GetTypeString()))
}
}
func boolCast(v *Value) (b bool, _ error) {
sToB := func(s string) (bool, error) {
switch s {
case "true":
return true, nil
case "false":
return false, nil
default:
return false, errCastFailure("cannot cast to Bool")
}
}
switch x := v.value.(type) {
case bool:
return x, nil
case string:
return sToB(strings.ToLower(x))
case []byte:
return sToB(strings.ToLower(string(x)))
default:
return false, errCastFailure("cannot cast %v to Bool")
}
}
internal/s3select/sql/jsondata/books.json
+84
View File
@@ -0,0 +1,84 @@
{
"title": "Murder on the Orient Express",
"authorInfo": {
"name": "Agatha Christie",
"yearRange": [1890, 1976],
"penName": "Mary Westmacott"
},
"genre": "Crime novel",
"publicationHistory": [
{
"year": 1934,
"publisher": "Collins Crime Club (London)",
"type": "Hardcover",
"pages": 256
},
{
"year": 1934,
"publisher": "Dodd Mead and Company (New York)",
"type": "Hardcover",
"pages": 302
},
{
"year": 2011,
"publisher": "Harper Collins",
"type": "Paperback",
"pages": 265
}
]
}
{
"title": "The Robots of Dawn",
"authorInfo": {
"name": "Isaac Asimov",
"yearRange": [1920, 1992],
"penName": "Paul French"
},
"genre": "Science fiction",
"publicationHistory": [
{
"year": 1983,
"publisher": "Phantasia Press",
"type": "Hardcover",
"pages": 336
},
{
"year": 1984,
"publisher": "Granada",
"type": "Hardcover",
"pages": 419
},
{
"year": 2018,
"publisher": "Harper Voyager",
"type": "Paperback",
"pages": 432
}
]
}
{
"title": "Pigs Have Wings",
"authorInfo": {
"name": "P. G. Wodehouse",
"yearRange": [1881, 1975]
},
"genre": "Comic novel",
"publicationHistory": [
{
"year": 1952,
"publisher": "Doubleday & Company",
"type": "Hardcover"
},
{
"year": 2000,
"publisher": "Harry N. Abrams",
"type": "Hardcover"
},
{
"year": 2019,
"publisher": "Ulverscroft Collections",
"type": "Paperback",
"pages": 294
}
]
}
internal/s3select/sql/jsonpath.go
+129
View File
@@ -0,0 +1,129 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"errors"
"github.com/bcicen/jstream"
"github.com/minio/simdjson-go"
)
var (
errKeyLookup = errors.New("Cannot look up key in non-object value")
errIndexLookup = errors.New("Cannot look up array index in non-array value")
errWildcardObjectLookup = errors.New("Object wildcard used on non-object value")
errWildcardArrayLookup = errors.New("Array wildcard used on non-array value")
errWilcardObjectUsageInvalid = errors.New("Invalid usage of object wildcard")
)
// jsonpathEval evaluates a JSON path and returns the value at the path.
// If the value should be considered flat (from wildcards) any array returned should be considered individual values.
func jsonpathEval(p []*JSONPathElement, v interface{}) (r interface{}, flat bool, err error) {
// fmt.Printf("JPATHexpr: %v jsonobj: %v\n\n", p, v)
if len(p) == 0 || v == nil {
return v, false, nil
}
switch {
case p[0].Key != nil:
key := p[0].Key.keyString()
switch kvs := v.(type) {
case jstream.KVS:
for _, kv := range kvs {
if kv.Key == key {
return jsonpathEval(p[1:], kv.Value)
}
}
// Key not found - return nil result
return nil, false, nil
case simdjson.Object:
elem := kvs.FindKey(key, nil)
if elem == nil {
// Key not found - return nil result
return nil, false, nil
}
val, err := IterToValue(elem.Iter)
if err != nil {
return nil, false, err
}
return jsonpathEval(p[1:], val)
default:
return nil, false, errKeyLookup
}
case p[0].Index != nil:
idx := *p[0].Index
arr, ok := v.([]interface{})
if !ok {
return nil, false, errIndexLookup
}
if idx >= len(arr) {
return nil, false, nil
}
return jsonpathEval(p[1:], arr[idx])
case p[0].ObjectWildcard:
switch kvs := v.(type) {
case jstream.KVS:
if len(p[1:]) > 0 {
return nil, false, errWilcardObjectUsageInvalid
}
return kvs, false, nil
case simdjson.Object:
if len(p[1:]) > 0 {
return nil, false, errWilcardObjectUsageInvalid
}
return kvs, false, nil
default:
return nil, false, errWildcardObjectLookup
}
case p[0].ArrayWildcard:
arr, ok := v.([]interface{})
if !ok {
return nil, false, errWildcardArrayLookup
}
// Lookup remainder of path in each array element and
// make result array.
var result []interface{}
for _, a := range arr {
rval, flatten, err := jsonpathEval(p[1:], a)
if err != nil {
return nil, false, err
}
if flatten {
// Flatten if array.
if arr, ok := rval.([]interface{}); ok {
result = append(result, arr...)
continue
}
}
result = append(result, rval)
}
return result, true, nil
}
panic("cannot reach here")
}
internal/s3select/sql/jsonpath_test.go
+97
View File
@@ -0,0 +1,97 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"reflect"
"testing"
"github.com/alecthomas/participle"
"github.com/bcicen/jstream"
)
func getJSONStructs(b []byte) ([]interface{}, error) {
dec := jstream.NewDecoder(bytes.NewBuffer(b), 0).ObjectAsKVS()
var result []interface{}
for parsedVal := range dec.Stream() {
result = append(result, parsedVal.Value)
}
if err := dec.Err(); err != nil {
return nil, err
}
return result, nil
}
func TestJsonpathEval(t *testing.T) {
f, err := os.Open(filepath.Join("jsondata", "books.json"))
if err != nil {
t.Fatal(err)
}
b, err := ioutil.ReadAll(f)
if err != nil {
t.Fatal(err)
}
p := participle.MustBuild(
&JSONPath{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
)
cases := []struct {
str string
res []interface{}
}{
{"s.title", []interface{}{"Murder on the Orient Express", "The Robots of Dawn", "Pigs Have Wings"}},
{"s.authorInfo.yearRange", []interface{}{[]interface{}{1890.0, 1976.0}, []interface{}{1920.0, 1992.0}, []interface{}{1881.0, 1975.0}}},
{"s.authorInfo.name", []interface{}{"Agatha Christie", "Isaac Asimov", "P. G. Wodehouse"}},
{"s.authorInfo.yearRange[0]", []interface{}{1890.0, 1920.0, 1881.0}},
{"s.publicationHistory[0].pages", []interface{}{256.0, 336.0, nil}},
}
for i, tc := range cases {
jp := JSONPath{}
err := p.ParseString(tc.str, &jp)
// fmt.Println(jp)
if err != nil {
t.Fatalf("parse failed!: %d %v %s", i, err, tc)
}
// Read only the first json object from the file
recs, err := getJSONStructs(b)
if err != nil || len(recs) != 3 {
t.Fatalf("%v or length was not 3", err)
}
for j, rec := range recs {
// fmt.Println(rec)
r, _, err := jsonpathEval(jp.PathExpr, rec)
if err != nil {
t.Errorf("Error: %d %d %v", i, j, err)
}
if !reflect.DeepEqual(r, tc.res[j]) {
fmt.Printf("%#v (%v) != %v (%v)\n", r, reflect.TypeOf(r), tc.res[j], reflect.TypeOf(tc.res[j]))
t.Errorf("case: %d %d failed", i, j)
}
}
}
}
internal/s3select/sql/parser.go
+371
View File
@@ -0,0 +1,371 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"strings"
"github.com/alecthomas/participle"
"github.com/alecthomas/participle/lexer"
)
// Types with custom Capture interface for parsing
// Boolean is a type for a parsed Boolean literal
type Boolean bool
// Capture interface used by participle
func (b *Boolean) Capture(values []string) error {
*b = strings.ToLower(values[0]) == "true"
return nil
}
// LiteralString is a type for parsed SQL string literals
type LiteralString string
// Capture interface used by participle
func (ls *LiteralString) Capture(values []string) error {
// Remove enclosing single quote
n := len(values[0])
r := values[0][1 : n-1]
// Translate doubled quotes
*ls = LiteralString(strings.Replace(r, "''", "'", -1))
return nil
}
// LiteralList is a type for parsed SQL lists literals
type LiteralList []string
// Capture interface used by participle
func (ls *LiteralList) Capture(values []string) error {
// Remove enclosing parenthesis.
n := len(values[0])
r := values[0][1 : n-1]
// Translate doubled quotes
*ls = LiteralList(strings.Split(r, ","))
return nil
}
// ObjectKey is a type for parsed strings occurring in key paths
type ObjectKey struct {
Lit *LiteralString `parser:" \"[\" @LitString \"]\""`
ID *Identifier `parser:"| \".\" @@"`
}
// QuotedIdentifier is a type for parsed strings that are double
// quoted.
type QuotedIdentifier string
// Capture interface used by participle
func (qi *QuotedIdentifier) Capture(values []string) error {
// Remove enclosing quotes
n := len(values[0])
r := values[0][1 : n-1]
// Translate doubled quotes
*qi = QuotedIdentifier(strings.Replace(r, `""`, `"`, -1))
return nil
}
// Types representing AST of SQL statement. Only SELECT is supported.
// Select is the top level AST node type
type Select struct {
Expression *SelectExpression `parser:"\"SELECT\" @@"`
From *TableExpression `parser:"\"FROM\" @@"`
Where *Expression `parser:"( \"WHERE\" @@ )?"`
Limit *LitValue `parser:"( \"LIMIT\" @@ )?"`
}
// SelectExpression represents the items requested in the select
// statement
type SelectExpression struct {
All bool `parser:" @\"*\""`
Expressions []*AliasedExpression `parser:"| @@ { \",\" @@ }"`
}
// TableExpression represents the FROM clause
type TableExpression struct {
Table *JSONPath `parser:"@@"`
As string `parser:"( \"AS\"? @Ident )?"`
}
// JSONPathElement represents a keypath component
type JSONPathElement struct {
Key *ObjectKey `parser:" @@"` // ['name'] and .name forms
Index *int `parser:"| \"[\" @Int \"]\""` // [3] form
ObjectWildcard bool `parser:"| @\".*\""` // .* form
ArrayWildcard bool `parser:"| @\"[*]\""` // [*] form
}
// JSONPath represents a keypath.
// Instances should be treated idempotent and not change once created.
type JSONPath struct {
BaseKey *Identifier `parser:" @@"`
PathExpr []*JSONPathElement `parser:"(@@)*"`
// Cached values:
pathString string
strippedTableAlias string
strippedPathExpr []*JSONPathElement
}
// AliasedExpression is an expression that can be optionally named
type AliasedExpression struct {
Expression *Expression `parser:"@@"`
As string `parser:"[ \"AS\" @Ident ]"`
}
// Grammar for Expression
//
// Expression → AndCondition ("OR" AndCondition)*
// AndCondition → Condition ("AND" Condition)*
// Condition → "NOT" Condition | ConditionExpression
// ConditionExpression → ValueExpression ("=" | "<>" | "<=" | ">=" | "<" | ">") ValueExpression
// | ValueExpression "LIKE" ValueExpression ("ESCAPE" LitString)?
// | ValueExpression ("NOT"? "BETWEEN" ValueExpression "AND" ValueExpression)
// | ValueExpression "IN" "(" Expression ("," Expression)* ")"
// | ValueExpression
// ValueExpression → Operand
//
// Operand grammar follows below
// Expression represents a logical disjunction of clauses
type Expression struct {
And []*AndCondition `parser:"@@ ( \"OR\" @@ )*"`
}
// ListExpr represents a literal list with elements as expressions.
type ListExpr struct {
Elements []*Expression `parser:"\"(\" @@ ( \",\" @@ )* \")\" | \"[\" @@ ( \",\" @@ )* \"]\""`
}
// AndCondition represents logical conjunction of clauses
type AndCondition struct {
Condition []*Condition `parser:"@@ ( \"AND\" @@ )*"`
}
// Condition represents a negation or a condition operand
type Condition struct {
Operand *ConditionOperand `parser:" @@"`
Not *Condition `parser:"| \"NOT\" @@"`
}
// ConditionOperand is a operand followed by an an optional operation
// expression
type ConditionOperand struct {
Operand *Operand `parser:"@@"`
ConditionRHS *ConditionRHS `parser:"@@?"`
}
// ConditionRHS represents the right-hand-side of Compare, Between, In
// or Like expressions.
type ConditionRHS struct {
Compare *Compare `parser:" @@"`
Between *Between `parser:"| @@"`
In *In `parser:"| \"IN\" @@"`
Like *Like `parser:"| @@"`
}
// Compare represents the RHS of a comparison expression
type Compare struct {
Operator string `parser:"@( \"<>\" | \"<=\" | \">=\" | \"=\" | \"<\" | \">\" | \"!=\" )"`
Operand *Operand `parser:" @@"`
}
// Like represents the RHS of a LIKE expression
type Like struct {
Not bool `parser:" @\"NOT\"? "`
Pattern *Operand `parser:" \"LIKE\" @@ "`
EscapeChar *Operand `parser:" (\"ESCAPE\" @@)? "`
}
// Between represents the RHS of a BETWEEN expression
type Between struct {
Not bool `parser:" @\"NOT\"? "`
Start *Operand `parser:" \"BETWEEN\" @@ "`
End *Operand `parser:" \"AND\" @@ "`
}
// In represents the RHS of an IN expression
type In struct {
ListExpression *Expression `parser:"@@ "`
}
// Grammar for Operand:
//
// operand → multOp ( ("-" | "+") multOp )*
// multOp → unary ( ("/" | "*" | "%") unary )*
// unary → "-" unary | primary
// primary → Value | Variable | "(" expression ")"
//
// An Operand is a single term followed by an optional sequence of
// terms separated by +/-
type Operand struct {
Left *MultOp `parser:"@@"`
Right []*OpFactor `parser:"(@@)*"`
}
// OpFactor represents the right-side of a +/- operation.
type OpFactor struct {
Op string `parser:"@(\"+\" | \"-\")"`
Right *MultOp `parser:"@@"`
}
// MultOp represents a single term followed by an optional sequence of
// terms separated by *, / or % operators.
type MultOp struct {
Left *UnaryTerm `parser:"@@"`
Right []*OpUnaryTerm `parser:"(@@)*"`
}
// OpUnaryTerm represents the right side of *, / or % binary operations.
type OpUnaryTerm struct {
Op string `parser:"@(\"*\" | \"/\" | \"%\")"`
Right *UnaryTerm `parser:"@@"`
}
// UnaryTerm represents a single negated term or a primary term
type UnaryTerm struct {
Negated *NegatedTerm `parser:" @@"`
Primary *PrimaryTerm `parser:"| @@"`
}
// NegatedTerm has a leading minus sign.
type NegatedTerm struct {
Term *PrimaryTerm `parser:"\"-\" @@"`
}
// PrimaryTerm represents a Value, Path expression, a Sub-expression
// or a function call.
type PrimaryTerm struct {
Value *LitValue `parser:" @@"`
JPathExpr *JSONPath `parser:"| @@"`
ListExpr *ListExpr `parser:"| @@"`
SubExpression *Expression `parser:"| \"(\" @@ \")\""`
// Include function expressions here.
FuncCall *FuncExpr `parser:"| @@"`
}
// FuncExpr represents a function call
type FuncExpr struct {
SFunc *SimpleArgFunc `parser:" @@"`
Count *CountFunc `parser:"| @@"`
Cast *CastFunc `parser:"| @@"`
Substring *SubstringFunc `parser:"| @@"`
Extract *ExtractFunc `parser:"| @@"`
Trim *TrimFunc `parser:"| @@"`
DateAdd *DateAddFunc `parser:"| @@"`
DateDiff *DateDiffFunc `parser:"| @@"`
// Used during evaluation for aggregation funcs
aggregate *aggVal
}
// SimpleArgFunc represents functions with simple expression
// arguments.
type SimpleArgFunc struct {
FunctionName string `parser:" @(\"AVG\" | \"MAX\" | \"MIN\" | \"SUM\" | \"COALESCE\" | \"NULLIF\" | \"TO_STRING\" | \"TO_TIMESTAMP\" | \"UTCNOW\" | \"CHAR_LENGTH\" | \"CHARACTER_LENGTH\" | \"LOWER\" | \"UPPER\") "`
ArgsList []*Expression `parser:"\"(\" (@@ (\",\" @@)*)?\")\""`
}
// CountFunc represents the COUNT sql function
type CountFunc struct {
StarArg bool `parser:" \"COUNT\" \"(\" ( @\"*\"?"`
ExprArg *Expression `parser:" @@? )! \")\""`
}
// CastFunc represents CAST sql function
type CastFunc struct {
Expr *Expression `parser:" \"CAST\" \"(\" @@ "`
CastType string `parser:" \"AS\" @(\"BOOL\" | \"INT\" | \"INTEGER\" | \"STRING\" | \"FLOAT\" | \"DECIMAL\" | \"NUMERIC\" | \"TIMESTAMP\") \")\" "`
}
// SubstringFunc represents SUBSTRING sql function
type SubstringFunc struct {
Expr *PrimaryTerm `parser:" \"SUBSTRING\" \"(\" @@ "`
From *Operand `parser:" ( \"FROM\" @@ "`
For *Operand `parser:" (\"FOR\" @@)? \")\" "`
Arg2 *Operand `parser:" | \",\" @@ "`
Arg3 *Operand `parser:" (\",\" @@)? \")\" )"`
}
// ExtractFunc represents EXTRACT sql function
type ExtractFunc struct {
Timeword string `parser:" \"EXTRACT\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword | \"TIMEZONE_HOUR\":Timeword | \"TIMEZONE_MINUTE\":Timeword ) "`
From *PrimaryTerm `parser:" \"FROM\" @@ \")\" "`
}
// TrimFunc represents TRIM sql function
type TrimFunc struct {
TrimWhere *string `parser:" \"TRIM\" \"(\" ( @( \"LEADING\" | \"TRAILING\" | \"BOTH\" ) "`
TrimChars *PrimaryTerm `parser:" @@? "`
TrimFrom *PrimaryTerm `parser:" \"FROM\" )? @@ \")\" "`
}
// DateAddFunc represents the DATE_ADD function
type DateAddFunc struct {
DatePart string `parser:" \"DATE_ADD\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword ) \",\""`
Quantity *Operand `parser:" @@ \",\""`
Timestamp *PrimaryTerm `parser:" @@ \")\""`
}
// DateDiffFunc represents the DATE_DIFF function
type DateDiffFunc struct {
DatePart string `parser:" \"DATE_DIFF\" \"(\" @( \"YEAR\":Timeword | \"MONTH\":Timeword | \"DAY\":Timeword | \"HOUR\":Timeword | \"MINUTE\":Timeword | \"SECOND\":Timeword ) \",\" "`
Timestamp1 *PrimaryTerm `parser:" @@ \",\" "`
Timestamp2 *PrimaryTerm `parser:" @@ \")\" "`
}
// LitValue represents a literal value parsed from the sql
type LitValue struct {
Float *float64 `parser:"( @Float"`
Int *float64 `parser:" | @Int"` // To avoid value out of range, use float64 instead
String *LiteralString `parser:" | @LitString"`
Boolean *Boolean `parser:" | @(\"TRUE\" | \"FALSE\")"`
Null bool `parser:" | @\"NULL\")"`
}
// Identifier represents a parsed identifier
type Identifier struct {
Unquoted *string `parser:" @Ident"`
Quoted *QuotedIdentifier `parser:"| @QuotIdent"`
}
var (
sqlLexer = lexer.Must(lexer.Regexp(`(\s+)` +
`|(?P<Timeword>(?i)\b(?:YEAR|MONTH|DAY|HOUR|MINUTE|SECOND|TIMEZONE_HOUR|TIMEZONE_MINUTE)\b)` +
`|(?P<Keyword>(?i)\b(?:SELECT|FROM|TOP|DISTINCT|ALL|WHERE|GROUP|BY|HAVING|UNION|MINUS|EXCEPT|INTERSECT|ORDER|LIMIT|OFFSET|TRUE|FALSE|NULL|IS|NOT|ANY|SOME|BETWEEN|AND|OR|LIKE|ESCAPE|AS|IN|BOOL|INT|INTEGER|STRING|FLOAT|DECIMAL|NUMERIC|TIMESTAMP|AVG|COUNT|MAX|MIN|SUM|COALESCE|NULLIF|CAST|DATE_ADD|DATE_DIFF|EXTRACT|TO_STRING|TO_TIMESTAMP|UTCNOW|CHAR_LENGTH|CHARACTER_LENGTH|LOWER|SUBSTRING|TRIM|UPPER|LEADING|TRAILING|BOTH|FOR)\b)` +
`|(?P<Ident>[a-zA-Z_][a-zA-Z0-9_]*)` +
`|(?P<QuotIdent>"([^"]*("")?)*")` +
`|(?P<Float>\d*\.\d+([eE][-+]?\d+)?)` +
`|(?P<Int>\d+)` +
`|(?P<LitString>'([^']*('')?)*')` +
`|(?P<Operators><>|!=|<=|>=|\.\*|\[\*\]|[-+*/%,.()=<>\[\]])`,
))
// SQLParser is used to parse SQL statements
SQLParser = participle.MustBuild(
&Select{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
participle.CaseInsensitive("Timeword"),
)
)
internal/s3select/sql/parser_test.go
+386
View File
@@ -0,0 +1,386 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"bytes"
"testing"
"github.com/alecthomas/participle"
"github.com/alecthomas/participle/lexer"
)
func TestJSONPathElement(t *testing.T) {
p := participle.MustBuild(
&JSONPathElement{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
participle.CaseInsensitive("Timeword"),
)
j := JSONPathElement{}
cases := []string{
// Key
"['name']", ".name", `."name"`,
// Index
"[2]", "[0]", "[100]",
// Object wilcard
".*",
// array wildcard
"[*]",
}
for i, tc := range cases {
err := p.ParseString(tc, &j)
if err != nil {
t.Fatalf("%d: %v", i, err)
}
// repr.Println(j, repr.Indent(" "), repr.OmitEmpty(true))
}
}
func TestJSONPath(t *testing.T) {
p := participle.MustBuild(
&JSONPath{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
participle.CaseInsensitive("Timeword"),
)
j := JSONPath{}
cases := []string{
"S3Object",
"S3Object.id",
"S3Object.book.title",
"S3Object.id[1]",
"S3Object.id['abc']",
"S3Object.id['ab']",
"S3Object.words.*.id",
"S3Object.words.name[*].val",
"S3Object.words.name[*].val[*]",
"S3Object.words.name[*].val.*",
}
for i, tc := range cases {
err := p.ParseString(tc, &j)
if err != nil {
t.Fatalf("%d: %v", i, err)
}
// repr.Println(j, repr.Indent(" "), repr.OmitEmpty(true))
}
}
func TestIdentifierParsing(t *testing.T) {
p := participle.MustBuild(
&Identifier{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
)
id := Identifier{}
validCases := []string{
"a",
"_a",
"abc_a",
"a2",
`"abc"`,
`"abc\a""ac"`,
}
for i, tc := range validCases {
err := p.ParseString(tc, &id)
if err != nil {
t.Fatalf("%d: %v", i, err)
}
// repr.Println(id, repr.Indent(" "), repr.OmitEmpty(true))
}
invalidCases := []string{
"+a",
"-a",
"1a",
`"ab`,
`abc"`,
`aa""a`,
`"a"a"`,
}
for i, tc := range invalidCases {
err := p.ParseString(tc, &id)
if err == nil {
t.Fatalf("%d: %v", i, err)
}
// fmt.Println(tc, err)
}
}
func TestLiteralStringParsing(t *testing.T) {
var k ObjectKey
p := participle.MustBuild(
&ObjectKey{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
)
validCases := []string{
"['abc']",
"['ab''c']",
"['a''b''c']",
"['abc-x_1##@(*&(#*))/\\']",
}
for i, tc := range validCases {
err := p.ParseString(tc, &k)
if err != nil {
t.Fatalf("%d: %v", i, err)
}
if string(*k.Lit) == "" {
t.Fatalf("Incorrect parse %#v", k)
}
// repr.Println(k, repr.Indent(" "), repr.OmitEmpty(true))
}
invalidCases := []string{
"['abc'']",
"['-abc'sc']",
"[abc']",
"['ac]",
}
for i, tc := range invalidCases {
err := p.ParseString(tc, &k)
if err == nil {
t.Fatalf("%d: %v", i, err)
}
// fmt.Println(tc, err)
}
}
func TestFunctionParsing(t *testing.T) {
var fex FuncExpr
p := participle.MustBuild(
&FuncExpr{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
participle.CaseInsensitive("Timeword"),
)
validCases := []string{
"count(*)",
"sum(2 + s.id)",
"sum(t)",
"avg(s.id[1])",
"coalesce(s.id[1], 2, 2 + 3)",
"cast(s as string)",
"cast(s AS INT)",
"cast(s as DECIMAL)",
"extract(YEAR from '2018-01-09')",
"extract(month from '2018-01-09')",
"extract(hour from '2018-01-09')",
"extract(day from '2018-01-09')",
"substring('abcd' from 2 for 2)",
"substring('abcd' from 2)",
"substring('abcd' , 2 , 2)",
"substring('abcd' , 22 )",
"trim(' aab ')",
"trim(leading from ' aab ')",
"trim(trailing from ' aab ')",
"trim(both from ' aab ')",
"trim(both '12' from ' aab ')",
"trim(leading '12' from ' aab ')",
"trim(trailing '12' from ' aab ')",
"count(23)",
}
for i, tc := range validCases {
err := p.ParseString(tc, &fex)
if err != nil {
t.Fatalf("%d: %v", i, err)
}
// repr.Println(fex, repr.Indent(" "), repr.OmitEmpty(true))
}
}
func TestSqlLexer(t *testing.T) {
// s := bytes.NewBuffer([]byte("s.['name'].*.[*].abc.[\"abc\"]"))
s := bytes.NewBuffer([]byte("S3Object.words.*.id"))
// s := bytes.NewBuffer([]byte("COUNT(Id)"))
lex, err := sqlLexer.Lex(s)
if err != nil {
t.Fatal(err)
}
tokens, err := lexer.ConsumeAll(lex)
if err != nil {
t.Fatal(err)
}
// for i, t := range tokens {
// fmt.Printf("%d: %#v\n", i, t)
// }
if len(tokens) != 7 {
t.Fatalf("Expected 7 got %d", len(tokens))
}
}
func TestSelectWhere(t *testing.T) {
p := participle.MustBuild(
&Select{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
)
s := Select{}
cases := []string{
"select * from s3object",
"select a, b from s3object s",
"select a, b from s3object as s",
"select a, b from s3object as s where a = 1",
"select a, b from s3object s where a = 1",
"select a, b from s3object where a = 1",
}
for i, tc := range cases {
err := p.ParseString(tc, &s)
if err != nil {
t.Fatalf("%d: %v", i, err)
}
// repr.Println(s, repr.Indent(" "), repr.OmitEmpty(true))
}
}
func TestLikeClause(t *testing.T) {
p := participle.MustBuild(
&Select{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
)
s := Select{}
cases := []string{
`select * from s3object where Name like 'abcd'`,
`select Name like 'abc' from s3object`,
`select * from s3object where Name not like 'abc'`,
`select * from s3object where Name like 'abc' escape 't'`,
`select * from s3object where Name like 'a\%' escape '?'`,
`select * from s3object where Name not like 'abc\' escape '?'`,
`select * from s3object where Name like 'a\%' escape LOWER('?')`,
`select * from s3object where Name not like LOWER('Bc\') escape '?'`,
}
for i, tc := range cases {
err := p.ParseString(tc, &s)
if err != nil {
t.Errorf("%d: %v", i, err)
}
}
}
func TestBetweenClause(t *testing.T) {
p := participle.MustBuild(
&Select{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
)
s := Select{}
cases := []string{
`select * from s3object where Id between 1 and 2`,
`select * from s3object where Id between 1 and 2 and name = 'Ab'`,
`select * from s3object where Id not between 1 and 2`,
`select * from s3object where Id not between 1 and 2 and name = 'Bc'`,
}
for i, tc := range cases {
err := p.ParseString(tc, &s)
if err != nil {
t.Errorf("%d: %v", i, err)
}
}
}
func TestFromClauseJSONPath(t *testing.T) {
p := participle.MustBuild(
&Select{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
)
s := Select{}
cases := []string{
"select * from s3object",
"select * from s3object[*].name",
"select * from s3object[*].books[*]",
"select * from s3object[*].books[*].name",
"select * from s3object where name > 2",
"select * from s3object[*].name where name > 2",
"select * from s3object[*].books[*] where name > 2",
"select * from s3object[*].books[*].name where name > 2",
"select * from s3object[*].books[*] s",
"select * from s3object[*].books[*].name as s",
"select * from s3object s where name > 2",
"select * from s3object[*].name as s where name > 2",
}
for i, tc := range cases {
err := p.ParseString(tc, &s)
if err != nil {
t.Fatalf("%d: %v", i, err)
}
// repr.Println(s, repr.Indent(" "), repr.OmitEmpty(true))
}
}
func TestSelectParsing(t *testing.T) {
p := participle.MustBuild(
&Select{},
participle.Lexer(sqlLexer),
participle.CaseInsensitive("Keyword"),
)
s := Select{}
cases := []string{
"select * from s3object where name > 2 or value > 1 or word > 2",
"select s.word.id + 2 from s3object s",
"select 1-2-3 from s3object s limit 1",
}
for i, tc := range cases {
err := p.ParseString(tc, &s)
if err != nil {
t.Fatalf("%d: %v", i, err)
}
// repr.Println(s, repr.Indent(" "), repr.OmitEmpty(true))
}
}
func TestSqlLexerArithOps(t *testing.T) {
s := bytes.NewBuffer([]byte("year from select month hour distinct"))
lex, err := sqlLexer.Lex(s)
if err != nil {
t.Fatal(err)
}
tokens, err := lexer.ConsumeAll(lex)
if err != nil {
t.Fatal(err)
}
if len(tokens) != 7 {
t.Errorf("Expected 7 got %d", len(tokens))
}
// for i, t := range tokens {
// fmt.Printf("%d: %#v\n", i, t)
// }
}
internal/s3select/sql/record.go
+142
View File
@@ -0,0 +1,142 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"fmt"
"io"
"github.com/minio/simdjson-go"
)
// SelectObjectFormat specifies the format of the underlying data
type SelectObjectFormat int
const (
// SelectFmtUnknown - unknown format (default value)
SelectFmtUnknown SelectObjectFormat = iota
// SelectFmtCSV - CSV format
SelectFmtCSV
// SelectFmtJSON - JSON format
SelectFmtJSON
// SelectFmtSIMDJSON - SIMD JSON format
SelectFmtSIMDJSON
// SelectFmtParquet - Parquet format
SelectFmtParquet
)
// WriteCSVOpts - encapsulates options for Select CSV output
type WriteCSVOpts struct {
FieldDelimiter rune
Quote rune
QuoteEscape rune
AlwaysQuote bool
}
// Record - is a type containing columns and their values.
type Record interface {
Get(name string) (*Value, error)
// Set a value.
// Can return a different record type.
Set(name string, value *Value) (Record, error)
WriteCSV(writer io.Writer, opts WriteCSVOpts) error
WriteJSON(writer io.Writer) error
// Clone the record and if possible use the destination provided.
Clone(dst Record) Record
Reset()
// Returns underlying representation
Raw() (SelectObjectFormat, interface{})
// Replaces the underlying data
Replace(k interface{}) error
}
// IterToValue converts a simdjson Iter to its underlying value.
// Objects are returned as simdjson.Object
// Arrays are returned as []interface{} with parsed values.
func IterToValue(iter simdjson.Iter) (interface{}, error) {
switch iter.Type() {
case simdjson.TypeString:
v, err := iter.String()
if err != nil {
return nil, err
}
return v, nil
case simdjson.TypeFloat:
v, err := iter.Float()
if err != nil {
return nil, err
}
return v, nil
case simdjson.TypeInt:
v, err := iter.Int()
if err != nil {
return nil, err
}
return v, nil
case simdjson.TypeUint:
v, err := iter.Int()
if err != nil {
// Can't fit into int, convert to float.
v, err := iter.Float()
return v, err
}
return v, nil
case simdjson.TypeBool:
v, err := iter.Bool()
if err != nil {
return nil, err
}
return v, nil
case simdjson.TypeObject:
obj, err := iter.Object(nil)
if err != nil {
return nil, err
}
return *obj, err
case simdjson.TypeArray:
arr, err := iter.Array(nil)
if err != nil {
return nil, err
}
iter := arr.Iter()
var dst []interface{}
var next simdjson.Iter
for {
typ, err := iter.AdvanceIter(&next)
if err != nil {
return nil, err
}
if typ == simdjson.TypeNone {
break
}
v, err := IterToValue(next)
if err != nil {
return nil, err
}
dst = append(dst, v)
}
return dst, err
case simdjson.TypeNull:
return nil, nil
}
return nil, fmt.Errorf("IterToValue: unknown JSON type: %s", iter.Type().String())
}
internal/s3select/sql/statement.go
+345
View File
@@ -0,0 +1,345 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"errors"
"fmt"
"strings"
"github.com/bcicen/jstream"
"github.com/minio/simdjson-go"
)
var (
errBadLimitSpecified = errors.New("Limit value must be a positive integer")
)
const (
baseTableName = "s3object"
)
// SelectStatement is the top level parsed and analyzed structure
type SelectStatement struct {
selectAST *Select
// Analysis result of the statement
selectQProp qProp
// Result of parsing the limit clause if one is present
// (otherwise -1)
limitValue int64
// Count of rows that have been output.
outputCount int64
// Table alias
tableAlias string
}
// ParseSelectStatement - parses a select query from the given string
// and analyzes it.
func ParseSelectStatement(s string) (stmt SelectStatement, err error) {
var selectAST Select
err = SQLParser.ParseString(s, &selectAST)
if err != nil {
err = errQueryParseFailure(err)
return
}
// Check if select is "SELECT s.* from S3Object s"
if !selectAST.Expression.All &&
len(selectAST.Expression.Expressions) == 1 &&
len(selectAST.Expression.Expressions[0].Expression.And) == 1 &&
len(selectAST.Expression.Expressions[0].Expression.And[0].Condition) == 1 &&
selectAST.Expression.Expressions[0].Expression.And[0].Condition[0].Operand != nil &&
selectAST.Expression.Expressions[0].Expression.And[0].Condition[0].Operand.Operand.Left != nil &&
selectAST.Expression.Expressions[0].Expression.And[0].Condition[0].Operand.Operand.Left.Left != nil &&
selectAST.Expression.Expressions[0].Expression.And[0].Condition[0].Operand.Operand.Left.Left.Primary != nil &&
selectAST.Expression.Expressions[0].Expression.And[0].Condition[0].Operand.Operand.Left.Left.Primary.JPathExpr != nil {
if selectAST.Expression.Expressions[0].Expression.And[0].Condition[0].Operand.Operand.Left.Left.Primary.JPathExpr.String() == selectAST.From.As+".*" {
selectAST.Expression.All = true
}
}
stmt.selectAST = &selectAST
// Check the parsed limit value
stmt.limitValue, err = parseLimit(selectAST.Limit)
if err != nil {
err = errQueryAnalysisFailure(err)
return
}
// Analyze where clause
if selectAST.Where != nil {
whereQProp := selectAST.Where.analyze(&selectAST)
if whereQProp.err != nil {
err = errQueryAnalysisFailure(fmt.Errorf("Where clause error: %w", whereQProp.err))
return
}
if whereQProp.isAggregation {
err = errQueryAnalysisFailure(errors.New("WHERE clause cannot have an aggregation"))
return
}
}
// Validate table name
err = validateTableName(selectAST.From)
if err != nil {
return
}
// Analyze main select expression
stmt.selectQProp = selectAST.Expression.analyze(&selectAST)
err = stmt.selectQProp.err
if err != nil {
err = errQueryAnalysisFailure(err)
}
// Set table alias
stmt.tableAlias = selectAST.From.As
return
}
func validateTableName(from *TableExpression) error {
if strings.ToLower(from.Table.BaseKey.String()) != baseTableName {
return errBadTableName(errors.New("table name must be `s3object`"))
}
if len(from.Table.PathExpr) > 0 {
if !from.Table.PathExpr[0].ArrayWildcard {
return errBadTableName(errors.New("keypath table name is invalid - please check the service documentation"))
}
}
return nil
}
func parseLimit(v *LitValue) (int64, error) {
switch {
case v == nil:
return -1, nil
case v.Int == nil:
return -1, errBadLimitSpecified
default:
r := int64(*v.Int)
if r < 0 {
return -1, errBadLimitSpecified
}
return r, nil
}
}
// EvalFrom evaluates the From clause on the input record. It only
// applies to JSON input data format (currently).
func (e *SelectStatement) EvalFrom(format string, input Record) ([]*Record, error) {
if !e.selectAST.From.HasKeypath() {
return []*Record{&input}, nil
}
_, rawVal := input.Raw()
if format != "json" {
return nil, errDataSource(errors.New("path not supported"))
}
switch rec := rawVal.(type) {
case jstream.KVS:
txedRec, _, err := jsonpathEval(e.selectAST.From.Table.PathExpr[1:], rec)
if err != nil {
return nil, err
}
var kvs jstream.KVS
switch v := txedRec.(type) {
case jstream.KVS:
kvs = v
case []interface{}:
recs := make([]*Record, len(v))
for i, val := range v {
tmpRec := input.Clone(nil)
if err = tmpRec.Replace(val); err != nil {
return nil, err
}
recs[i] = &tmpRec
}
return recs, nil
default:
kvs = jstream.KVS{jstream.KV{Key: "_1", Value: v}}
}
if err = input.Replace(kvs); err != nil {
return nil, err
}
return []*Record{&input}, nil
case simdjson.Object:
txedRec, _, err := jsonpathEval(e.selectAST.From.Table.PathExpr[1:], rec)
if err != nil {
return nil, err
}
switch v := txedRec.(type) {
case simdjson.Object:
err := input.Replace(v)
if err != nil {
return nil, err
}
case []interface{}:
recs := make([]*Record, len(v))
for i, val := range v {
tmpRec := input.Clone(nil)
if err = tmpRec.Replace(val); err != nil {
return nil, err
}
recs[i] = &tmpRec
}
return recs, nil
default:
input.Reset()
input, err = input.Set("_1", &Value{value: v})
if err != nil {
return nil, err
}
}
return []*Record{&input}, nil
}
return nil, errDataSource(errors.New("unexpected non JSON input"))
}
// IsAggregated returns if the statement involves SQL aggregation
func (e *SelectStatement) IsAggregated() bool {
return e.selectQProp.isAggregation
}
// AggregateResult - returns the aggregated result after all input
// records have been processed. Applies only to aggregation queries.
func (e *SelectStatement) AggregateResult(output Record) error {
for i, expr := range e.selectAST.Expression.Expressions {
v, err := expr.evalNode(nil, e.tableAlias)
if err != nil {
return err
}
if expr.As != "" {
output, err = output.Set(expr.As, v)
} else {
output, err = output.Set(fmt.Sprintf("_%d", i+1), v)
}
if err != nil {
return err
}
}
return nil
}
func (e *SelectStatement) isPassingWhereClause(input Record) (bool, error) {
if e.selectAST.Where == nil {
return true, nil
}
value, err := e.selectAST.Where.evalNode(input, e.tableAlias)
if err != nil {
return false, err
}
b, ok := value.ToBool()
if !ok {
err = fmt.Errorf("WHERE expression did not return bool")
return false, err
}
return b, nil
}
// AggregateRow - aggregates the input record. Applies only to
// aggregation queries.
func (e *SelectStatement) AggregateRow(input Record) error {
ok, err := e.isPassingWhereClause(input)
if err != nil {
return err
}
if !ok {
return nil
}
for _, expr := range e.selectAST.Expression.Expressions {
err := expr.aggregateRow(input, e.tableAlias)
if err != nil {
return err
}
}
return nil
}
// Eval - evaluates the Select statement for the given record. It
// applies only to non-aggregation queries.
// The function returns whether the statement passed the WHERE clause and should be outputted.
func (e *SelectStatement) Eval(input, output Record) (Record, error) {
ok, err := e.isPassingWhereClause(input)
if err != nil || !ok {
// Either error or row did not pass where clause
return nil, err
}
if e.selectAST.Expression.All {
// Return the input record for `SELECT * FROM
// .. WHERE ..`
// Update count of records output.
if e.limitValue > -1 {
e.outputCount++
}
return input.Clone(output), nil
}
for i, expr := range e.selectAST.Expression.Expressions {
v, err := expr.evalNode(input, e.tableAlias)
if err != nil {
return nil, err
}
// Pick output column names
if expr.As != "" {
output, err = output.Set(expr.As, v)
} else if comp, ok := getLastKeypathComponent(expr.Expression); ok {
output, err = output.Set(comp, v)
} else {
output, err = output.Set(fmt.Sprintf("_%d", i+1), v)
}
if err != nil {
return nil, err
}
}
// Update count of records output.
if e.limitValue > -1 {
e.outputCount++
}
return output, nil
}
// LimitReached - returns true if the number of records output has
// reached the value of the `LIMIT` clause.
func (e *SelectStatement) LimitReached() bool {
if e.limitValue == -1 {
return false
}
return e.outputCount >= e.limitValue
}
internal/s3select/sql/stringfuncs.go
+200
View File
@@ -0,0 +1,200 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"errors"
"strings"
)
var (
errMalformedEscapeSequence = errors.New("Malformed escape sequence in LIKE clause")
errInvalidTrimArg = errors.New("Trim argument is invalid - this should not happen")
errInvalidSubstringIndexLen = errors.New("Substring start index or length falls outside the string")
)
const (
percent rune = '%'
underscore rune = '_'
runeZero rune = 0
)
func evalSQLLike(text, pattern string, escape rune) (match bool, err error) {
s := []rune{}
prev := runeZero
hasLeadingPercent := false
patLen := len([]rune(pattern))
for i, r := range pattern {
if i > 0 && prev == escape {
switch r {
case percent, escape, underscore:
s = append(s, r)
prev = r
if r == escape {
prev = runeZero
}
default:
return false, errMalformedEscapeSequence
}
continue
}
prev = r
var ok bool
switch r {
case percent:
if len(s) == 0 {
hasLeadingPercent = true
continue
}
text, ok = matcher(text, string(s), hasLeadingPercent)
if !ok {
return false, nil
}
hasLeadingPercent = true
s = []rune{}
if i == patLen-1 {
// Last pattern character is a %, so
// we are done.
return true, nil
}
case underscore:
if len(s) == 0 {
text, ok = dropRune(text)
if !ok {
return false, nil
}
continue
}
text, ok = matcher(text, string(s), hasLeadingPercent)
if !ok {
return false, nil
}
hasLeadingPercent = false
text, ok = dropRune(text)
if !ok {
return false, nil
}
s = []rune{}
case escape:
if i == patLen-1 {
return false, errMalformedEscapeSequence
}
// Otherwise do nothing.
default:
s = append(s, r)
}
}
if hasLeadingPercent {
return strings.HasSuffix(text, string(s)), nil
}
return string(s) == text, nil
}
// matcher - Finds `pat` in `text`, and returns the part remainder of
// `text`, after the match. If leadingPercent is false, `pat` must be
// the prefix of `text`, otherwise it must be a substring.
func matcher(text, pat string, leadingPercent bool) (res string, match bool) {
if !leadingPercent {
res = strings.TrimPrefix(text, pat)
if len(text) == len(res) {
return "", false
}
} else {
parts := strings.SplitN(text, pat, 2)
if len(parts) == 1 {
return "", false
}
res = parts[1]
}
return res, true
}
func dropRune(text string) (res string, ok bool) {
r := []rune(text)
if len(r) == 0 {
return "", false
}
return string(r[1:]), true
}
func evalSQLSubstring(s string, startIdx, length int) (res string, err error) {
rs := []rune(s)
// According to s3 document, if startIdx < 1, it is set to 1.
if startIdx < 1 {
startIdx = 1
}
if startIdx > len(rs) {
startIdx = len(rs) + 1
}
// StartIdx is 1-based in the input
startIdx--
endIdx := len(rs)
if length != -1 {
if length < 0 {
return "", errInvalidSubstringIndexLen
}
if length > (endIdx - startIdx) {
length = endIdx - startIdx
}
endIdx = startIdx + length
}
return string(rs[startIdx:endIdx]), nil
}
const (
trimLeading = "LEADING"
trimTrailing = "TRAILING"
trimBoth = "BOTH"
)
func evalSQLTrim(where *string, trimChars, text string) (result string, err error) {
cutSet := " "
if trimChars != "" {
cutSet = trimChars
}
trimFunc := strings.Trim
switch {
case where == nil:
case *where == trimBoth:
case *where == trimLeading:
trimFunc = strings.TrimLeft
case *where == trimTrailing:
trimFunc = strings.TrimRight
default:
return "", errInvalidTrimArg
}
return trimFunc(text, cutSet), nil
}
internal/s3select/sql/stringfuncs_contrib_test.go
+43
View File
@@ -0,0 +1,43 @@
/*
* MinIO Object Storage (c) 2021 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sql
import "testing"
func TestEvalSQLSubstring(t *testing.T) {
evalCases := []struct {
s string
startIdx int
length int
resExpected string
errExpected error
}{
{"abcd", 1, 1, "a", nil},
{"abcd", -1, 1, "a", nil},
{"abcd", 999, 999, "", nil},
{"", 999, 999, "", nil},
{"测试abc", 1, 1, "测", nil},
{"测试abc", 5, 5, "c", nil},
}
for i, tc := range evalCases {
res, err := evalSQLSubstring(tc.s, tc.startIdx, tc.length)
if res != tc.resExpected || err != tc.errExpected {
t.Errorf("Eval Case %d failed: %v %v", i, res, err)
}
}
}
internal/s3select/sql/stringfuncs_test.go
+108
View File
@@ -0,0 +1,108 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"testing"
)
func TestEvalSQLLike(t *testing.T) {
dropCases := []struct {
input, resultExpected string
matchExpected bool
}{
{"", "", false},
{"a", "", true},
{"ab", "b", true},
{"தமிழ்", "மிழ்", true},
}
for i, tc := range dropCases {
res, ok := dropRune(tc.input)
if res != tc.resultExpected || ok != tc.matchExpected {
t.Errorf("DropRune Case %d failed", i)
}
}
matcherCases := []struct {
iText, iPat string
iHasLeadingPercent bool
resultExpected string
matchExpected bool
}{
{"abcd", "bcd", false, "", false},
{"abcd", "bcd", true, "", true},
{"abcd", "abcd", false, "", true},
{"abcd", "abcd", true, "", true},
{"abcd", "ab", false, "cd", true},
{"abcd", "ab", true, "cd", true},
{"abcd", "bc", false, "", false},
{"abcd", "bc", true, "d", true},
}
for i, tc := range matcherCases {
res, ok := matcher(tc.iText, tc.iPat, tc.iHasLeadingPercent)
if res != tc.resultExpected || ok != tc.matchExpected {
t.Errorf("Matcher Case %d failed", i)
}
}
evalCases := []struct {
iText, iPat string
iEsc rune
matchExpected bool
errExpected error
}{
{"abcd", "abc", runeZero, false, nil},
{"abcd", "abcd", runeZero, true, nil},
{"abcd", "abc_", runeZero, true, nil},
{"abcd", "_bdd", runeZero, false, nil},
{"abcd", "_b_d", runeZero, true, nil},
{"abcd", "____", runeZero, true, nil},
{"abcd", "____%", runeZero, true, nil},
{"abcd", "%____", runeZero, true, nil},
{"abcd", "%__", runeZero, true, nil},
{"abcd", "____", runeZero, true, nil},
{"", "_", runeZero, false, nil},
{"", "%", runeZero, true, nil},
{"abcd", "%%%%%", runeZero, true, nil},
{"abcd", "_____", runeZero, false, nil},
{"abcd", "%%%%%", runeZero, true, nil},
{"a%%d", `a\%\%d`, '\\', true, nil},
{"a%%d", `a\%d`, '\\', false, nil},
{`a%%\d`, `a\%\%\\d`, '\\', true, nil},
{`a%%\`, `a\%\%\\`, '\\', true, nil},
{`a%__%\`, `a\%\_\_\%\\`, '\\', true, nil},
{`a%__%\`, `a\%\_\_\%_`, '\\', true, nil},
{`a%__%\`, `a\%\_\__`, '\\', false, nil},
{`a%__%\`, `a\%\_\_%`, '\\', true, nil},
{`a%__%\`, `a?%?_?_?%\`, '?', true, nil},
}
for i, tc := range evalCases {
// fmt.Println("Case:", i)
res, err := evalSQLLike(tc.iText, tc.iPat, tc.iEsc)
if res != tc.matchExpected || err != tc.errExpected {
t.Errorf("Eval Case %d failed: %v %v", i, res, err)
}
}
}
internal/s3select/sql/timestampfuncs.go
+183
View File
@@ -0,0 +1,183 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"time"
)
const (
layoutYear = "2006T"
layoutMonth = "2006-01T"
layoutDay = "2006-01-02T"
layoutMinute = "2006-01-02T15:04Z07:00"
layoutSecond = "2006-01-02T15:04:05Z07:00"
layoutNanosecond = "2006-01-02T15:04:05.999999999Z07:00"
)
var (
tformats = []string{
layoutYear,
layoutMonth,
layoutDay,
layoutMinute,
layoutSecond,
layoutNanosecond,
}
)
func parseSQLTimestamp(s string) (t time.Time, err error) {
for _, f := range tformats {
t, err = time.Parse(f, s)
if err == nil {
break
}
}
return
}
// FormatSQLTimestamp - returns the a string representation of the
// timestamp as used in S3 Select
func FormatSQLTimestamp(t time.Time) string {
_, zoneOffset := t.Zone()
hasZone := zoneOffset != 0
hasFracSecond := t.Nanosecond() != 0
hasSecond := t.Second() != 0
hasTime := t.Hour() != 0 || t.Minute() != 0
hasDay := t.Day() != 1
hasMonth := t.Month() != 1
switch {
case hasFracSecond:
return t.Format(layoutNanosecond)
case hasSecond:
return t.Format(layoutSecond)
case hasTime || hasZone:
return t.Format(layoutMinute)
case hasDay:
return t.Format(layoutDay)
case hasMonth:
return t.Format(layoutMonth)
default:
return t.Format(layoutYear)
}
}
const (
timePartYear = "YEAR"
timePartMonth = "MONTH"
timePartDay = "DAY"
timePartHour = "HOUR"
timePartMinute = "MINUTE"
timePartSecond = "SECOND"
timePartTimezoneHour = "TIMEZONE_HOUR"
timePartTimezoneMinute = "TIMEZONE_MINUTE"
)
func extract(what string, t time.Time) (v *Value, err error) {
switch what {
case timePartYear:
return FromInt(int64(t.Year())), nil
case timePartMonth:
return FromInt(int64(t.Month())), nil
case timePartDay:
return FromInt(int64(t.Day())), nil
case timePartHour:
return FromInt(int64(t.Hour())), nil
case timePartMinute:
return FromInt(int64(t.Minute())), nil
case timePartSecond:
return FromInt(int64(t.Second())), nil
case timePartTimezoneHour:
_, zoneOffset := t.Zone()
return FromInt(int64(zoneOffset / 3600)), nil
case timePartTimezoneMinute:
_, zoneOffset := t.Zone()
return FromInt(int64((zoneOffset % 3600) / 60)), nil
default:
// This does not happen
return nil, errNotImplemented
}
}
func dateAdd(timePart string, qty float64, t time.Time) (*Value, error) {
var duration time.Duration
switch timePart {
case timePartYear:
return FromTimestamp(t.AddDate(int(qty), 0, 0)), nil
case timePartMonth:
return FromTimestamp(t.AddDate(0, int(qty), 0)), nil
case timePartDay:
return FromTimestamp(t.AddDate(0, 0, int(qty))), nil
case timePartHour:
duration = time.Duration(qty) * time.Hour
case timePartMinute:
duration = time.Duration(qty) * time.Minute
case timePartSecond:
duration = time.Duration(qty) * time.Second
default:
return nil, errNotImplemented
}
return FromTimestamp(t.Add(duration)), nil
}
// dateDiff computes the difference between two times in terms of the
// `timePart` which can be years, months, days, hours, minutes or
// seconds. For difference in years, months or days, the time part,
// including timezone is ignored.
func dateDiff(timePart string, ts1, ts2 time.Time) (*Value, error) {
if ts2.Before(ts1) {
v, err := dateDiff(timePart, ts2, ts1)
if err == nil {
v.negate()
}
return v, err
}
duration := ts2.Sub(ts1)
y1, m1, d1 := ts1.Date()
y2, m2, d2 := ts2.Date()
switch timePart {
case timePartYear:
dy := int64(y2 - y1)
if m2 > m1 || (m2 == m1 && d2 >= d1) {
return FromInt(dy), nil
}
return FromInt(dy - 1), nil
case timePartMonth:
m1 += time.Month(12 * y1)
m2 += time.Month(12 * y2)
return FromInt(int64(m2 - m1)), nil
case timePartDay:
return FromInt(int64(duration / (24 * time.Hour))), nil
case timePartHour:
hours := duration / time.Hour
return FromInt(int64(hours)), nil
case timePartMinute:
minutes := duration / time.Minute
return FromInt(int64(minutes)), nil
case timePartSecond:
seconds := duration / time.Second
return FromInt(int64(seconds)), nil
default:
}
return nil, errNotImplemented
}
internal/s3select/sql/timestampfuncs_test.go
+61
View File
@@ -0,0 +1,61 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"testing"
"time"
)
func TestParseAndDisplaySQLTimestamp(t *testing.T) {
beijing := time.FixedZone("", int((8 * time.Hour).Seconds()))
fakeLosAngeles := time.FixedZone("", -int((8 * time.Hour).Seconds()))
cases := []struct {
s string
t time.Time
}{
{"2010T", time.Date(2010, 1, 1, 0, 0, 0, 0, time.UTC)},
{"2010-02T", time.Date(2010, 2, 1, 0, 0, 0, 0, time.UTC)},
{"2010-02-03T", time.Date(2010, 2, 3, 0, 0, 0, 0, time.UTC)},
{"2010-02-03T04:11Z", time.Date(2010, 2, 3, 4, 11, 0, 0, time.UTC)},
{"2010-02-03T04:11:30Z", time.Date(2010, 2, 3, 4, 11, 30, 0, time.UTC)},
{"2010-02-03T04:11:30.23Z", time.Date(2010, 2, 3, 4, 11, 30, 230000000, time.UTC)},
{"2010-02-03T04:11+08:00", time.Date(2010, 2, 3, 4, 11, 0, 0, beijing)},
{"2010-02-03T04:11:30+08:00", time.Date(2010, 2, 3, 4, 11, 30, 0, beijing)},
{"2010-02-03T04:11:30.23+08:00", time.Date(2010, 2, 3, 4, 11, 30, 230000000, beijing)},
{"2010-02-03T04:11:30-08:00", time.Date(2010, 2, 3, 4, 11, 30, 0, fakeLosAngeles)},
{"2010-02-03T04:11:30.23-08:00", time.Date(2010, 2, 3, 4, 11, 30, 230000000, fakeLosAngeles)},
}
for i, tc := range cases {
tval, err := parseSQLTimestamp(tc.s)
if err != nil {
t.Errorf("Case %d: Unexpected error: %v", i+1, err)
continue
}
if !tval.Equal(tc.t) {
t.Errorf("Case %d: Expected %v got %v", i+1, tc.t, tval)
continue
}
tstr := FormatSQLTimestamp(tc.t)
if tstr != tc.s {
t.Errorf("Case %d: Expected %s got %s", i+1, tc.s, tstr)
continue
}
}
}
internal/s3select/sql/utils.go
+134
View File
@@ -0,0 +1,134 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"fmt"
"strings"
)
// String functions
// String - returns the JSONPath representation
func (e *JSONPath) String() string {
if len(e.pathString) == 0 {
parts := make([]string, len(e.PathExpr)+1)
parts[0] = e.BaseKey.String()
for i, pe := range e.PathExpr {
parts[i+1] = pe.String()
}
e.pathString = strings.Join(parts, "")
}
return e.pathString
}
// StripTableAlias removes a table alias from the path. The result is also
// cached for repeated lookups during SQL query evaluation.
func (e *JSONPath) StripTableAlias(tableAlias string) []*JSONPathElement {
if e.strippedTableAlias == tableAlias {
return e.strippedPathExpr
}
hasTableAlias := e.BaseKey.String() == tableAlias || strings.ToLower(e.BaseKey.String()) == baseTableName
var pathExpr []*JSONPathElement
if hasTableAlias {
pathExpr = e.PathExpr
} else {
pathExpr = make([]*JSONPathElement, len(e.PathExpr)+1)
pathExpr[0] = &JSONPathElement{Key: &ObjectKey{ID: e.BaseKey}}
copy(pathExpr[1:], e.PathExpr)
}
e.strippedTableAlias = tableAlias
e.strippedPathExpr = pathExpr
return e.strippedPathExpr
}
func (e *JSONPathElement) String() string {
switch {
case e.Key != nil:
return e.Key.String()
case e.Index != nil:
return fmt.Sprintf("[%d]", *e.Index)
case e.ObjectWildcard:
return ".*"
case e.ArrayWildcard:
return "[*]"
}
return ""
}
// String removes double quotes in quoted identifiers
func (i *Identifier) String() string {
if i.Unquoted != nil {
return *i.Unquoted
}
return string(*i.Quoted)
}
func (o *ObjectKey) String() string {
if o.Lit != nil {
return fmt.Sprintf("['%s']", string(*o.Lit))
}
return fmt.Sprintf(".%s", o.ID.String())
}
func (o *ObjectKey) keyString() string {
if o.Lit != nil {
return string(*o.Lit)
}
return o.ID.String()
}
// getLastKeypathComponent checks if the given expression is a path
// expression, and if so extracts the last dot separated component of
// the path. Otherwise it returns false.
func getLastKeypathComponent(e *Expression) (string, bool) {
if len(e.And) > 1 ||
len(e.And[0].Condition) > 1 ||
e.And[0].Condition[0].Not != nil ||
e.And[0].Condition[0].Operand.ConditionRHS != nil {
return "", false
}
operand := e.And[0].Condition[0].Operand.Operand
if operand.Right != nil ||
operand.Left.Right != nil ||
operand.Left.Left.Negated != nil ||
operand.Left.Left.Primary.JPathExpr == nil {
return "", false
}
// Check if path expression ends in a key
jpath := operand.Left.Left.Primary.JPathExpr
n := len(jpath.PathExpr)
if n > 0 && jpath.PathExpr[n-1].Key == nil {
return "", false
}
ps := jpath.String()
if idx := strings.LastIndex(ps, "."); idx >= 0 {
// Get last part of path string.
ps = ps[idx+1:]
}
return ps, true
}
// HasKeypath returns if the from clause has a key path -
// e.g. S3object[*].id
func (from *TableExpression) HasKeypath() bool {
return len(from.Table.PathExpr) > 1
}
internal/s3select/sql/value.go
+914
View File
@@ -0,0 +1,914 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"encoding/json"
"errors"
"fmt"
"math"
"reflect"
"strconv"
"strings"
"time"
"unicode/utf8"
)
var (
errArithMismatchedTypes = errors.New("cannot perform arithmetic on mismatched types")
errArithInvalidOperator = errors.New("invalid arithmetic operator")
errArithDivideByZero = errors.New("cannot divide by 0")
errCmpMismatchedTypes = errors.New("cannot compare values of different types")
errCmpInvalidBoolOperator = errors.New("invalid comparison operator for boolean arguments")
)
// Value represents a value of restricted type reduced from an
// expression represented by an ASTNode. Only one of the fields is
// non-nil.
//
// In cases where we are fetching data from a data source (like csv),
// the type may not be determined yet. In these cases, a byte-slice is
// used.
type Value struct {
value interface{}
}
// MarshalJSON provides json marshaling of values.
func (v Value) MarshalJSON() ([]byte, error) {
if b, ok := v.ToBytes(); ok {
return b, nil
}
return json.Marshal(v.value)
}
// GetTypeString returns a string representation for vType
func (v Value) GetTypeString() string {
switch v.value.(type) {
case nil:
return "NULL"
case bool:
return "BOOL"
case string:
return "STRING"
case int64:
return "INT"
case float64:
return "FLOAT"
case time.Time:
return "TIMESTAMP"
case []byte:
return "BYTES"
case []Value:
return "ARRAY"
}
return "--"
}
// Repr returns a string representation of value.
func (v Value) Repr() string {
switch x := v.value.(type) {
case nil:
return ":NULL"
case bool, int64, float64:
return fmt.Sprintf("%v:%s", v.value, v.GetTypeString())
case time.Time:
return fmt.Sprintf("%s:TIMESTAMP", x)
case string:
return fmt.Sprintf("\"%s\":%s", x, v.GetTypeString())
case []byte:
return fmt.Sprintf("\"%s\":BYTES", string(x))
case []Value:
var s strings.Builder
s.WriteByte('[')
for i, v := range x {
s.WriteString(v.Repr())
if i < len(x)-1 {
s.WriteByte(',')
}
}
s.WriteString("]:ARRAY")
return s.String()
default:
return fmt.Sprintf("%v:INVALID", v.value)
}
}
// FromFloat creates a Value from a number
func FromFloat(f float64) *Value {
return &Value{value: f}
}
// FromInt creates a Value from an int
func FromInt(f int64) *Value {
return &Value{value: f}
}
// FromString creates a Value from a string
func FromString(str string) *Value {
return &Value{value: str}
}
// FromBool creates a Value from a bool
func FromBool(b bool) *Value {
return &Value{value: b}
}
// FromTimestamp creates a Value from a timestamp
func FromTimestamp(t time.Time) *Value {
return &Value{value: t}
}
// FromNull creates a Value with Null value
func FromNull() *Value {
return &Value{value: nil}
}
// FromBytes creates a Value from a []byte
func FromBytes(b []byte) *Value {
return &Value{value: b}
}
// FromArray creates a Value from an array of values.
func FromArray(a []Value) *Value {
return &Value{value: a}
}
// ToFloat works for int and float values
func (v Value) ToFloat() (val float64, ok bool) {
switch x := v.value.(type) {
case float64:
return x, true
case int64:
return float64(x), true
}
return 0, false
}
// ToInt returns the value if int.
func (v Value) ToInt() (val int64, ok bool) {
val, ok = v.value.(int64)
return
}
// ToString returns the value if string.
func (v Value) ToString() (val string, ok bool) {
val, ok = v.value.(string)
return
}
// Equals returns whether the values strictly match.
// Both type and value must match.
func (v Value) Equals(b Value) (ok bool) {
if !v.SameTypeAs(b) {
return false
}
return reflect.DeepEqual(v.value, b.value)
}
// SameTypeAs return whether the two types are strictly the same.
func (v Value) SameTypeAs(b Value) (ok bool) {
switch v.value.(type) {
case bool:
_, ok = b.value.(bool)
case string:
_, ok = b.value.(string)
case int64:
_, ok = b.value.(int64)
case float64:
_, ok = b.value.(float64)
case time.Time:
_, ok = b.value.(time.Time)
case []byte:
_, ok = b.value.([]byte)
case []Value:
_, ok = b.value.([]Value)
default:
ok = reflect.TypeOf(v.value) == reflect.TypeOf(b.value)
}
return ok
}
// ToBool returns the bool value; second return value refers to if the bool
// conversion succeeded.
func (v Value) ToBool() (val bool, ok bool) {
val, ok = v.value.(bool)
return
}
// ToTimestamp returns the timestamp value if present.
func (v Value) ToTimestamp() (t time.Time, ok bool) {
t, ok = v.value.(time.Time)
return
}
// ToBytes returns the value if byte-slice.
func (v Value) ToBytes() (val []byte, ok bool) {
val, ok = v.value.([]byte)
return
}
// ToArray returns the value if it is a slice of values.
func (v Value) ToArray() (val []Value, ok bool) {
val, ok = v.value.([]Value)
return
}
// IsNull - checks if value is missing.
func (v Value) IsNull() bool {
switch v.value.(type) {
case nil:
return true
}
return false
}
// IsArray returns whether the value is an array.
func (v Value) IsArray() (ok bool) {
_, ok = v.value.([]Value)
return ok
}
func (v Value) isNumeric() bool {
switch v.value.(type) {
case int64, float64:
return true
}
return false
}
// setters used internally to mutate values
func (v *Value) setInt(i int64) {
v.value = i
}
func (v *Value) setFloat(f float64) {
v.value = f
}
func (v *Value) setString(s string) {
v.value = s
}
func (v *Value) setBool(b bool) {
v.value = b
}
func (v *Value) setTimestamp(t time.Time) {
v.value = t
}
func (v Value) String() string {
return fmt.Sprintf("%#v", v.value)
}
// CSVString - convert to string for CSV serialization
func (v Value) CSVString() string {
switch x := v.value.(type) {
case nil:
return ""
case bool:
if x {
return "true"
}
return "false"
case string:
return x
case int64:
return strconv.FormatInt(x, 10)
case float64:
return strconv.FormatFloat(x, 'g', -1, 64)
case time.Time:
return FormatSQLTimestamp(x)
case []byte:
return string(x)
case []Value:
b, _ := json.Marshal(x)
return string(b)
default:
return "CSV serialization not implemented for this type"
}
}
// negate negates a numeric value
func (v *Value) negate() {
switch x := v.value.(type) {
case float64:
v.value = -x
case int64:
v.value = -x
}
}
// Value comparison functions: we do not expose them outside the
// module. Logical operators "<", ">", ">=", "<=" work on strings and
// numbers. Equality operators "=", "!=" work on strings,
// numbers and booleans.
// Supported comparison operators
const (
opLt = "<"
opLte = "<="
opGt = ">"
opGte = ">="
opEq = "="
opIneq = "!="
)
// InferBytesType will attempt to infer the data type of bytes.
// Will fail if value type is not bytes or it would result in invalid utf8.
// ORDER: int, float, bool, JSON (object or array), timestamp, string
// If the content is valid JSON, the type will still be bytes.
func (v *Value) InferBytesType() (err error) {
b, ok := v.ToBytes()
if !ok {
return fmt.Errorf("InferByteType: Input is not bytes, but %v", v.GetTypeString())
}
// Check for numeric inference
if x, ok := v.bytesToInt(); ok {
v.setInt(x)
return nil
}
if x, ok := v.bytesToFloat(); ok {
v.setFloat(x)
return nil
}
if x, ok := v.bytesToBool(); ok {
v.setBool(x)
return nil
}
asString := strings.TrimSpace(v.bytesToString())
if len(b) > 0 &&
(strings.HasPrefix(asString, "{") || strings.HasPrefix(asString, "[")) {
return nil
}
if t, err := parseSQLTimestamp(asString); err == nil {
v.setTimestamp(t)
return nil
}
if !utf8.Valid(b) {
return errors.New("value is not valid utf-8")
}
// Fallback to string
v.setString(asString)
return
}
// When numeric types are compared, type promotions could happen. If
// values do not have types (e.g. when reading from CSV), for
// comparison operations, automatic type conversion happens by trying
// to check if the value is a number (first an integer, then a float),
// and falling back to string.
func (v *Value) compareOp(op string, a *Value) (res bool, err error) {
if !isValidComparisonOperator(op) {
return false, errArithInvalidOperator
}
// Check if type conversion/inference is needed - it is needed
// if the Value is a byte-slice.
err = inferTypesForCmp(v, a)
if err != nil {
return false, err
}
// Check if either is nil
if v.IsNull() || a.IsNull() {
// If one is, both must be.
return boolCompare(op, v.IsNull(), a.IsNull())
}
// Check array values
aArr, aOK := a.ToArray()
vArr, vOK := v.ToArray()
if aOK && vOK {
return arrayCompare(op, aArr, vArr)
}
isNumeric := v.isNumeric() && a.isNumeric()
if isNumeric {
intV, ok1i := v.ToInt()
intA, ok2i := a.ToInt()
if ok1i && ok2i {
return intCompare(op, intV, intA), nil
}
// If both values are numeric, then at least one is
// float since we got here, so we convert.
flV, _ := v.ToFloat()
flA, _ := a.ToFloat()
return floatCompare(op, flV, flA), nil
}
strV, ok1s := v.ToString()
strA, ok2s := a.ToString()
if ok1s && ok2s {
return stringCompare(op, strV, strA), nil
}
boolV, ok1b := v.ToBool()
boolA, ok2b := a.ToBool()
if ok1b && ok2b {
return boolCompare(op, boolV, boolA)
}
timestampV, ok1t := v.ToTimestamp()
timestampA, ok2t := a.ToTimestamp()
if ok1t && ok2t {
return timestampCompare(op, timestampV, timestampA), nil
}
// Types cannot be compared, they do not match.
switch op {
case opEq:
return false, nil
case opIneq:
return true, nil
}
return false, errCmpInvalidBoolOperator
}
func inferTypesForCmp(a *Value, b *Value) error {
_, okA := a.ToBytes()
_, okB := b.ToBytes()
switch {
case !okA && !okB:
// Both Values already have types
return nil
case okA && okB:
// Both Values are untyped so try the types in order:
// int, float, bool, string
// Check for numeric inference
iA, okAi := a.bytesToInt()
iB, okBi := b.bytesToInt()
if okAi && okBi {
a.setInt(iA)
b.setInt(iB)
return nil
}
fA, okAf := a.bytesToFloat()
fB, okBf := b.bytesToFloat()
if okAf && okBf {
a.setFloat(fA)
b.setFloat(fB)
return nil
}
// Check if they int and float combination.
if okAi && okBf {
a.setInt(iA)
b.setFloat(fA)
return nil
}
if okBi && okAf {
a.setFloat(fA)
b.setInt(iB)
return nil
}
// Not numeric types at this point.
// Check for bool inference
bA, okAb := a.bytesToBool()
bB, okBb := b.bytesToBool()
if okAb && okBb {
a.setBool(bA)
b.setBool(bB)
return nil
}
// Fallback to string
sA := a.bytesToString()
sB := b.bytesToString()
a.setString(sA)
b.setString(sB)
return nil
case okA && !okB:
// Here a has `a` is untyped, but `b` has a fixed
// type.
switch b.value.(type) {
case string:
s := a.bytesToString()
a.setString(s)
case int64, float64:
if iA, ok := a.bytesToInt(); ok {
a.setInt(iA)
} else if fA, ok := a.bytesToFloat(); ok {
a.setFloat(fA)
} else {
return fmt.Errorf("Could not convert %s to a number", a.String())
}
case bool:
if bA, ok := a.bytesToBool(); ok {
a.setBool(bA)
} else {
return fmt.Errorf("Could not convert %s to a boolean", a.String())
}
default:
return errCmpMismatchedTypes
}
return nil
case !okA && okB:
// swap arguments to avoid repeating code
return inferTypesForCmp(b, a)
default:
// Does not happen
return nil
}
}
// Value arithmetic functions: we do not expose them outside the
// module. All arithmetic works only on numeric values with automatic
// promotion to the "larger" type that can represent the value. TODO:
// Add support for large number arithmetic.
// Supported arithmetic operators
const (
opPlus = "+"
opMinus = "-"
opDivide = "/"
opMultiply = "*"
opModulo = "%"
)
// For arithmetic operations, if both values are numeric then the
// operation shall succeed. If the types are unknown automatic type
// conversion to a number is attempted.
func (v *Value) arithOp(op string, a *Value) error {
err := inferTypeForArithOp(v)
if err != nil {
return err
}
err = inferTypeForArithOp(a)
if err != nil {
return err
}
if !v.isNumeric() || !a.isNumeric() {
return errInvalidDataType(errArithMismatchedTypes)
}
if !isValidArithOperator(op) {
return errInvalidDataType(errArithMismatchedTypes)
}
intV, ok1i := v.ToInt()
intA, ok2i := a.ToInt()
switch {
case ok1i && ok2i:
res, err := intArithOp(op, intV, intA)
v.setInt(res)
return err
default:
// Convert arguments to float
flV, _ := v.ToFloat()
flA, _ := a.ToFloat()
res, err := floatArithOp(op, flV, flA)
v.setFloat(res)
return err
}
}
func inferTypeForArithOp(a *Value) error {
if _, ok := a.ToBytes(); !ok {
return nil
}
if i, ok := a.bytesToInt(); ok {
a.setInt(i)
return nil
}
if f, ok := a.bytesToFloat(); ok {
a.setFloat(f)
return nil
}
err := fmt.Errorf("Could not convert %q to a number", string(a.value.([]byte)))
return errInvalidDataType(err)
}
// All the bytesTo* functions defined below assume the value is a byte-slice.
// Converts untyped value into int. The bool return implies success -
// it returns false only if there is a conversion failure.
func (v Value) bytesToInt() (int64, bool) {
bytes, _ := v.ToBytes()
i, err := strconv.ParseInt(strings.TrimSpace(string(bytes)), 10, 64)
return i, err == nil
}
// Converts untyped value into float. The bool return implies success
// - it returns false only if there is a conversion failure.
func (v Value) bytesToFloat() (float64, bool) {
bytes, _ := v.ToBytes()
i, err := strconv.ParseFloat(strings.TrimSpace(string(bytes)), 64)
return i, err == nil
}
// Converts untyped value into bool. The second bool return implies
// success - it returns false in case of a conversion failure.
func (v Value) bytesToBool() (val bool, ok bool) {
bytes, _ := v.ToBytes()
ok = true
switch strings.ToLower(strings.TrimSpace(string(bytes))) {
case "t", "true", "1":
val = true
case "f", "false", "0":
val = false
default:
ok = false
}
return val, ok
}
// bytesToString - never fails, but returns empty string if value is not bytes.
func (v Value) bytesToString() string {
bytes, _ := v.ToBytes()
return string(bytes)
}
// Calculates minimum or maximum of v and a and assigns the result to
// v - it works only on numeric arguments, where `v` is already
// assumed to be numeric. Attempts conversion to numeric type for `a`
// (first int, then float) only if the underlying values do not have a
// type.
func (v *Value) minmax(a *Value, isMax, isFirstRow bool) error {
err := inferTypeForArithOp(a)
if err != nil {
return err
}
if !a.isNumeric() {
return errArithMismatchedTypes
}
// In case of first row, set v to a.
if isFirstRow {
intA, okI := a.ToInt()
if okI {
v.setInt(intA)
return nil
}
floatA, _ := a.ToFloat()
v.setFloat(floatA)
return nil
}
intV, ok1i := v.ToInt()
intA, ok2i := a.ToInt()
if ok1i && ok2i {
result := intV
if !isMax {
if intA < result {
result = intA
}
} else {
if intA > result {
result = intA
}
}
v.setInt(result)
return nil
}
floatV, _ := v.ToFloat()
floatA, _ := a.ToFloat()
var result float64
if !isMax {
result = math.Min(floatV, floatA)
} else {
result = math.Max(floatV, floatA)
}
v.setFloat(result)
return nil
}
func inferTypeAsTimestamp(v *Value) error {
if s, ok := v.ToString(); ok {
t, err := parseSQLTimestamp(s)
if err != nil {
return err
}
v.setTimestamp(t)
} else if b, ok := v.ToBytes(); ok {
s := string(b)
t, err := parseSQLTimestamp(s)
if err != nil {
return err
}
v.setTimestamp(t)
}
return nil
}
// inferTypeAsString is used to convert untyped values to string - it
// is called when the caller requires a string context to proceed.
func inferTypeAsString(v *Value) {
b, ok := v.ToBytes()
if !ok {
return
}
v.setString(string(b))
}
func isValidComparisonOperator(op string) bool {
switch op {
case opLt:
case opLte:
case opGt:
case opGte:
case opEq:
case opIneq:
default:
return false
}
return true
}
func intCompare(op string, left, right int64) bool {
switch op {
case opLt:
return left < right
case opLte:
return left <= right
case opGt:
return left > right
case opGte:
return left >= right
case opEq:
return left == right
case opIneq:
return left != right
}
// This case does not happen
return false
}
func floatCompare(op string, left, right float64) bool {
diff := math.Abs(left - right)
switch op {
case opLt:
return left < right
case opLte:
return left <= right
case opGt:
return left > right
case opGte:
return left >= right
case opEq:
return diff < floatCmpTolerance
case opIneq:
return diff > floatCmpTolerance
}
// This case does not happen
return false
}
func stringCompare(op string, left, right string) bool {
switch op {
case opLt:
return left < right
case opLte:
return left <= right
case opGt:
return left > right
case opGte:
return left >= right
case opEq:
return left == right
case opIneq:
return left != right
}
// This case does not happen
return false
}
func boolCompare(op string, left, right bool) (bool, error) {
switch op {
case opEq:
return left == right, nil
case opIneq:
return left != right, nil
default:
return false, errCmpInvalidBoolOperator
}
}
func arrayCompare(op string, left, right []Value) (bool, error) {
switch op {
case opEq:
if len(left) != len(right) {
return false, nil
}
for i, l := range left {
eq, err := l.compareOp(op, &right[i])
if !eq || err != nil {
return eq, err
}
}
return true, nil
case opIneq:
for i, l := range left {
eq, err := l.compareOp(op, &right[i])
if eq || err != nil {
return eq, err
}
}
return false, nil
default:
return false, errCmpInvalidBoolOperator
}
}
func isValidArithOperator(op string) bool {
switch op {
case opPlus:
case opMinus:
case opDivide:
case opMultiply:
case opModulo:
default:
return false
}
return true
}
// Overflow errors are ignored.
func intArithOp(op string, left, right int64) (int64, error) {
switch op {
case opPlus:
return left + right, nil
case opMinus:
return left - right, nil
case opDivide:
if right == 0 {
return 0, errArithDivideByZero
}
return left / right, nil
case opMultiply:
return left * right, nil
case opModulo:
if right == 0 {
return 0, errArithDivideByZero
}
return left % right, nil
}
// This does not happen
return 0, nil
}
// Overflow errors are ignored.
func floatArithOp(op string, left, right float64) (float64, error) {
switch op {
case opPlus:
return left + right, nil
case opMinus:
return left - right, nil
case opDivide:
if right == 0 {
return 0, errArithDivideByZero
}
return left / right, nil
case opMultiply:
return left * right, nil
case opModulo:
if right == 0 {
return 0, errArithDivideByZero
}
return math.Mod(left, right), nil
}
// This does not happen
return 0, nil
}
internal/s3select/sql/value_contrib.go
+38
View File
@@ -0,0 +1,38 @@
/*
* MinIO Object Storage (c) 2021 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package sql
import "time"
func timestampCompare(op string, left, right time.Time) bool {
switch op {
case opLt:
return left.Before(right)
case opLte:
return left.Before(right) || left.Equal(right)
case opGt:
return left.After(right)
case opGte:
return left.After(right) || left.Equal(right)
case opEq:
return left.Equal(right)
case opIneq:
return !left.Equal(right)
}
// This case does not happen
return false
}
internal/s3select/sql/value_test.go
+687
View File
@@ -0,0 +1,687 @@
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package sql
import (
"fmt"
"math"
"strconv"
"testing"
"time"
)
// valueBuilders contains one constructor for each value type.
// Values should match if type is the same.
var valueBuilders = []func() *Value{
func() *Value {
return FromNull()
},
func() *Value {
return FromBool(true)
},
func() *Value {
return FromBytes([]byte("byte contents"))
},
func() *Value {
return FromFloat(math.Pi)
},
func() *Value {
return FromInt(0x1337)
},
func() *Value {
t, err := time.Parse(time.RFC3339, "2006-01-02T15:04:05Z")
if err != nil {
panic(err)
}
return FromTimestamp(t)
},
func() *Value {
return FromString("string contents")
},
}
// altValueBuilders contains one constructor for each value type.
// Values are zero values and should NOT match the values in valueBuilders, except Null type.
var altValueBuilders = []func() *Value{
func() *Value {
return FromNull()
},
func() *Value {
return FromBool(false)
},
func() *Value {
return FromBytes(nil)
},
func() *Value {
return FromFloat(0)
},
func() *Value {
return FromInt(0)
},
func() *Value {
return FromTimestamp(time.Time{})
},
func() *Value {
return FromString("")
},
}
func TestValue_SameTypeAs(t *testing.T) {
type fields struct {
a, b Value
}
type test struct {
name string
fields fields
wantOk bool
}
var tests []test
for i := range valueBuilders {
a := valueBuilders[i]()
for j := range valueBuilders {
b := valueBuilders[j]()
tests = append(tests, test{
name: fmt.Sprint(a.GetTypeString(), "==", b.GetTypeString()),
fields: fields{
a: *a, b: *b,
},
wantOk: i == j,
})
}
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotOk := tt.fields.a.SameTypeAs(tt.fields.b); gotOk != tt.wantOk {
t.Errorf("SameTypeAs() = %v, want %v", gotOk, tt.wantOk)
}
})
}
}
func TestValue_Equals(t *testing.T) {
type fields struct {
a, b Value
}
type test struct {
name string
fields fields
wantOk bool
}
var tests []test
for i := range valueBuilders {
a := valueBuilders[i]()
for j := range valueBuilders {
b := valueBuilders[j]()
tests = append(tests, test{
name: fmt.Sprint(a.GetTypeString(), "==", b.GetTypeString()),
fields: fields{
a: *a, b: *b,
},
wantOk: i == j,
})
}
}
for i := range valueBuilders {
a := valueBuilders[i]()
for j := range altValueBuilders {
b := altValueBuilders[j]()
tests = append(tests, test{
name: fmt.Sprint(a.GetTypeString(), "!=", b.GetTypeString()),
fields: fields{
a: *a, b: *b,
},
// Only Null == Null
wantOk: a.IsNull() && b.IsNull() && i == 0 && j == 0,
})
}
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if gotOk := tt.fields.a.Equals(tt.fields.b); gotOk != tt.wantOk {
t.Errorf("Equals() = %v, want %v", gotOk, tt.wantOk)
}
})
}
}
func TestValue_CSVString(t *testing.T) {
type test struct {
name string
want string
wantAlt string
}
tests := []test{
{
name: valueBuilders[0]().String(),
want: "",
wantAlt: "",
},
{
name: valueBuilders[1]().String(),
want: "true",
wantAlt: "false",
},
{
name: valueBuilders[2]().String(),
want: "byte contents",
wantAlt: "",
},
{
name: valueBuilders[3]().String(),
want: "3.141592653589793",
wantAlt: "0",
},
{
name: valueBuilders[4]().String(),
want: "4919",
wantAlt: "0",
},
{
name: valueBuilders[5]().String(),
want: "2006-01-02T15:04:05Z",
wantAlt: "0001T",
},
{
name: valueBuilders[6]().String(),
want: "string contents",
wantAlt: "",
},
}
for i, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
v := valueBuilders[i]()
vAlt := altValueBuilders[i]()
if got := v.CSVString(); got != tt.want {
t.Errorf("CSVString() = %v, want %v", got, tt.want)
}
if got := vAlt.CSVString(); got != tt.wantAlt {
t.Errorf("CSVString() = %v, want %v", got, tt.wantAlt)
}
})
}
}
func TestValue_bytesToInt(t *testing.T) {
type fields struct {
value interface{}
}
tests := []struct {
name string
fields fields
want int64
wantOK bool
}{
{
name: "zero",
fields: fields{
value: []byte("0"),
},
want: 0,
wantOK: true,
},
{
name: "minuszero",
fields: fields{
value: []byte("-0"),
},
want: 0,
wantOK: true,
},
{
name: "one",
fields: fields{
value: []byte("1"),
},
want: 1,
wantOK: true,
},
{
name: "minusone",
fields: fields{
value: []byte("-1"),
},
want: -1,
wantOK: true,
},
{
name: "plusone",
fields: fields{
value: []byte("+1"),
},
want: 1,
wantOK: true,
},
{
name: "max",
fields: fields{
value: []byte(strconv.FormatInt(math.MaxInt64, 10)),
},
want: math.MaxInt64,
wantOK: true,
},
{
name: "min",
fields: fields{
value: []byte(strconv.FormatInt(math.MinInt64, 10)),
},
want: math.MinInt64,
wantOK: true,
},
{
name: "max-overflow",
fields: fields{
value: []byte("9223372036854775808"),
},
// Seems to be what strconv.ParseInt returns
want: math.MaxInt64,
wantOK: false,
},
{
name: "min-underflow",
fields: fields{
value: []byte("-9223372036854775809"),
},
// Seems to be what strconv.ParseInt returns
want: math.MinInt64,
wantOK: false,
},
{
name: "zerospace",
fields: fields{
value: []byte(" 0"),
},
want: 0,
wantOK: true,
},
{
name: "onespace",
fields: fields{
value: []byte("1 "),
},
want: 1,
wantOK: true,
},
{
name: "minusonespace",
fields: fields{
value: []byte(" -1 "),
},
want: -1,
wantOK: true,
},
{
name: "plusonespace",
fields: fields{
value: []byte("\t+1\t"),
},
want: 1,
wantOK: true,
},
{
name: "scientific",
fields: fields{
value: []byte("3e5"),
},
want: 0,
wantOK: false,
},
{
// No support for prefixes
name: "hex",
fields: fields{
value: []byte("0xff"),
},
want: 0,
wantOK: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
v := &Value{
value: tt.fields.value,
}
got, got1 := v.bytesToInt()
if got != tt.want {
t.Errorf("bytesToInt() got = %v, want %v", got, tt.want)
}
if got1 != tt.wantOK {
t.Errorf("bytesToInt() got1 = %v, want %v", got1, tt.wantOK)
}
})
}
}
func TestValue_bytesToFloat(t *testing.T) {
type fields struct {
value interface{}
}
tests := []struct {
name string
fields fields
want float64
wantOK bool
}{
// Copied from TestValue_bytesToInt.
{
name: "zero",
fields: fields{
value: []byte("0"),
},
want: 0,
wantOK: true,
},
{
name: "minuszero",
fields: fields{
value: []byte("-0"),
},
want: 0,
wantOK: true,
},
{
name: "one",
fields: fields{
value: []byte("1"),
},
want: 1,
wantOK: true,
},
{
name: "minusone",
fields: fields{
value: []byte("-1"),
},
want: -1,
wantOK: true,
},
{
name: "plusone",
fields: fields{
value: []byte("+1"),
},
want: 1,
wantOK: true,
},
{
name: "maxint",
fields: fields{
value: []byte(strconv.FormatInt(math.MaxInt64, 10)),
},
want: math.MaxInt64,
wantOK: true,
},
{
name: "minint",
fields: fields{
value: []byte(strconv.FormatInt(math.MinInt64, 10)),
},
want: math.MinInt64,
wantOK: true,
},
{
name: "max-overflow-int",
fields: fields{
value: []byte("9223372036854775808"),
},
// Seems to be what strconv.ParseInt returns
want: math.MaxInt64,
wantOK: true,
},
{
name: "min-underflow-int",
fields: fields{
value: []byte("-9223372036854775809"),
},
// Seems to be what strconv.ParseInt returns
want: math.MinInt64,
wantOK: true,
},
{
name: "max",
fields: fields{
value: []byte(strconv.FormatFloat(math.MaxFloat64, 'g', -1, 64)),
},
want: math.MaxFloat64,
wantOK: true,
},
{
name: "min",
fields: fields{
value: []byte(strconv.FormatFloat(-math.MaxFloat64, 'g', -1, 64)),
},
want: -math.MaxFloat64,
wantOK: true,
},
{
name: "max-overflow",
fields: fields{
value: []byte("1.797693134862315708145274237317043567981e+309"),
},
// Seems to be what strconv.ParseInt returns
want: math.Inf(1),
wantOK: false,
},
{
name: "min-underflow",
fields: fields{
value: []byte("-1.797693134862315708145274237317043567981e+309"),
},
// Seems to be what strconv.ParseInt returns
want: math.Inf(-1),
wantOK: false,
},
{
name: "smallest-pos",
fields: fields{
value: []byte(strconv.FormatFloat(math.SmallestNonzeroFloat64, 'g', -1, 64)),
},
want: math.SmallestNonzeroFloat64,
wantOK: true,
},
{
name: "smallest-pos",
fields: fields{
value: []byte(strconv.FormatFloat(-math.SmallestNonzeroFloat64, 'g', -1, 64)),
},
want: -math.SmallestNonzeroFloat64,
wantOK: true,
},
{
name: "zerospace",
fields: fields{
value: []byte(" 0"),
},
want: 0,
wantOK: true,
},
{
name: "onespace",
fields: fields{
value: []byte("1 "),
},
want: 1,
wantOK: true,
},
{
name: "minusonespace",
fields: fields{
value: []byte(" -1 "),
},
want: -1,
wantOK: true,
},
{
name: "plusonespace",
fields: fields{
value: []byte("\t+1\t"),
},
want: 1,
wantOK: true,
},
{
name: "scientific",
fields: fields{
value: []byte("3e5"),
},
want: 300000,
wantOK: true,
},
{
// No support for prefixes
name: "hex",
fields: fields{
value: []byte("0xff"),
},
want: 0,
wantOK: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
v := Value{
value: tt.fields.value,
}
got, got1 := v.bytesToFloat()
diff := math.Abs(got - tt.want)
if diff > floatCmpTolerance {
t.Errorf("bytesToFloat() got = %v, want %v", got, tt.want)
}
if got1 != tt.wantOK {
t.Errorf("bytesToFloat() got1 = %v, want %v", got1, tt.wantOK)
}
})
}
}
func TestValue_bytesToBool(t *testing.T) {
type fields struct {
value interface{}
}
tests := []struct {
name string
fields fields
wantVal bool
wantOk bool
}{
{
name: "true",
fields: fields{
value: []byte("true"),
},
wantVal: true,
wantOk: true,
},
{
name: "false",
fields: fields{
value: []byte("false"),
},
wantVal: false,
wantOk: true,
},
{
name: "t",
fields: fields{
value: []byte("t"),
},
wantVal: true,
wantOk: true,
},
{
name: "f",
fields: fields{
value: []byte("f"),
},
wantVal: false,
wantOk: true,
},
{
name: "1",
fields: fields{
value: []byte("1"),
},
wantVal: true,
wantOk: true,
},
{
name: "0",
fields: fields{
value: []byte("0"),
},
wantVal: false,
wantOk: true,
},
{
name: "truespace",
fields: fields{
value: []byte(" true "),
},
wantVal: true,
wantOk: true,
},
{
name: "truetabs",
fields: fields{
value: []byte("\ttrue\t"),
},
wantVal: true,
wantOk: true,
},
{
name: "TRUE",
fields: fields{
value: []byte("TRUE"),
},
wantVal: true,
wantOk: true,
},
{
name: "FALSE",
fields: fields{
value: []byte("FALSE"),
},
wantVal: false,
wantOk: true,
},
{
name: "invalid",
fields: fields{
value: []byte("no"),
},
wantVal: false,
wantOk: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
v := Value{
value: tt.fields.value,
}
gotVal, gotOk := v.bytesToBool()
if gotVal != tt.wantVal {
t.Errorf("bytesToBool() gotVal = %v, want %v", gotVal, tt.wantVal)
}
if gotOk != tt.wantOk {
t.Errorf("bytesToBool() gotOk = %v, want %v", gotOk, tt.wantOk)
}
})
}
}
internal/s3select/testdata/lineitem_shipdate.parquet
Vendored
BIN
View File
Binary file not shown.
internal/s3select/testdata/testdata.parquet
Vendored
BIN
View File
Binary file not shown.
internal/s3select/unused-errors.go
+643
View File
@@ -0,0 +1,643 @@
// +build ignore
// Copyright (c) 2015-2021 MinIO, Inc.
//
// This file is part of MinIO Object Storage stack
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package s3select
///////////////////////////////////////////////////////////////////////
//
// Validation errors.
//
///////////////////////////////////////////////////////////////////////
func errExpressionTooLong(err error) *s3Error {
return &s3Error{
code: "ExpressionTooLong",
message: "The SQL expression is too long: The maximum byte-length for the SQL expression is 256 KB.",
statusCode: 400,
cause: err,
}
}
func errColumnTooLong(err error) *s3Error {
return &s3Error{
code: "ColumnTooLong",
message: "The length of a column in the result is greater than maxCharsPerColumn of 1 MB.",
statusCode: 400,
cause: err,
}
}
func errOverMaxColumn(err error) *s3Error {
return &s3Error{
code: "OverMaxColumn",
message: "The number of columns in the result is greater than the maximum allowable number of columns.",
statusCode: 400,
cause: err,
}
}
func errOverMaxRecordSize(err error) *s3Error {
return &s3Error{
code: "OverMaxRecordSize",
message: "The length of a record in the input or result is greater than maxCharsPerRecord of 1 MB.",
statusCode: 400,
cause: err,
}
}
func errInvalidColumnIndex(err error) *s3Error {
return &s3Error{
code: "InvalidColumnIndex",
message: "Column index in the SQL expression is invalid.",
statusCode: 400,
cause: err,
}
}
func errInvalidTextEncoding(err error) *s3Error {
return &s3Error{
code: "InvalidTextEncoding",
message: "Invalid encoding type. Only UTF-8 encoding is supported.",
statusCode: 400,
cause: err,
}
}
func errInvalidTableAlias(err error) *s3Error {
return &s3Error{
code: "InvalidTableAlias",
message: "The SQL expression contains an invalid table alias.",
statusCode: 400,
cause: err,
}
}
func errUnsupportedSyntax(err error) *s3Error {
return &s3Error{
code: "UnsupportedSyntax",
message: "Encountered invalid syntax.",
statusCode: 400,
cause: err,
}
}
func errAmbiguousFieldName(err error) *s3Error {
return &s3Error{
code: "AmbiguousFieldName",
message: "Field name matches to multiple fields in the file. Check the SQL expression and the file, and try again.",
statusCode: 400,
cause: err,
}
}
func errIntegerOverflow(err error) *s3Error {
return &s3Error{
code: "IntegerOverflow",
message: "Integer overflow or underflow in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errIllegalSQLFunctionArgument(err error) *s3Error {
return &s3Error{
code: "IllegalSqlFunctionArgument",
message: "Illegal argument was used in the SQL function.",
statusCode: 400,
cause: err,
}
}
func errMultipleDataSourcesUnsupported(err error) *s3Error {
return &s3Error{
code: "MultipleDataSourcesUnsupported",
message: "Multiple data sources are not supported.",
statusCode: 400,
cause: err,
}
}
func errMissingHeaders(err error) *s3Error {
return &s3Error{
code: "MissingHeaders",
message: "Some headers in the query are missing from the file. Check the file and try again.",
statusCode: 400,
cause: err,
}
}
func errUnrecognizedFormatException(err error) *s3Error {
return &s3Error{
code: "UnrecognizedFormatException",
message: "Encountered an invalid record type.",
statusCode: 400,
cause: err,
}
}
//////////////////////////////////////////////////////////////////////////////////////
//
// SQL parsing errors.
//
//////////////////////////////////////////////////////////////////////////////////////
func errLexerInvalidChar(err error) *s3Error {
return &s3Error{
code: "LexerInvalidChar",
message: "The SQL expression contains an invalid character.",
statusCode: 400,
cause: err,
}
}
func errLexerInvalidOperator(err error) *s3Error {
return &s3Error{
code: "LexerInvalidOperator",
message: "The SQL expression contains an invalid literal.",
statusCode: 400,
cause: err,
}
}
func errLexerInvalidLiteral(err error) *s3Error {
return &s3Error{
code: "LexerInvalidLiteral",
message: "The SQL expression contains an invalid operator.",
statusCode: 400,
cause: err,
}
}
func errLexerInvalidIONLiteral(err error) *s3Error {
return &s3Error{
code: "LexerInvalidIONLiteral",
message: "The SQL expression contains an invalid operator.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedDatePart(err error) *s3Error {
return &s3Error{
code: "ParseExpectedDatePart",
message: "Did not find the expected date part in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedKeyword(err error) *s3Error {
return &s3Error{
code: "ParseExpectedKeyword",
message: "Did not find the expected keyword in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedTokenType(err error) *s3Error {
return &s3Error{
code: "ParseExpectedTokenType",
message: "Did not find the expected token in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpected2TokenTypes(err error) *s3Error {
return &s3Error{
code: "ParseExpected2TokenTypes",
message: "Did not find the expected token in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedNumber(err error) *s3Error {
return &s3Error{
code: "ParseExpectedNumber",
message: "Did not find the expected number in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedRightParenBuiltinFunctionCall(err error) *s3Error {
return &s3Error{
code: "ParseExpectedRightParenBuiltinFunctionCall",
message: "Did not find the expected right parenthesis character in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedTypeName(err error) *s3Error {
return &s3Error{
code: "ParseExpectedTypeName",
message: "Did not find the expected type name in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedWhenClause(err error) *s3Error {
return &s3Error{
code: "ParseExpectedWhenClause",
message: "Did not find the expected WHEN clause in the SQL expression. CASE is not supported.",
statusCode: 400,
cause: err,
}
}
func errParseUnsupportedToken(err error) *s3Error {
return &s3Error{
code: "ParseUnsupportedToken",
message: "The SQL expression contains an unsupported token.",
statusCode: 400,
cause: err,
}
}
func errParseUnsupportedLiteralsGroupBy(err error) *s3Error {
return &s3Error{
code: "ParseUnsupportedLiteralsGroupBy",
message: "The SQL expression contains an unsupported use of GROUP BY.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedMember(err error) *s3Error {
return &s3Error{
code: "ParseExpectedMember",
message: "The SQL expression contains an unsupported use of MEMBER.",
statusCode: 400,
cause: err,
}
}
func errParseUnsupportedCase(err error) *s3Error {
return &s3Error{
code: "ParseUnsupportedCase",
message: "The SQL expression contains an unsupported use of CASE.",
statusCode: 400,
cause: err,
}
}
func errParseUnsupportedCaseClause(err error) *s3Error {
return &s3Error{
code: "ParseUnsupportedCaseClause",
message: "The SQL expression contains an unsupported use of CASE.",
statusCode: 400,
cause: err,
}
}
func errParseUnsupportedAlias(err error) *s3Error {
return &s3Error{
code: "ParseUnsupportedAlias",
message: "The SQL expression contains an unsupported use of ALIAS.",
statusCode: 400,
cause: err,
}
}
func errParseInvalidPathComponent(err error) *s3Error {
return &s3Error{
code: "ParseInvalidPathComponent",
message: "The SQL expression contains an invalid path component.",
statusCode: 400,
cause: err,
}
}
func errParseMissingIdentAfterAt(err error) *s3Error {
return &s3Error{
code: "ParseMissingIdentAfterAt",
message: "Did not find the expected identifier after the @ symbol in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseUnexpectedOperator(err error) *s3Error {
return &s3Error{
code: "ParseUnexpectedOperator",
message: "The SQL expression contains an unexpected operator.",
statusCode: 400,
cause: err,
}
}
func errParseUnexpectedTerm(err error) *s3Error {
return &s3Error{
code: "ParseUnexpectedTerm",
message: "The SQL expression contains an unexpected term.",
statusCode: 400,
cause: err,
}
}
func errParseUnexpectedToken(err error) *s3Error {
return &s3Error{
code: "ParseUnexpectedToken",
message: "The SQL expression contains an unexpected token.",
statusCode: 400,
cause: err,
}
}
func errParseUnExpectedKeyword(err error) *s3Error {
return &s3Error{
code: "ParseUnExpectedKeyword",
message: "The SQL expression contains an unexpected keyword.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedExpression(err error) *s3Error {
return &s3Error{
code: "ParseExpectedExpression",
message: "Did not find the expected SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedLeftParenAfterCast(err error) *s3Error {
return &s3Error{
code: "ParseExpectedLeftParenAfterCast",
message: "Did not find the expected left parenthesis after CAST in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedLeftParenValueConstructor(err error) *s3Error {
return &s3Error{
code: "ParseExpectedLeftParenValueConstructor",
message: "Did not find expected the left parenthesis in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedLeftParenBuiltinFunctionCall(err error) *s3Error {
return &s3Error{
code: "ParseExpectedLeftParenBuiltinFunctionCall",
message: "Did not find the expected left parenthesis in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedArgumentDelimiter(err error) *s3Error {
return &s3Error{
code: "ParseExpectedArgumentDelimiter",
message: "Did not find the expected argument delimiter in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseCastArity(err error) *s3Error {
return &s3Error{
code: "ParseCastArity",
message: "The SQL expression CAST has incorrect arity.",
statusCode: 400,
cause: err,
}
}
func errParseEmptySelect(err error) *s3Error {
return &s3Error{
code: "ParseEmptySelect",
message: "The SQL expression contains an empty SELECT.",
statusCode: 400,
cause: err,
}
}
func errParseSelectMissingFrom(err error) *s3Error {
return &s3Error{
code: "ParseSelectMissingFrom",
message: "The SQL expression contains a missing FROM after SELECT list.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedIdentForGroupName(err error) *s3Error {
return &s3Error{
code: "ParseExpectedIdentForGroupName",
message: "GROUP is not supported in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedIdentForAlias(err error) *s3Error {
return &s3Error{
code: "ParseExpectedIdentForAlias",
message: "Did not find the expected identifier for the alias in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseUnsupportedCallWithStar(err error) *s3Error {
return &s3Error{
code: "ParseUnsupportedCallWithStar",
message: "Only COUNT with (*) as a parameter is supported in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseMalformedJoin(err error) *s3Error {
return &s3Error{
code: "ParseMalformedJoin",
message: "JOIN is not supported in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseExpectedIdentForAt(err error) *s3Error {
return &s3Error{
code: "ParseExpectedIdentForAt",
message: "Did not find the expected identifier for AT name in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errParseCannotMixSqbAndWildcardInSelectList(err error) *s3Error {
return &s3Error{
code: "ParseCannotMixSqbAndWildcardInSelectList",
message: "Cannot mix [] and * in the same expression in a SELECT list in SQL expression.",
statusCode: 400,
cause: err,
}
}
//////////////////////////////////////////////////////////////////////////////////////
//
// CAST() related errors.
//
//////////////////////////////////////////////////////////////////////////////////////
func errCastFailed(err error) *s3Error {
return &s3Error{
code: "CastFailed",
message: "Attempt to convert from one data type to another using CAST failed in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errInvalidCast(err error) *s3Error {
return &s3Error{
code: "InvalidCast",
message: "Attempt to convert from one data type to another using CAST failed in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errEvaluatorInvalidTimestampFormatPattern(err error) *s3Error {
return &s3Error{
code: "EvaluatorInvalidTimestampFormatPattern",
message: "Invalid time stamp format string in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errEvaluatorInvalidTimestampFormatPatternAdditionalFieldsRequired(err error) *s3Error {
return &s3Error{
code: "EvaluatorInvalidTimestampFormatPattern",
message: "Time stamp format pattern requires additional fields in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errEvaluatorInvalidTimestampFormatPatternSymbolForParsing(err error) *s3Error {
return &s3Error{
code: "EvaluatorInvalidTimestampFormatPatternSymbolForParsing",
message: "Time stamp format pattern contains a valid format symbol that cannot be applied to time stamp parsing in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errEvaluatorTimestampFormatPatternDuplicateFields(err error) *s3Error {
return &s3Error{
code: "EvaluatorTimestampFormatPatternDuplicateFields",
message: "Time stamp format pattern contains multiple format specifiers representing the time stamp field in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errEvaluatorTimestampFormatPatternHourClockAmPmMismatch(err error) *s3Error {
return &s3Error{
code: "EvaluatorTimestampFormatPatternHourClockAmPmMismatch",
message: "Time stamp format pattern contains a 12-hour hour of day format symbol but doesn't also contain an AM/PM field, or it contains a 24-hour hour of day format specifier and contains an AM/PM field in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errEvaluatorUnterminatedTimestampFormatPatternToken(err error) *s3Error {
return &s3Error{
code: "EvaluatorUnterminatedTimestampFormatPatternToken",
message: "Time stamp format pattern contains unterminated token in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errEvaluatorInvalidTimestampFormatPatternToken(err error) *s3Error {
return &s3Error{
code: "EvaluatorInvalidTimestampFormatPatternToken",
message: "Time stamp format pattern contains an invalid token in the SQL expression.",
statusCode: 400,
cause: err,
}
}
func errEvaluatorInvalidTimestampFormatPatternSymbol(err error) *s3Error {
return &s3Error{
code: "EvaluatorInvalidTimestampFormatPatternSymbol",
message: "Time stamp format pattern contains an invalid symbol in the SQL expression.",
statusCode: 400,
cause: err,
}
}
////////////////////////////////////////////////////////////////////////
//
// Generic S3 HTTP handler errors.
//
////////////////////////////////////////////////////////////////////////
func errBusy(err error) *s3Error {
return &s3Error{
code: "Busy",
message: "The service is unavailable. Please retry.",
statusCode: 503,
cause: err,
}
}
func errUnauthorizedAccess(err error) *s3Error {
return &s3Error{
code: "UnauthorizedAccess",
message: "You are not authorized to perform this operation",
statusCode: 401,
cause: err,
}
}
func errEmptyRequestBody(err error) *s3Error {
return &s3Error{
code: "EmptyRequestBody",
message: "Request body cannot be empty.",
statusCode: 400,
cause: err,
}
}
func errUnsupportedRangeHeader(err error) *s3Error {
return &s3Error{
code: "UnsupportedRangeHeader",
message: "Range header is not supported for this operation.",
statusCode: 400,
cause: err,
}
}
func errUnsupportedStorageClass(err error) *s3Error {
return &s3Error{
code: "UnsupportedStorageClass",
message: "Encountered an invalid storage class. Only STANDARD, STANDARD_IA, and ONEZONE_IA storage classes are supported.",
statusCode: 400,
cause: err,
}
}