mirror of
https://github.com/pgsty/minio.git
synced 2026-08-09 07:43:29 +03:00
Add new SQL parser to support S3 Select syntax (#7102)
- New parser written from scratch, allows easier and complete parsing of the full S3 Select SQL syntax. Parser definition is directly provided by the AST defined for the SQL grammar. - Bring support to parse and interpret SQL involving JSON path expressions; evaluation of JSON path expressions will be subsequently added. - Bring automatic type inference and conversion for untyped values (e.g. CSV data).
This commit is contained in:
Aditya Manthramurthy
committed by
Harshavardhana
Harshavardhana
parent
0a28c28a8c
commit
2786055df4
+19
@@ -0,0 +1,19 @@
|
||||
// Package lexer defines interfaces and implementations used by Participle to perform lexing.
|
||||
//
|
||||
// The primary interfaces are Definition and Lexer. There are three implementations of these
|
||||
// interfaces:
|
||||
//
|
||||
// TextScannerLexer is based on text/scanner. This is the fastest, but least flexible, in that
|
||||
// tokens are restricted to those supported by that package. It can scan about 5M tokens/second on a
|
||||
// late 2013 15" MacBook Pro.
|
||||
//
|
||||
// The second lexer is constructed via the Regexp() function, mapping regexp capture groups
|
||||
// to tokens. The complete input source is read into memory, so it is unsuitable for large inputs.
|
||||
//
|
||||
// The final lexer provided accepts a lexical grammar in EBNF. Each capitalised production is a
|
||||
// lexical token supported by the resulting Lexer. This is very flexible, but a bit slower, scanning
|
||||
// around 730K tokens/second on the same machine, though it is currently completely unoptimised.
|
||||
// This could/should be converted to a table-based lexer.
|
||||
//
|
||||
// Lexer implementations must use Panic/Panicf to report errors.
|
||||
package lexer
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
package lexer
|
||||
|
||||
import "fmt"
|
||||
|
||||
// Error represents an error while parsing.
|
||||
type Error struct {
|
||||
Message string
|
||||
Pos Position
|
||||
}
|
||||
|
||||
// Errorf creats a new Error at the given position.
|
||||
func Errorf(pos Position, format string, args ...interface{}) *Error {
|
||||
return &Error{
|
||||
Message: fmt.Sprintf(format, args...),
|
||||
Pos: pos,
|
||||
}
|
||||
}
|
||||
|
||||
// Error complies with the error interface and reports the position of an error.
|
||||
func (e *Error) Error() string {
|
||||
filename := e.Pos.Filename
|
||||
if filename == "" {
|
||||
filename = "<source>"
|
||||
}
|
||||
return fmt.Sprintf("%s:%d:%d: %s", filename, e.Pos.Line, e.Pos.Column, e.Message)
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
package lexer
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
)
|
||||
|
||||
const (
|
||||
// EOF represents an end of file.
|
||||
EOF rune = -(iota + 1)
|
||||
)
|
||||
|
||||
// EOFToken creates a new EOF token at the given position.
|
||||
func EOFToken(pos Position) Token {
|
||||
return Token{Type: EOF, Pos: pos}
|
||||
}
|
||||
|
||||
// Definition provides the parser with metadata for a lexer.
|
||||
type Definition interface {
|
||||
// Lex an io.Reader.
|
||||
Lex(io.Reader) (Lexer, error)
|
||||
// Symbols returns a map of symbolic names to the corresponding pseudo-runes for those symbols.
|
||||
// This is the same approach as used by text/scanner. For example, "EOF" might have the rune
|
||||
// value of -1, "Ident" might be -2, and so on.
|
||||
Symbols() map[string]rune
|
||||
}
|
||||
|
||||
// A Lexer returns tokens from a source.
|
||||
type Lexer interface {
|
||||
// Next consumes and returns the next token.
|
||||
Next() (Token, error)
|
||||
}
|
||||
|
||||
// A PeekingLexer returns tokens from a source and allows peeking.
|
||||
type PeekingLexer interface {
|
||||
Lexer
|
||||
// Peek at the next token.
|
||||
Peek(n int) (Token, error)
|
||||
}
|
||||
|
||||
// SymbolsByRune returns a map of lexer symbol names keyed by rune.
|
||||
func SymbolsByRune(def Definition) map[rune]string {
|
||||
out := map[rune]string{}
|
||||
for s, r := range def.Symbols() {
|
||||
out[r] = s
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// NameOfReader attempts to retrieve the filename of a reader.
|
||||
func NameOfReader(r interface{}) string {
|
||||
if nr, ok := r.(interface{ Name() string }); ok {
|
||||
return nr.Name()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// Must takes the result of a Definition constructor call and returns the definition, but panics if
|
||||
// it errors
|
||||
//
|
||||
// eg.
|
||||
//
|
||||
// lex = lexer.Must(lexer.Build(`Symbol = "symbol" .`))
|
||||
func Must(def Definition, err error) Definition {
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// ConsumeAll reads all tokens from a Lexer.
|
||||
func ConsumeAll(lexer Lexer) ([]Token, error) {
|
||||
tokens := []Token{}
|
||||
for {
|
||||
token, err := lexer.Next()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tokens = append(tokens, token)
|
||||
if token.Type == EOF {
|
||||
return tokens, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Position of a token.
|
||||
type Position struct {
|
||||
Filename string
|
||||
Offset int
|
||||
Line int
|
||||
Column int
|
||||
}
|
||||
|
||||
func (p Position) GoString() string {
|
||||
return fmt.Sprintf("Position{Filename: %q, Offset: %d, Line: %d, Column: %d}",
|
||||
p.Filename, p.Offset, p.Line, p.Column)
|
||||
}
|
||||
|
||||
func (p Position) String() string {
|
||||
filename := p.Filename
|
||||
if filename == "" {
|
||||
filename = "<source>"
|
||||
}
|
||||
return fmt.Sprintf("%s:%d:%d", filename, p.Line, p.Column)
|
||||
}
|
||||
|
||||
// A Token returned by a Lexer.
|
||||
type Token struct {
|
||||
// Type of token. This is the value keyed by symbol as returned by Definition.Symbols().
|
||||
Type rune
|
||||
Value string
|
||||
Pos Position
|
||||
}
|
||||
|
||||
// RuneToken represents a rune as a Token.
|
||||
func RuneToken(r rune) Token {
|
||||
return Token{Type: r, Value: string(r)}
|
||||
}
|
||||
|
||||
// EOF returns true if this Token is an EOF token.
|
||||
func (t Token) EOF() bool {
|
||||
return t.Type == EOF
|
||||
}
|
||||
|
||||
func (t Token) String() string {
|
||||
if t.EOF() {
|
||||
return "<EOF>"
|
||||
}
|
||||
return t.Value
|
||||
}
|
||||
|
||||
func (t Token) GoString() string {
|
||||
return fmt.Sprintf("Token{%d, %q}", t.Type, t.Value)
|
||||
}
|
||||
|
||||
// MakeSymbolTable builds a lookup table for checking token ID existence.
|
||||
//
|
||||
// For each symbolic name in "types", the returned map will contain the corresponding token ID as a key.
|
||||
func MakeSymbolTable(def Definition, types ...string) (map[rune]bool, error) {
|
||||
symbols := def.Symbols()
|
||||
table := map[rune]bool{}
|
||||
for _, symbol := range types {
|
||||
rn, ok := symbols[symbol]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("lexer does not support symbol %q", symbol)
|
||||
}
|
||||
table[rn] = true
|
||||
}
|
||||
return table, nil
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package lexer
|
||||
|
||||
// Upgrade a Lexer to a PeekingLexer with arbitrary lookahead.
|
||||
func Upgrade(lexer Lexer) PeekingLexer {
|
||||
if peeking, ok := lexer.(PeekingLexer); ok {
|
||||
return peeking
|
||||
}
|
||||
return &lookaheadLexer{Lexer: lexer}
|
||||
}
|
||||
|
||||
type lookaheadLexer struct {
|
||||
Lexer
|
||||
peeked []Token
|
||||
}
|
||||
|
||||
func (l *lookaheadLexer) Peek(n int) (Token, error) {
|
||||
for len(l.peeked) <= n {
|
||||
t, err := l.Lexer.Next()
|
||||
if err != nil {
|
||||
return Token{}, err
|
||||
}
|
||||
if t.EOF() {
|
||||
return t, nil
|
||||
}
|
||||
l.peeked = append(l.peeked, t)
|
||||
}
|
||||
return l.peeked[n], nil
|
||||
}
|
||||
|
||||
func (l *lookaheadLexer) Next() (Token, error) {
|
||||
if len(l.peeked) > 0 {
|
||||
t := l.peeked[0]
|
||||
l.peeked = l.peeked[1:]
|
||||
return t, nil
|
||||
}
|
||||
return l.Lexer.Next()
|
||||
}
|
||||
+112
@@ -0,0 +1,112 @@
|
||||
package lexer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"regexp"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
var eolBytes = []byte("\n")
|
||||
|
||||
type regexpDefinition struct {
|
||||
re *regexp.Regexp
|
||||
symbols map[string]rune
|
||||
}
|
||||
|
||||
// Regexp creates a lexer definition from a regular expression.
|
||||
//
|
||||
// Each named sub-expression in the regular expression matches a token. Anonymous sub-expressions
|
||||
// will be matched and discarded.
|
||||
//
|
||||
// eg.
|
||||
//
|
||||
// def, err := Regexp(`(?P<Ident>[a-z]+)|(\s+)|(?P<Number>\d+)`)
|
||||
func Regexp(pattern string) (Definition, error) {
|
||||
re, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
symbols := map[string]rune{
|
||||
"EOF": EOF,
|
||||
}
|
||||
for i, sym := range re.SubexpNames()[1:] {
|
||||
if sym != "" {
|
||||
symbols[sym] = EOF - 1 - rune(i)
|
||||
}
|
||||
}
|
||||
return ®expDefinition{re: re, symbols: symbols}, nil
|
||||
}
|
||||
|
||||
func (d *regexpDefinition) Lex(r io.Reader) (Lexer, error) {
|
||||
b, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return ®expLexer{
|
||||
pos: Position{
|
||||
Filename: NameOfReader(r),
|
||||
Line: 1,
|
||||
Column: 1,
|
||||
},
|
||||
b: b,
|
||||
re: d.re,
|
||||
names: d.re.SubexpNames(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (d *regexpDefinition) Symbols() map[string]rune {
|
||||
return d.symbols
|
||||
}
|
||||
|
||||
type regexpLexer struct {
|
||||
pos Position
|
||||
b []byte
|
||||
re *regexp.Regexp
|
||||
names []string
|
||||
}
|
||||
|
||||
func (r *regexpLexer) Next() (Token, error) {
|
||||
nextToken:
|
||||
for len(r.b) != 0 {
|
||||
matches := r.re.FindSubmatchIndex(r.b)
|
||||
if matches == nil || matches[0] != 0 {
|
||||
rn, _ := utf8.DecodeRune(r.b)
|
||||
return Token{}, Errorf(r.pos, "invalid token %q", rn)
|
||||
}
|
||||
match := r.b[:matches[1]]
|
||||
token := Token{
|
||||
Pos: r.pos,
|
||||
Value: string(match),
|
||||
}
|
||||
|
||||
// Update lexer state.
|
||||
r.pos.Offset += matches[1]
|
||||
lines := bytes.Count(match, eolBytes)
|
||||
r.pos.Line += lines
|
||||
// Update column.
|
||||
if lines == 0 {
|
||||
r.pos.Column += utf8.RuneCount(match)
|
||||
} else {
|
||||
r.pos.Column = utf8.RuneCount(match[bytes.LastIndex(match, eolBytes):])
|
||||
}
|
||||
// Move slice along.
|
||||
r.b = r.b[matches[1]:]
|
||||
|
||||
// Finally, assign token type. If it is not a named group, we continue to the next token.
|
||||
for i := 2; i < len(matches); i += 2 {
|
||||
if matches[i] != -1 {
|
||||
if r.names[i/2] == "" {
|
||||
continue nextToken
|
||||
}
|
||||
token.Type = EOF - rune(i/2)
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return token, nil
|
||||
}
|
||||
|
||||
return EOFToken(r.pos), nil
|
||||
}
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
package lexer
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/scanner"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// TextScannerLexer is a lexer that uses the text/scanner module.
|
||||
var (
|
||||
TextScannerLexer Definition = &defaultDefinition{}
|
||||
|
||||
// DefaultDefinition defines properties for the default lexer.
|
||||
DefaultDefinition = TextScannerLexer
|
||||
)
|
||||
|
||||
type defaultDefinition struct{}
|
||||
|
||||
func (d *defaultDefinition) Lex(r io.Reader) (Lexer, error) {
|
||||
return Lex(r), nil
|
||||
}
|
||||
|
||||
func (d *defaultDefinition) Symbols() map[string]rune {
|
||||
return map[string]rune{
|
||||
"EOF": scanner.EOF,
|
||||
"Char": scanner.Char,
|
||||
"Ident": scanner.Ident,
|
||||
"Int": scanner.Int,
|
||||
"Float": scanner.Float,
|
||||
"String": scanner.String,
|
||||
"RawString": scanner.RawString,
|
||||
"Comment": scanner.Comment,
|
||||
}
|
||||
}
|
||||
|
||||
// textScannerLexer is a Lexer based on text/scanner.Scanner
|
||||
type textScannerLexer struct {
|
||||
scanner *scanner.Scanner
|
||||
filename string
|
||||
err error
|
||||
}
|
||||
|
||||
// Lex an io.Reader with text/scanner.Scanner.
|
||||
//
|
||||
// This provides very fast lexing of source code compatible with Go tokens.
|
||||
//
|
||||
// Note that this differs from text/scanner.Scanner in that string tokens will be unquoted.
|
||||
func Lex(r io.Reader) Lexer {
|
||||
lexer := lexWithScanner(r, &scanner.Scanner{})
|
||||
lexer.scanner.Error = func(s *scanner.Scanner, msg string) {
|
||||
// This is to support single quoted strings. Hacky.
|
||||
if msg != "illegal char literal" {
|
||||
lexer.err = Errorf(Position(lexer.scanner.Pos()), msg)
|
||||
}
|
||||
}
|
||||
return lexer
|
||||
}
|
||||
|
||||
// LexWithScanner creates a Lexer from a user-provided scanner.Scanner.
|
||||
//
|
||||
// Useful if you need to customise the Scanner.
|
||||
func LexWithScanner(r io.Reader, scan *scanner.Scanner) Lexer {
|
||||
return lexWithScanner(r, scan)
|
||||
}
|
||||
|
||||
func lexWithScanner(r io.Reader, scan *scanner.Scanner) *textScannerLexer {
|
||||
lexer := &textScannerLexer{
|
||||
filename: NameOfReader(r),
|
||||
scanner: scan,
|
||||
}
|
||||
lexer.scanner.Init(r)
|
||||
return lexer
|
||||
}
|
||||
|
||||
// LexBytes returns a new default lexer over bytes.
|
||||
func LexBytes(b []byte) Lexer {
|
||||
return Lex(bytes.NewReader(b))
|
||||
}
|
||||
|
||||
// LexString returns a new default lexer over a string.
|
||||
func LexString(s string) Lexer {
|
||||
return Lex(strings.NewReader(s))
|
||||
}
|
||||
|
||||
func (t *textScannerLexer) Next() (Token, error) {
|
||||
typ := t.scanner.Scan()
|
||||
text := t.scanner.TokenText()
|
||||
pos := Position(t.scanner.Position)
|
||||
pos.Filename = t.filename
|
||||
if t.err != nil {
|
||||
return Token{}, t.err
|
||||
}
|
||||
return textScannerTransform(Token{
|
||||
Type: typ,
|
||||
Value: text,
|
||||
Pos: pos,
|
||||
})
|
||||
}
|
||||
|
||||
func textScannerTransform(token Token) (Token, error) {
|
||||
// Unquote strings.
|
||||
switch token.Type {
|
||||
case scanner.Char:
|
||||
// FIXME(alec): This is pretty hacky...we convert a single quoted char into a double
|
||||
// quoted string in order to support single quoted strings.
|
||||
token.Value = fmt.Sprintf("\"%s\"", token.Value[1:len(token.Value)-1])
|
||||
fallthrough
|
||||
case scanner.String:
|
||||
s, err := strconv.Unquote(token.Value)
|
||||
if err != nil {
|
||||
return Token{}, Errorf(token.Pos, "%s: %q", err.Error(), token.Value)
|
||||
}
|
||||
token.Value = s
|
||||
if token.Type == scanner.Char && utf8.RuneCountInString(s) > 1 {
|
||||
token.Type = scanner.String
|
||||
}
|
||||
case scanner.RawString:
|
||||
token.Value = token.Value[1 : len(token.Value)-1]
|
||||
}
|
||||
return token, nil
|
||||
}
|
||||
Reference in New Issue
Block a user