mirror of
https://github.com/pgsty/minio.git
synced 2026-07-19 12:10:24 +03:00
select: Add ScanRange to CSV&JSON (#14546)
Implements https://docs.aws.amazon.com/AmazonS3/latest/API/API_SelectObjectContent.html#AmazonS3-SelectObjectContent-request-ScanRange Fixes #14539
This commit is contained in:
Klaus Post
@@ -20,6 +20,7 @@ package s3select
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/xml"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
@@ -1374,6 +1375,314 @@ func TestJSONInput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCSVRanges(t *testing.T) {
|
||||
testInput := []byte(`id,time,num,num2,text
|
||||
1,2010-01-01T,7867786,4565.908123,"a text, with comma"
|
||||
2,2017-01-02T03:04Z,-5, 0.765111,
|
||||
`)
|
||||
testTable := []struct {
|
||||
name string
|
||||
query string
|
||||
input []byte
|
||||
requestXML []byte // override request XML
|
||||
wantResult string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "select-all",
|
||||
input: testInput,
|
||||
query: ``,
|
||||
// Since we are doing offset, no headers are used.
|
||||
wantResult: `{"_1":"2","_2":"2017-01-02T03:04Z","_3":"-5","_4":" 0.765111","_5":""}`,
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT * from s3object AS s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<CSV>
|
||||
<FileHeaderInfo>NONE</FileHeaderInfo>
|
||||
<QuoteCharacter>"</QuoteCharacter>
|
||||
</CSV>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
<ScanRange><Start>76</Start><End>109</End></ScanRange>
|
||||
</SelectObjectContentRequest>`),
|
||||
},
|
||||
{
|
||||
name: "select-remain",
|
||||
input: testInput,
|
||||
// Since we are doing offset, no headers are used.
|
||||
wantResult: `{"_1":"2","_2":"2017-01-02T03:04Z","_3":"-5","_4":" 0.765111","_5":""}`,
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT * from s3object AS s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<CSV>
|
||||
<FileHeaderInfo>NONE</FileHeaderInfo>
|
||||
<QuoteCharacter>"</QuoteCharacter>
|
||||
</CSV>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
<ScanRange><Start>76</Start></ScanRange>
|
||||
</SelectObjectContentRequest>`),
|
||||
},
|
||||
{
|
||||
name: "select-end-bytes",
|
||||
input: testInput,
|
||||
query: ``,
|
||||
// Since we are doing offset, no headers are used.
|
||||
wantResult: `{"_1":"2","_2":"2017-01-02T03:04Z","_3":"-5","_4":" 0.765111","_5":""}`,
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT * from s3object AS s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<CSV>
|
||||
<FileHeaderInfo>NONE</FileHeaderInfo>
|
||||
<QuoteCharacter>"</QuoteCharacter>
|
||||
</CSV>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
<ScanRange><End>35</End></ScanRange>
|
||||
</SelectObjectContentRequest>`),
|
||||
},
|
||||
{
|
||||
name: "select-middle",
|
||||
input: testInput,
|
||||
// Since we are doing offset, no headers are used.
|
||||
wantResult: `{"_1":"a text, with comma"}`,
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT * from s3object AS s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<CSV>
|
||||
<FileHeaderInfo>NONE</FileHeaderInfo>
|
||||
<QuoteCharacter>"</QuoteCharacter>
|
||||
</CSV>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
<ScanRange><Start>56</Start><End>76</End></ScanRange>
|
||||
</SelectObjectContentRequest>`),
|
||||
},
|
||||
{
|
||||
name: "error-end-before-start",
|
||||
input: testInput,
|
||||
// Since we are doing offset, no headers are used.
|
||||
wantResult: ``,
|
||||
wantErr: true,
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT * from s3object AS s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<CSV>
|
||||
<FileHeaderInfo>NONE</FileHeaderInfo>
|
||||
<QuoteCharacter>"</QuoteCharacter>
|
||||
</CSV>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
<ScanRange><Start>56</Start><End>26</End></ScanRange>
|
||||
</SelectObjectContentRequest>`),
|
||||
},
|
||||
{
|
||||
name: "error-empty",
|
||||
input: testInput,
|
||||
// Since we are doing offset, no headers are used.
|
||||
wantResult: ``,
|
||||
wantErr: true,
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT * from s3object AS s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<CSV>
|
||||
<FileHeaderInfo>NONE</FileHeaderInfo>
|
||||
<QuoteCharacter>"</QuoteCharacter>
|
||||
</CSV>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
<ScanRange></ScanRange>
|
||||
</SelectObjectContentRequest>`),
|
||||
},
|
||||
{
|
||||
name: "error-after-eof",
|
||||
input: testInput,
|
||||
// Since we are doing offset, no headers are used.
|
||||
wantResult: ``,
|
||||
wantErr: true,
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT * from s3object AS s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<CSV>
|
||||
<FileHeaderInfo>NONE</FileHeaderInfo>
|
||||
<QuoteCharacter>"</QuoteCharacter>
|
||||
</CSV>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
<ScanRange><Start>2600000</Start></ScanRange>
|
||||
</SelectObjectContentRequest>`),
|
||||
},
|
||||
{
|
||||
name: "error-after-eof",
|
||||
input: testInput,
|
||||
// Since we are doing offset, no headers are used.
|
||||
wantResult: ``,
|
||||
wantErr: true,
|
||||
requestXML: []byte(`<?xml version="1.0" encoding="UTF-8"?>
|
||||
<SelectObjectContentRequest>
|
||||
<Expression>SELECT * from s3object AS s</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization>
|
||||
<CompressionType>NONE</CompressionType>
|
||||
<CSV>
|
||||
<FileHeaderInfo>NONE</FileHeaderInfo>
|
||||
<QuoteCharacter>"</QuoteCharacter>
|
||||
</CSV>
|
||||
</InputSerialization>
|
||||
<OutputSerialization>
|
||||
<JSON>
|
||||
</JSON>
|
||||
</OutputSerialization>
|
||||
<RequestProgress>
|
||||
<Enabled>FALSE</Enabled>
|
||||
</RequestProgress>
|
||||
<ScanRange><Start>2600000</Start><End>2600001</End></ScanRange>
|
||||
</SelectObjectContentRequest>`),
|
||||
},
|
||||
}
|
||||
|
||||
for _, testCase := range testTable {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
testReq := testCase.requestXML
|
||||
s3Select, err := NewS3Select(bytes.NewReader(testReq))
|
||||
if err != nil {
|
||||
if !testCase.wantErr {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("got expected error: %v", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = s3Select.Open(func(offset, length int64) (io.ReadCloser, error) {
|
||||
in := testCase.input
|
||||
if offset != 0 || length != -1 {
|
||||
// Copy from SelectObjectContentHandler
|
||||
isSuffixLength := false
|
||||
if offset < 0 {
|
||||
isSuffixLength = true
|
||||
}
|
||||
|
||||
if length > 0 {
|
||||
length--
|
||||
}
|
||||
|
||||
rs := &httpRangeSpec{
|
||||
IsSuffixLength: isSuffixLength,
|
||||
Start: offset,
|
||||
End: offset + length,
|
||||
}
|
||||
if length == -1 {
|
||||
rs.End = -1
|
||||
}
|
||||
t.Log("input, offset:", offset, "length:", length, "size:", len(in))
|
||||
offset, length, err = rs.GetOffsetLength(int64(len(in)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
t.Log("rs:", *rs, "offset:", offset, "length:", length)
|
||||
in = in[offset : offset+length]
|
||||
}
|
||||
return ioutil.NopCloser(bytes.NewBuffer(in)), nil
|
||||
}); err != nil {
|
||||
if !testCase.wantErr {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Logf("got expected error: %v", err)
|
||||
return
|
||||
} else if testCase.wantErr {
|
||||
t.Error("did not get expected error")
|
||||
return
|
||||
}
|
||||
|
||||
w := &testResponseWriter{}
|
||||
s3Select.Evaluate(w)
|
||||
s3Select.Close()
|
||||
resp := http.Response{
|
||||
StatusCode: http.StatusOK,
|
||||
Body: ioutil.NopCloser(bytes.NewReader(w.response)),
|
||||
ContentLength: int64(len(w.response)),
|
||||
}
|
||||
res, err := minio.NewSelectResults(&resp, "testbucket")
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
got, err := ioutil.ReadAll(res)
|
||||
if err != nil {
|
||||
t.Error(err)
|
||||
return
|
||||
}
|
||||
gotS := strings.TrimSpace(string(got))
|
||||
if !reflect.DeepEqual(gotS, testCase.wantResult) {
|
||||
t.Errorf("received response does not match with expected reply. Query: %s\ngot: %s\nwant:%s", testCase.query, gotS, testCase.wantResult)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestParquetInput(t *testing.T) {
|
||||
os.Setenv("MINIO_API_SELECT_PARQUET", "on")
|
||||
defer os.Setenv("MINIO_API_SELECT_PARQUET", "off")
|
||||
@@ -1728,3 +2037,76 @@ func TestParquetInputSchemaCSV(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// httpRangeSpec represents a range specification as supported by S3 GET
|
||||
// object request.
|
||||
//
|
||||
// Case 1: Not present -> represented by a nil RangeSpec
|
||||
// Case 2: bytes=1-10 (absolute start and end offsets) -> RangeSpec{false, 1, 10}
|
||||
// Case 3: bytes=10- (absolute start offset with end offset unspecified) -> RangeSpec{false, 10, -1}
|
||||
// Case 4: bytes=-30 (suffix length specification) -> RangeSpec{true, -30, -1}
|
||||
type httpRangeSpec struct {
|
||||
// Does the range spec refer to a suffix of the object?
|
||||
IsSuffixLength bool
|
||||
|
||||
// Start and end offset specified in range spec
|
||||
Start, End int64
|
||||
}
|
||||
|
||||
func (h *httpRangeSpec) GetLength(resourceSize int64) (rangeLength int64, err error) {
|
||||
switch {
|
||||
case resourceSize < 0:
|
||||
return 0, errors.New("Resource size cannot be negative")
|
||||
|
||||
case h == nil:
|
||||
rangeLength = resourceSize
|
||||
|
||||
case h.IsSuffixLength:
|
||||
specifiedLen := -h.Start
|
||||
rangeLength = specifiedLen
|
||||
if specifiedLen > resourceSize {
|
||||
rangeLength = resourceSize
|
||||
}
|
||||
|
||||
case h.Start >= resourceSize:
|
||||
return 0, errors.New("errInvalidRange")
|
||||
|
||||
case h.End > -1:
|
||||
end := h.End
|
||||
if resourceSize <= end {
|
||||
end = resourceSize - 1
|
||||
}
|
||||
rangeLength = end - h.Start + 1
|
||||
|
||||
case h.End == -1:
|
||||
rangeLength = resourceSize - h.Start
|
||||
|
||||
default:
|
||||
return 0, errors.New("Unexpected range specification case")
|
||||
}
|
||||
|
||||
return rangeLength, nil
|
||||
}
|
||||
|
||||
// GetOffsetLength computes the start offset and length of the range
|
||||
// given the size of the resource
|
||||
func (h *httpRangeSpec) GetOffsetLength(resourceSize int64) (start, length int64, err error) {
|
||||
if h == nil {
|
||||
// No range specified, implies whole object.
|
||||
return 0, resourceSize, nil
|
||||
}
|
||||
|
||||
length, err = h.GetLength(resourceSize)
|
||||
if err != nil {
|
||||
return 0, 0, err
|
||||
}
|
||||
|
||||
start = h.Start
|
||||
if h.IsSuffixLength {
|
||||
start = resourceSize + h.Start
|
||||
if start < 0 {
|
||||
start = 0
|
||||
}
|
||||
}
|
||||
return start, length, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user