s3select should honour custom record delimiter (#6419)

Allow custom delimiters like `\r\n`, `a`, `\r` etc in input csv and 
replace with `\n`.

Fixes #6403

pkg/s3select/input.go

@@ -29,6 +29,7 @@ import (
 	"net/http"
 
 	gzip "github.com/klauspost/pgzip"
+	"github.com/minio/minio/pkg/ioutil"
 )
 
 const (
@@ -79,6 +80,9 @@ type Options struct {
 	// HasHeader when true, will treat the first row as a header row.
 	HasHeader bool
 
+	// RecordDelimiter is the string that records are delimited by.
+	RecordDelimiter string
+
 	// FieldDelimiter is the string that fields are delimited by.
 	FieldDelimiter string
 
@@ -127,7 +131,8 @@ func NewInput(opts *Options) (*Input, error) {
 		tempBytesScanned = opts.StreamSize
 		myReader = bzip2.NewReader(opts.ReadFrom)
 	}
-
+	// DelimitedReader treats custom record delimiter like `\r\n`,`\r`,`ab` etc and replaces it with `\n`.
+	normalizedReader := ioutil.NewDelimitedReader(myReader, []rune(opts.RecordDelimiter))
 	progress := &statInfo{
 		BytesScanned:   tempBytesScanned,
 		BytesProcessed: 0,
@@ -135,7 +140,7 @@ func NewInput(opts *Options) (*Input, error) {
 	}
 	reader := &Input{
 		options: opts,
-		reader:  csv.NewReader(myReader),
+		reader:  csv.NewReader(normalizedReader),
 		stats:   progress,
 	}
 	reader.firstRow = nil

pkg/s3select/select_test.go

@@ -48,6 +48,7 @@ func TestCheckForDuplicates(t *testing.T) {
 func TestMyProcessing(t *testing.T) {
 	options := &Options{
 		HasHeader:            false,
+		RecordDelimiter:      "\n",
 		FieldDelimiter:       ",",
 		Comments:             "",
 		Name:                 "S3Object", // Default table name for all objects
@@ -96,6 +97,7 @@ func TestMyProcessing(t *testing.T) {
 func TestMyRowIndexResults(t *testing.T) {
 	options := &Options{
 		HasHeader:            false,
+		RecordDelimiter:      "\n",
 		FieldDelimiter:       ",",
 		Comments:             "",
 		Name:                 "S3Object", // Default table name for all objects
@@ -271,6 +273,7 @@ func TestMyParser(t *testing.T) {
 	for _, table := range tables {
 		options := &Options{
 			HasHeader:            false,
+			RecordDelimiter:      "\n",
 			FieldDelimiter:       ",",
 			Comments:             "",
 			Name:                 "S3Object", // Default table name for all objects
@@ -346,6 +349,7 @@ func TestMyAggregationFunc(t *testing.T) {
 func TestToStringAgg(t *testing.T) {
 	options := &Options{
 		HasHeader:            false,
+		RecordDelimiter:      "\n",
 		FieldDelimiter:       ",",
 		Comments:             "",
 		Name:                 "S3Object", // Default table name for all objects
@@ -380,6 +384,7 @@ func TestToStringAgg(t *testing.T) {
 func TestMyRowColLiteralResults(t *testing.T) {
 	options := &Options{
 		HasHeader:            false,
+		RecordDelimiter:      "\n",
 		FieldDelimiter:       ",",
 		Comments:             "",
 		Name:                 "S3Object", // Default table name for all objects
@@ -459,6 +464,7 @@ func TestMyWhereEval(t *testing.T) {
 	for _, table := range tables {
 		options := &Options{
 			HasHeader:            false,
+			RecordDelimiter:      "\n",
 			FieldDelimiter:       ",",
 			Comments:             "",
 			Name:                 "S3Object", // Default table name for all objects
@@ -610,6 +616,7 @@ func TestInterpreter(t *testing.T) {
 	for _, table := range tables {
 		options := &Options{
 			HasHeader:            false,
+			RecordDelimiter:      "\n",
 			FieldDelimiter:       ",",
 			Comments:             "",
 			Name:                 "S3Object", // Default table name for all objects
@@ -651,6 +658,7 @@ func TestInterpreter(t *testing.T) {
 func TestMyXMLFunction(t *testing.T) {
 	options := &Options{
 		HasHeader:            false,
+		RecordDelimiter:      "\n",
 		FieldDelimiter:       ",",
 		Comments:             "",
 		Name:                 "S3Object", // Default table name for all objects
@@ -689,6 +697,7 @@ func TestMyXMLFunction(t *testing.T) {
 func TestMyProtocolFunction(t *testing.T) {
 	options := &Options{
 		HasHeader:            false,
+		RecordDelimiter:      "\n",
 		FieldDelimiter:       ",",
 		Comments:             "",
 		Name:                 "S3Object", // Default table name for all objects
@@ -732,6 +741,7 @@ func TestMyProtocolFunction(t *testing.T) {
 func TestMyInfoProtocolFunctions(t *testing.T) {
 	options := &Options{
 		HasHeader:            true,
+		RecordDelimiter:      "\n",
 		FieldDelimiter:       ",",
 		Comments:             "",
 		Name:                 "S3Object", // Default table name for all objects
@@ -773,6 +783,7 @@ func TestMyInfoProtocolFunctions(t *testing.T) {
 func TestMyErrorProtocolFunctions(t *testing.T) {
 	options := &Options{
 		HasHeader:            false,
+		RecordDelimiter:      "\n",
 		FieldDelimiter:       ",",
 		Comments:             "",
 		Name:                 "S3Object", // Default table name for all objects
@@ -1007,6 +1018,7 @@ func TestMyValids(t *testing.T) {
 	for _, table := range tables {
 		options := &Options{
 			HasHeader:            false,
+			RecordDelimiter:      "\n",
 			FieldDelimiter:       ",",
 			Comments:             "",
 			Name:                 "S3Object", // Default table name for all objects