Refactor s3select to support parquet. (#7023)

Also handle pretty formatted JSON documents.

vendor/github.com/minio/parquet-go/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/minio/parquet-go/Makefile

@@ -0,0 +1,36 @@
+GOPATH := $(shell go env GOPATH)
+
+all: check
+
+getdeps:
+	@if [ ! -f ${GOPATH}/bin/golint ]; then echo "Installing golint" && go get -u golang.org/x/lint/golint; fi
+	@if [ ! -f ${GOPATH}/bin/gocyclo ]; then echo "Installing gocyclo" && go get -u github.com/fzipp/gocyclo; fi
+	@if [ ! -f ${GOPATH}/bin/misspell ]; then echo "Installing misspell" && go get -u github.com/client9/misspell/cmd/misspell; fi
+	@if [ ! -f ${GOPATH}/bin/ineffassign ]; then echo "Installing ineffassign" && go get -u github.com/gordonklaus/ineffassign; fi
+
+vet:
+	@echo "Running $@"
+	@go tool vet -atomic -bool -copylocks -nilfunc -printf -shadow -rangeloops -unreachable -unsafeptr -unusedresult *.go
+
+fmt:
+	@echo "Running $@"
+	@gofmt -d *.go
+
+lint:
+	@echo "Running $@"
+	@${GOPATH}/bin/golint -set_exit_status
+
+cyclo:
+	@echo "Running $@"
+	@${GOPATH}/bin/gocyclo -over 200 .
+
+spelling:
+	@${GOPATH}/bin/misspell -locale US -error *.go README.md
+
+ineffassign:
+	@echo "Running $@"
+	@${GOPATH}/bin/ineffassign .
+
+check: getdeps vet fmt lint cyclo spelling ineffassign
+	@echo "Running unit tests"
+	@go test -tags kqueue .

vendor/github.com/minio/parquet-go/README.md

@@ -0,0 +1 @@
+# parquet-go

vendor/github.com/minio/parquet-go/column.go

@@ -0,0 +1,152 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parquet
+
+import (
+	"io"
+	"strings"
+
+	"git.apache.org/thrift.git/lib/go/thrift"
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/parquet-go/gen-go/parquet"
+)
+
+func getColumns(
+	rowGroup *parquet.RowGroup,
+	columnNames set.StringSet,
+	schemaElements []*parquet.SchemaElement,
+	getReaderFunc GetReaderFunc,
+) (nameColumnMap map[string]*column, err error) {
+	nameIndexMap := make(map[string]int)
+	for colIndex, columnChunk := range rowGroup.GetColumns() {
+		meta := columnChunk.GetMetaData()
+		columnName := strings.Join(meta.GetPathInSchema(), ".")
+		if columnNames != nil && !columnNames.Contains(columnName) {
+			continue
+		}
+
+		// Ignore column spanning into another file.
+		if columnChunk.GetFilePath() != "" {
+			continue
+		}
+
+		offset := meta.GetDataPageOffset()
+		if meta.DictionaryPageOffset != nil {
+			offset = meta.GetDictionaryPageOffset()
+		}
+
+		size := meta.GetTotalCompressedSize()
+
+		rc, err := getReaderFunc(offset, size)
+		if err != nil {
+			return nil, err
+		}
+
+		thriftReader := thrift.NewTBufferedTransport(thrift.NewStreamTransportR(rc), int(size))
+
+		if nameColumnMap == nil {
+			nameColumnMap = make(map[string]*column)
+		}
+
+		nameColumnMap[columnName] = &column{
+			name:           columnName,
+			metadata:       meta,
+			schemaElements: schemaElements,
+			rc:             rc,
+			thriftReader:   thriftReader,
+			valueType:      meta.GetType(),
+		}
+
+		nameIndexMap[columnName] = colIndex
+	}
+
+	for name := range nameColumnMap {
+		nameColumnMap[name].nameIndexMap = nameIndexMap
+	}
+
+	return nameColumnMap, nil
+}
+
+type column struct {
+	name           string
+	endOfValues    bool
+	valueIndex     int
+	valueType      parquet.Type
+	metadata       *parquet.ColumnMetaData
+	schemaElements []*parquet.SchemaElement
+	nameIndexMap   map[string]int
+	dictPage       *page
+	dataTable      *table
+	rc             io.ReadCloser
+	thriftReader   *thrift.TBufferedTransport
+}
+
+func (column *column) close() (err error) {
+	if column.rc != nil {
+		err = column.rc.Close()
+		column.rc = nil
+	}
+
+	return err
+}
+
+func (column *column) readPage() {
+	page, _, _, err := readPage(
+		column.thriftReader,
+		column.metadata,
+		column.nameIndexMap,
+		column.schemaElements,
+	)
+
+	if err != nil {
+		column.endOfValues = true
+		return
+	}
+
+	if page.Header.GetType() == parquet.PageType_DICTIONARY_PAGE {
+		column.dictPage = page
+		column.readPage()
+		return
+	}
+
+	page.decode(column.dictPage)
+
+	if column.dataTable == nil {
+		column.dataTable = newTableFromTable(page.DataTable)
+	}
+
+	column.dataTable.Merge(page.DataTable)
+}
+
+func (column *column) read() (value interface{}, valueType parquet.Type) {
+	if column.dataTable == nil {
+		column.readPage()
+		column.valueIndex = 0
+	}
+
+	if column.endOfValues {
+		return nil, column.metadata.GetType()
+	}
+
+	value = column.dataTable.Values[column.valueIndex]
+	column.valueIndex++
+	if len(column.dataTable.Values) == column.valueIndex {
+		column.dataTable = nil
+	}
+
+	return value, column.metadata.GetType()
+}

vendor/github.com/minio/parquet-go/compression.go

@@ -0,0 +1,57 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parquet
+
+import (
+	"bytes"
+	"compress/gzip"
+	"fmt"
+	"io/ioutil"
+
+	"github.com/golang/snappy"
+	"github.com/minio/parquet-go/gen-go/parquet"
+	"github.com/pierrec/lz4"
+	lzo "github.com/rasky/go-lzo"
+)
+
+type compressionCodec parquet.CompressionCodec
+
+func (c compressionCodec) uncompress(buf []byte) ([]byte, error) {
+	switch parquet.CompressionCodec(c) {
+	case parquet.CompressionCodec_UNCOMPRESSED:
+		return buf, nil
+
+	case parquet.CompressionCodec_SNAPPY:
+		return snappy.Decode(nil, buf)
+
+	case parquet.CompressionCodec_GZIP:
+		reader, err := gzip.NewReader(bytes.NewReader(buf))
+		if err != nil {
+			return nil, err
+		}
+		defer reader.Close()
+		return ioutil.ReadAll(reader)
+
+	case parquet.CompressionCodec_LZO:
+		return lzo.Decompress1X(bytes.NewReader(buf), len(buf), 0)
+
+	case parquet.CompressionCodec_LZ4:
+		return ioutil.ReadAll(lz4.NewReader(bytes.NewReader(buf)))
+	}
+
+	return nil, fmt.Errorf("invalid compression codec %v", c)
+}

vendor/github.com/minio/parquet-go/decode.go

@@ -0,0 +1,506 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parquet
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"math"
+
+	"github.com/minio/parquet-go/gen-go/parquet"
+)
+
+func uint32ToBytes(v uint32) []byte {
+	buf := make([]byte, 4)
+	binary.LittleEndian.PutUint32(buf, v)
+	return buf
+}
+
+func uint64ToBytes(v uint64) []byte {
+	buf := make([]byte, 8)
+	binary.LittleEndian.PutUint64(buf, v)
+	return buf
+}
+
+func bytesToUint32(buf []byte) uint32 {
+	return binary.LittleEndian.Uint32(buf)
+}
+
+func bytesToUint64(buf []byte) uint64 {
+	return binary.LittleEndian.Uint64(buf)
+}
+
+func i64sToi32s(i64s []int64) (i32s []int32) {
+	i32s = make([]int32, len(i64s))
+	for i := range i64s {
+		i32s[i] = int32(i64s[i])
+	}
+
+	return i32s
+}
+
+func readBitPacked(reader *bytes.Reader, header, bitWidth uint64) (result []int64, err error) {
+	count := header * 8
+
+	if count == 0 {
+		return result, nil
+	}
+
+	if bitWidth == 0 {
+		return make([]int64, count), nil
+	}
+
+	data := make([]byte, header*bitWidth)
+	if _, err = reader.Read(data); err != nil {
+		return nil, err
+	}
+
+	var val, used, left, b uint64
+
+	valNeedBits := bitWidth
+	i := -1
+	for {
+		if left <= 0 {
+			i++
+			if i >= len(data) {
+				break
+			}
+
+			b = uint64(data[i])
+			left = 8
+			used = 0
+		}
+
+		if left >= valNeedBits {
+			val |= ((b >> used) & ((1 << valNeedBits) - 1)) << (bitWidth - valNeedBits)
+			result = append(result, int64(val))
+			val = 0
+			left -= valNeedBits
+			used += valNeedBits
+			valNeedBits = bitWidth
+		} else {
+			val |= (b >> used) << (bitWidth - valNeedBits)
+			valNeedBits -= left
+			left = 0
+		}
+	}
+
+	return result, nil
+}
+
+func readBools(reader *bytes.Reader, count uint64) (result []bool, err error) {
+	i64s, err := readBitPacked(reader, count, 1)
+	if err != nil {
+		return nil, err
+	}
+
+	var i uint64
+	for i = 0; i < count; i++ {
+		result = append(result, i64s[i] > 0)
+	}
+
+	return result, nil
+}
+
+func readInt32s(reader *bytes.Reader, count uint64) (result []int32, err error) {
+	buf := make([]byte, 4)
+
+	var i uint64
+	for i = 0; i < count; i++ {
+		if _, err = reader.Read(buf); err != nil {
+			return nil, err
+		}
+
+		result = append(result, int32(bytesToUint32(buf)))
+	}
+
+	return result, nil
+}
+
+func readInt64s(reader *bytes.Reader, count uint64) (result []int64, err error) {
+	buf := make([]byte, 8)
+
+	var i uint64
+	for i = 0; i < count; i++ {
+		if _, err = reader.Read(buf); err != nil {
+			return nil, err
+		}
+
+		result = append(result, int64(bytesToUint64(buf)))
+	}
+
+	return result, nil
+}
+
+func readInt96s(reader *bytes.Reader, count uint64) (result [][]byte, err error) {
+	var i uint64
+	for i = 0; i < count; i++ {
+		buf := make([]byte, 12)
+
+		if _, err = reader.Read(buf); err != nil {
+			return nil, err
+		}
+
+		result = append(result, buf)
+	}
+
+	return result, nil
+}
+
+func readFloats(reader *bytes.Reader, count uint64) (result []float32, err error) {
+	buf := make([]byte, 4)
+
+	var i uint64
+	for i = 0; i < count; i++ {
+		if _, err = reader.Read(buf); err != nil {
+			return nil, err
+		}
+
+		result = append(result, math.Float32frombits(bytesToUint32(buf)))
+	}
+
+	return result, nil
+}
+
+func readDoubles(reader *bytes.Reader, count uint64) (result []float64, err error) {
+	buf := make([]byte, 8)
+
+	var i uint64
+	for i = 0; i < count; i++ {
+		if _, err = reader.Read(buf); err != nil {
+			return nil, err
+		}
+
+		result = append(result, math.Float64frombits(bytesToUint64(buf)))
+	}
+
+	return result, nil
+}
+
+func readByteArrays(reader *bytes.Reader, count uint64) (result [][]byte, err error) {
+	buf := make([]byte, 4)
+
+	var i uint64
+	for i = 0; i < count; i++ {
+		if _, err = reader.Read(buf); err != nil {
+			return nil, err
+		}
+
+		data := make([]byte, bytesToUint32(buf))
+		if _, err = reader.Read(data); err != nil {
+			return nil, err
+		}
+
+		result = append(result, data)
+	}
+
+	return result, nil
+}
+
+func readFixedLenByteArrays(reader *bytes.Reader, count, length uint64) (result [][]byte, err error) {
+	var i uint64
+	for i = 0; i < count; i++ {
+		data := make([]byte, length)
+		if _, err = reader.Read(data); err != nil {
+			return nil, err
+		}
+
+		result = append(result, data)
+	}
+
+	return result, nil
+}
+
+func readValues(reader *bytes.Reader, dataType parquet.Type, count, length uint64) (interface{}, error) {
+	switch dataType {
+	case parquet.Type_BOOLEAN:
+		return readBools(reader, count)
+	case parquet.Type_INT32:
+		return readInt32s(reader, count)
+	case parquet.Type_INT64:
+		return readInt64s(reader, count)
+	case parquet.Type_INT96:
+		return readInt96s(reader, count)
+	case parquet.Type_FLOAT:
+		return readFloats(reader, count)
+	case parquet.Type_DOUBLE:
+		return readDoubles(reader, count)
+	case parquet.Type_BYTE_ARRAY:
+		return readByteArrays(reader, count)
+	case parquet.Type_FIXED_LEN_BYTE_ARRAY:
+		return readFixedLenByteArrays(reader, count, length)
+	}
+
+	return nil, fmt.Errorf("unknown parquet type %v", dataType)
+}
+
+func readUnsignedVarInt(reader *bytes.Reader) (v uint64, err error) {
+	var b byte
+	var shift uint64
+
+	for {
+		if b, err = reader.ReadByte(); err != nil {
+			return 0, err
+		}
+
+		if v |= ((uint64(b) & 0x7F) << shift); b&0x80 == 0 {
+			break
+		}
+
+		shift += 7
+	}
+
+	return v, nil
+}
+
+func readRLE(reader *bytes.Reader, header, bitWidth uint64) (result []int64, err error) {
+	width := (bitWidth + 7) / 8
+	data := make([]byte, width)
+	if width > 0 {
+		if _, err = reader.Read(data); err != nil {
+			return nil, err
+		}
+	}
+
+	if width < 4 {
+		data = append(data, make([]byte, 4-width)...)
+	}
+
+	val := int64(bytesToUint32(data))
+
+	count := header >> 1
+	result = make([]int64, count)
+	for i := range result {
+		result[i] = val
+	}
+
+	return result, nil
+}
+
+func readRLEBitPackedHybrid(reader *bytes.Reader, length, bitWidth uint64) (result []int64, err error) {
+	if length <= 0 {
+		var i32s []int32
+		i32s, err = readInt32s(reader, 1)
+		if err != nil {
+			return nil, err
+		}
+		length = uint64(i32s[0])
+	}
+
+	buf := make([]byte, length)
+	if _, err = reader.Read(buf); err != nil {
+		return nil, err
+	}
+
+	reader = bytes.NewReader(buf)
+	for reader.Len() > 0 {
+		header, err := readUnsignedVarInt(reader)
+		if err != nil {
+			return nil, err
+		}
+
+		var i64s []int64
+		if header&1 == 0 {
+			i64s, err = readRLE(reader, header, bitWidth)
+		} else {
+			i64s, err = readBitPacked(reader, header>>1, bitWidth)
+		}
+
+		if err != nil {
+			return nil, err
+		}
+
+		result = append(result, i64s...)
+	}
+
+	return result, nil
+}
+
+func readDeltaBinaryPackedInt(reader *bytes.Reader) (result []int64, err error) {
+	blockSize, err := readUnsignedVarInt(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	numMiniblocksInBlock, err := readUnsignedVarInt(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	numValues, err := readUnsignedVarInt(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	firstValueZigZag, err := readUnsignedVarInt(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	v := int64(firstValueZigZag>>1) ^ (-int64(firstValueZigZag & 1))
+	result = append(result, v)
+
+	numValuesInMiniBlock := blockSize / numMiniblocksInBlock
+
+	bitWidths := make([]uint64, numMiniblocksInBlock)
+	for uint64(len(result)) < numValues {
+		minDeltaZigZag, err := readUnsignedVarInt(reader)
+		if err != nil {
+			return nil, err
+		}
+
+		for i := 0; uint64(i) < numMiniblocksInBlock; i++ {
+			b, err := reader.ReadByte()
+			if err != nil {
+				return nil, err
+			}
+			bitWidths[i] = uint64(b)
+		}
+
+		minDelta := int64(minDeltaZigZag>>1) ^ (-int64(minDeltaZigZag & 1))
+		for i := 0; uint64(i) < numMiniblocksInBlock; i++ {
+			i64s, err := readBitPacked(reader, numValuesInMiniBlock/8, bitWidths[i])
+			if err != nil {
+				return nil, err
+			}
+
+			for j := range i64s {
+				v += i64s[j] + minDelta
+				result = append(result, v)
+			}
+		}
+	}
+
+	return result[:numValues], nil
+}
+
+func readDeltaLengthByteArrays(reader *bytes.Reader) (result [][]byte, err error) {
+	i64s, err := readDeltaBinaryPackedInt(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	for i := 0; i < len(i64s); i++ {
+		arrays, err := readFixedLenByteArrays(reader, 1, uint64(i64s[i]))
+		if err != nil {
+			return nil, err
+		}
+
+		result = append(result, arrays[0])
+	}
+
+	return result, nil
+}
+
+func readDeltaByteArrays(reader *bytes.Reader) (result [][]byte, err error) {
+	i64s, err := readDeltaBinaryPackedInt(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	suffixes, err := readDeltaLengthByteArrays(reader)
+	if err != nil {
+		return nil, err
+	}
+
+	result = append(result, suffixes[0])
+	for i := 1; i < len(i64s); i++ {
+		prefixLength := i64s[i]
+		val := append([]byte{}, result[i-1][:prefixLength]...)
+		val = append(val, suffixes[i]...)
+		result = append(result, val)
+	}
+
+	return result, nil
+}
+
+func readDataPageValues(
+	bytesReader *bytes.Reader,
+	encoding parquet.Encoding,
+	dataType parquet.Type,
+	convertedType parquet.ConvertedType,
+	count, bitWidth uint64,
+) (result interface{}, resultDataType parquet.Type, err error) {
+	switch encoding {
+	case parquet.Encoding_PLAIN:
+		result, err = readValues(bytesReader, dataType, count, bitWidth)
+		return result, dataType, err
+
+	case parquet.Encoding_PLAIN_DICTIONARY:
+		b, err := bytesReader.ReadByte()
+		if err != nil {
+			return nil, -1, err
+		}
+
+		i64s, err := readRLEBitPackedHybrid(bytesReader, uint64(bytesReader.Len()), uint64(b))
+		if err != nil {
+			return nil, -1, err
+		}
+
+		return i64s[:count], parquet.Type_INT64, nil
+
+	case parquet.Encoding_RLE:
+		i64s, err := readRLEBitPackedHybrid(bytesReader, 0, bitWidth)
+		if err != nil {
+			return nil, -1, err
+		}
+
+		i64s = i64s[:count]
+
+		if dataType == parquet.Type_INT32 {
+			return i64sToi32s(i64s), parquet.Type_INT32, nil
+		}
+
+		return i64s, parquet.Type_INT64, nil
+
+	case parquet.Encoding_BIT_PACKED:
+		return nil, -1, fmt.Errorf("deprecated parquet encoding %v", parquet.Encoding_BIT_PACKED)
+
+	case parquet.Encoding_DELTA_BINARY_PACKED:
+		i64s, err := readDeltaBinaryPackedInt(bytesReader)
+		if err != nil {
+			return nil, -1, err
+		}
+
+		i64s = i64s[:count]
+
+		if dataType == parquet.Type_INT32 {
+			return i64sToi32s(i64s), parquet.Type_INT32, nil
+		}
+
+		return i64s, parquet.Type_INT64, nil
+
+	case parquet.Encoding_DELTA_LENGTH_BYTE_ARRAY:
+		byteSlices, err := readDeltaLengthByteArrays(bytesReader)
+		if err != nil {
+			return nil, -1, err
+		}
+
+		return byteSlices[:count], parquet.Type_FIXED_LEN_BYTE_ARRAY, nil
+
+	case parquet.Encoding_DELTA_BYTE_ARRAY:
+		byteSlices, err := readDeltaByteArrays(bytesReader)
+		if err != nil {
+			return nil, -1, err
+		}
+
+		return byteSlices[:count], parquet.Type_FIXED_LEN_BYTE_ARRAY, nil
+	}
+
+	return nil, -1, fmt.Errorf("unsupported parquet encoding %v", encoding)
+}

vendor/github.com/minio/parquet-go/gen-go/parquet/GoUnusedProtection__.go

@@ -0,0 +1,7 @@
+// Autogenerated by Thrift Compiler (0.10.0)
+// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+
+package parquet
+
+var GoUnusedProtection__ int;
+

vendor/github.com/minio/parquet-go/gen-go/parquet/parquet-consts.go

@@ -0,0 +1,20 @@
+// Autogenerated by Thrift Compiler (0.10.0)
+// DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
+
+package parquet
+
+import (
+	"bytes"
+	"fmt"
+	"git.apache.org/thrift.git/lib/go/thrift"
+)
+
+// (needed to ensure safety because of naive import list construction.)
+var _ = thrift.ZERO
+var _ = fmt.Printf
+var _ = bytes.Equal
+
+
+func init() {
+}
+

vendor/github.com/minio/parquet-go/gen-parquet-format-pkg.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+#
+# Minio Cloud Storage, (C) 2018 Minio, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -e
+
+rm -f parquet.thrift
+wget -q https://github.com/apache/parquet-format/raw/df6132b94f273521a418a74442085fdd5a0aa009/src/main/thrift/parquet.thrift
+thrift --gen go parquet.thrift

vendor/github.com/minio/parquet-go/page.go

@@ -0,0 +1,531 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parquet
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+
+	"git.apache.org/thrift.git/lib/go/thrift"
+	"github.com/minio/parquet-go/gen-go/parquet"
+)
+
+// getBitWidth - returns bits required to place num e.g.
+//
+//    num | width
+//   -----|-------
+//     0  |   0
+//     1  |   1
+//     2  |   2
+//     3  |   2
+//     4  |   3
+//     5  |   3
+//    ... |  ...
+//    ... |  ...
+//
+func getBitWidth(num uint64) (width uint64) {
+	for ; num != 0; num >>= 1 {
+		width++
+	}
+
+	return width
+}
+
+// getMaxDefLevel - get maximum definition level.
+func getMaxDefLevel(nameIndexMap map[string]int, schemaElements []*parquet.SchemaElement, path []string) (v int) {
+	for i := 1; i <= len(path); i++ {
+		name := strings.Join(path[:i], ".")
+		if index, ok := nameIndexMap[name]; ok {
+			if schemaElements[index].GetRepetitionType() != parquet.FieldRepetitionType_REQUIRED {
+				v++
+			}
+		}
+	}
+
+	return v
+}
+
+// getMaxRepLevel - get maximum repetition level.
+func getMaxRepLevel(nameIndexMap map[string]int, schemaElements []*parquet.SchemaElement, path []string) (v int) {
+	for i := 1; i <= len(path); i++ {
+		name := strings.Join(path[:i], ".")
+		if index, ok := nameIndexMap[name]; ok {
+			if schemaElements[index].GetRepetitionType() == parquet.FieldRepetitionType_REPEATED {
+				v++
+			}
+		}
+	}
+
+	return v
+}
+
+func readPageHeader(reader *thrift.TBufferedTransport) (*parquet.PageHeader, error) {
+	pageHeader := parquet.NewPageHeader()
+	if err := pageHeader.Read(thrift.NewTCompactProtocol(reader)); err != nil {
+		return nil, err
+	}
+
+	return pageHeader, nil
+}
+
+func readPageRawData(thriftReader *thrift.TBufferedTransport, metadata *parquet.ColumnMetaData) (page *page, err error) {
+	pageHeader, err := readPageHeader(thriftReader)
+	if err != nil {
+		return nil, err
+	}
+
+	switch pageType := pageHeader.GetType(); pageType {
+	case parquet.PageType_DICTIONARY_PAGE:
+		page = newDictPage()
+	case parquet.PageType_DATA_PAGE, parquet.PageType_DATA_PAGE_V2:
+		page = newDataPage()
+	default:
+		return nil, fmt.Errorf("unsupported page type %v", pageType)
+	}
+
+	compressedPageSize := pageHeader.GetCompressedPageSize()
+	buf := make([]byte, compressedPageSize)
+	if _, err := thriftReader.Read(buf); err != nil {
+		return nil, err
+	}
+
+	page.Header = pageHeader
+	page.CompressType = metadata.GetCodec()
+	page.RawData = buf
+	page.Path = append([]string{}, metadata.GetPathInSchema()...)
+	page.DataType = metadata.GetType()
+
+	return page, nil
+}
+
+func readPage(
+	thriftReader *thrift.TBufferedTransport,
+	metadata *parquet.ColumnMetaData,
+	columnNameIndexMap map[string]int,
+	schemaElements []*parquet.SchemaElement,
+) (page *page, definitionLevels, numRows int64, err error) {
+
+	pageHeader, err := readPageHeader(thriftReader)
+	if err != nil {
+		return nil, 0, 0, err
+	}
+
+	read := func() (data []byte, err error) {
+		var repLevelsLen, defLevelsLen int32
+		var repLevelsBuf, defLevelsBuf []byte
+
+		if pageHeader.GetType() == parquet.PageType_DATA_PAGE_V2 {
+			repLevelsLen = pageHeader.DataPageHeaderV2.GetRepetitionLevelsByteLength()
+			repLevelsBuf = make([]byte, repLevelsLen)
+			if _, err = thriftReader.Read(repLevelsBuf); err != nil {
+				return nil, err
+			}
+
+			defLevelsLen = pageHeader.DataPageHeaderV2.GetDefinitionLevelsByteLength()
+			defLevelsBuf = make([]byte, defLevelsLen)
+			if _, err = thriftReader.Read(defLevelsBuf); err != nil {
+				return nil, err
+			}
+		}
+
+		dataBuf := make([]byte, pageHeader.GetCompressedPageSize()-repLevelsLen-defLevelsLen)
+		if _, err = thriftReader.Read(dataBuf); err != nil {
+			return nil, err
+		}
+
+		if dataBuf, err = compressionCodec(metadata.GetCodec()).uncompress(dataBuf); err != nil {
+			return nil, err
+		}
+
+		if repLevelsLen == 0 && defLevelsLen == 0 {
+			return dataBuf, nil
+		}
+
+		if repLevelsLen > 0 {
+			data = append(data, uint32ToBytes(uint32(repLevelsLen))...)
+			data = append(data, repLevelsBuf...)
+		}
+
+		if defLevelsLen > 0 {
+			data = append(data, uint32ToBytes(uint32(defLevelsLen))...)
+			data = append(data, defLevelsBuf...)
+		}
+
+		data = append(data, dataBuf...)
+
+		return data, nil
+	}
+
+	buf, err := read()
+	if err != nil {
+		return nil, 0, 0, err
+	}
+
+	path := append([]string{}, metadata.GetPathInSchema()...)
+
+	bytesReader := bytes.NewReader(buf)
+	pageType := pageHeader.GetType()
+	switch pageType {
+	case parquet.PageType_INDEX_PAGE:
+		return nil, 0, 0, fmt.Errorf("page type %v is not supported", parquet.PageType_INDEX_PAGE)
+
+	case parquet.PageType_DICTIONARY_PAGE:
+		page = newDictPage()
+		page.Header = pageHeader
+		table := new(table)
+		table.Path = path
+		values, err := readValues(bytesReader, metadata.GetType(),
+			uint64(pageHeader.DictionaryPageHeader.GetNumValues()), 0)
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		table.Values = getTableValues(values, metadata.GetType())
+		page.DataTable = table
+
+		return page, 0, 0, nil
+
+	case parquet.PageType_DATA_PAGE, parquet.PageType_DATA_PAGE_V2:
+		name := strings.Join(path, ".")
+
+		page = newDataPage()
+		page.Header = pageHeader
+
+		maxDefinitionLevel := getMaxDefLevel(columnNameIndexMap, schemaElements, path)
+		maxRepetitionLevel := getMaxRepLevel(columnNameIndexMap, schemaElements, path)
+
+		var numValues uint64
+		var encodingType parquet.Encoding
+
+		if pageHeader.GetType() == parquet.PageType_DATA_PAGE {
+			numValues = uint64(pageHeader.DataPageHeader.GetNumValues())
+			encodingType = pageHeader.DataPageHeader.GetEncoding()
+		} else {
+			numValues = uint64(pageHeader.DataPageHeaderV2.GetNumValues())
+			encodingType = pageHeader.DataPageHeaderV2.GetEncoding()
+		}
+
+		var repetitionLevels []int64
+		if maxRepetitionLevel > 0 {
+			values, _, err := readDataPageValues(bytesReader, parquet.Encoding_RLE, parquet.Type_INT64,
+				-1, numValues, getBitWidth(uint64(maxRepetitionLevel)))
+			if err != nil {
+				return nil, 0, 0, err
+			}
+
+			if repetitionLevels = values.([]int64); uint64(len(repetitionLevels)) > numValues {
+				repetitionLevels = repetitionLevels[:numValues]
+			}
+		} else {
+			repetitionLevels = make([]int64, numValues)
+		}
+
+		var definitionLevels []int64
+		if maxDefinitionLevel > 0 {
+			values, _, err := readDataPageValues(bytesReader, parquet.Encoding_RLE, parquet.Type_INT64,
+				-1, numValues, getBitWidth(uint64(maxDefinitionLevel)))
+			if err != nil {
+				return nil, 0, 0, err
+			}
+			if definitionLevels = values.([]int64); uint64(len(definitionLevels)) > numValues {
+				definitionLevels = definitionLevels[:numValues]
+			}
+		} else {
+			definitionLevels = make([]int64, numValues)
+		}
+
+		var numNulls uint64
+		for i := 0; i < len(definitionLevels); i++ {
+			if definitionLevels[i] != int64(maxDefinitionLevel) {
+				numNulls++
+			}
+		}
+
+		var convertedType parquet.ConvertedType = -1
+		if schemaElements[columnNameIndexMap[name]].IsSetConvertedType() {
+			convertedType = schemaElements[columnNameIndexMap[name]].GetConvertedType()
+		}
+		values, valueType, err := readDataPageValues(bytesReader, encodingType, metadata.GetType(),
+			convertedType, uint64(len(definitionLevels))-numNulls,
+			uint64(schemaElements[columnNameIndexMap[name]].GetTypeLength()))
+		if err != nil {
+			return nil, 0, 0, err
+		}
+		tableValues := getTableValues(values, valueType)
+
+		table := new(table)
+		table.Path = path
+		table.RepetitionType = schemaElements[columnNameIndexMap[name]].GetRepetitionType()
+		table.MaxRepetitionLevel = int32(maxRepetitionLevel)
+		table.MaxDefinitionLevel = int32(maxDefinitionLevel)
+		table.Values = make([]interface{}, len(definitionLevels))
+		table.RepetitionLevels = make([]int32, len(definitionLevels))
+		table.DefinitionLevels = make([]int32, len(definitionLevels))
+
+		j := 0
+		numRows := int64(0)
+		for i := 0; i < len(definitionLevels); i++ {
+			table.RepetitionLevels[i] = int32(repetitionLevels[i])
+			table.DefinitionLevels[i] = int32(definitionLevels[i])
+			if int(table.DefinitionLevels[i]) == maxDefinitionLevel {
+				table.Values[i] = tableValues[j]
+				j++
+			}
+			if table.RepetitionLevels[i] == 0 {
+				numRows++
+			}
+		}
+		page.DataTable = table
+
+		return page, int64(len(definitionLevels)), numRows, nil
+	}
+
+	return nil, 0, 0, fmt.Errorf("unknown page type %v", pageType)
+}
+
+type page struct {
+	Header       *parquet.PageHeader      // Header of a page
+	DataTable    *table                   // Table to store values
+	RawData      []byte                   // Compressed data of the page, which is written in parquet file
+	CompressType parquet.CompressionCodec // Compress type: gzip/snappy/none
+	DataType     parquet.Type             // Parquet type of the values in the page
+	Path         []string                 // Path in schema(include the root)
+	MaxVal       interface{}              // Maximum of the values
+	MinVal       interface{}              // Minimum of the values
+	PageSize     int32
+}
+
+func newPage() *page {
+	return &page{
+		Header:   parquet.NewPageHeader(),
+		PageSize: 8 * 1024,
+	}
+}
+
+func newDictPage() *page {
+	page := newPage()
+	page.Header.DictionaryPageHeader = parquet.NewDictionaryPageHeader()
+	return page
+}
+
+func newDataPage() *page {
+	page := newPage()
+	page.Header.DataPageHeader = parquet.NewDataPageHeader()
+	return page
+}
+
+func (page *page) decode(dictPage *page) {
+	if dictPage == nil || page == nil || page.Header.DataPageHeader == nil ||
+		(page.Header.DataPageHeader.Encoding != parquet.Encoding_RLE_DICTIONARY &&
+			page.Header.DataPageHeader.Encoding != parquet.Encoding_PLAIN_DICTIONARY) {
+		return
+	}
+
+	for i := 0; i < len(page.DataTable.Values); i++ {
+		if page.DataTable.Values[i] != nil {
+			index := page.DataTable.Values[i].(int64)
+			page.DataTable.Values[i] = dictPage.DataTable.Values[index]
+		}
+	}
+}
+
+// Get RepetitionLevels and Definitions from RawData
+func (page *page) getRLDLFromRawData(columnNameIndexMap map[string]int, schemaElements []*parquet.SchemaElement) (numValues int64, numRows int64, err error) {
+	bytesReader := bytes.NewReader(page.RawData)
+
+	pageType := page.Header.GetType()
+
+	var buf []byte
+	if pageType == parquet.PageType_DATA_PAGE_V2 {
+		var repLevelsLen, defLevelsLen int32
+		var repLevelsBuf, defLevelsBuf []byte
+
+		repLevelsLen = page.Header.DataPageHeaderV2.GetRepetitionLevelsByteLength()
+		repLevelsBuf = make([]byte, repLevelsLen)
+		if _, err = bytesReader.Read(repLevelsBuf); err != nil {
+			return 0, 0, err
+		}
+
+		defLevelsLen = page.Header.DataPageHeaderV2.GetDefinitionLevelsByteLength()
+		defLevelsBuf = make([]byte, defLevelsLen)
+		if _, err = bytesReader.Read(defLevelsBuf); err != nil {
+			return 0, 0, err
+		}
+
+		dataBuf := make([]byte, len(page.RawData)-int(repLevelsLen)-int(defLevelsLen))
+		if _, err = bytesReader.Read(dataBuf); err != nil {
+			return 0, 0, err
+		}
+
+		if repLevelsLen == 0 && defLevelsLen == 0 {
+			buf = dataBuf
+		} else {
+			if repLevelsLen > 0 {
+				buf = append(buf, uint32ToBytes(uint32(repLevelsLen))...)
+				buf = append(buf, repLevelsBuf...)
+			}
+
+			if defLevelsLen > 0 {
+				buf = append(buf, uint32ToBytes(uint32(defLevelsLen))...)
+				buf = append(buf, defLevelsBuf...)
+			}
+
+			buf = append(buf, dataBuf...)
+		}
+	} else {
+		if buf, err = compressionCodec(page.CompressType).uncompress(page.RawData); err != nil {
+			return 0, 0, err
+		}
+	}
+
+	bytesReader = bytes.NewReader(buf)
+
+	switch pageType {
+	case parquet.PageType_DICTIONARY_PAGE:
+		table := new(table)
+		table.Path = page.Path
+		page.DataTable = table
+		return 0, 0, nil
+
+	case parquet.PageType_DATA_PAGE, parquet.PageType_DATA_PAGE_V2:
+		var numValues uint64
+		if pageType == parquet.PageType_DATA_PAGE {
+			numValues = uint64(page.Header.DataPageHeader.GetNumValues())
+		} else {
+			numValues = uint64(page.Header.DataPageHeaderV2.GetNumValues())
+		}
+
+		maxDefinitionLevel := getMaxDefLevel(columnNameIndexMap, schemaElements, page.Path)
+		maxRepetitionLevel := getMaxRepLevel(columnNameIndexMap, schemaElements, page.Path)
+
+		var repetitionLevels []int64
+		if maxRepetitionLevel > 0 {
+			values, _, err := readDataPageValues(bytesReader, parquet.Encoding_RLE, parquet.Type_INT64,
+				-1, numValues, getBitWidth(uint64(maxRepetitionLevel)))
+			if err != nil {
+				return 0, 0, err
+			}
+
+			if repetitionLevels = values.([]int64); uint64(len(repetitionLevels)) > numValues {
+				repetitionLevels = repetitionLevels[:numValues]
+			}
+		} else {
+			repetitionLevels = make([]int64, numValues)
+		}
+
+		var definitionLevels []int64
+		if maxDefinitionLevel > 0 {
+			values, _, err := readDataPageValues(bytesReader, parquet.Encoding_RLE, parquet.Type_INT64,
+				-1, numValues, getBitWidth(uint64(maxDefinitionLevel)))
+			if err != nil {
+				return 0, 0, err
+			}
+			if definitionLevels = values.([]int64); uint64(len(definitionLevels)) > numValues {
+				definitionLevels = definitionLevels[:numValues]
+			}
+		} else {
+			definitionLevels = make([]int64, numValues)
+		}
+
+		table := new(table)
+		table.Path = page.Path
+		name := strings.Join(page.Path, ".")
+		table.RepetitionType = schemaElements[columnNameIndexMap[name]].GetRepetitionType()
+		table.MaxRepetitionLevel = int32(maxRepetitionLevel)
+		table.MaxDefinitionLevel = int32(maxDefinitionLevel)
+		table.Values = make([]interface{}, len(definitionLevels))
+		table.RepetitionLevels = make([]int32, len(definitionLevels))
+		table.DefinitionLevels = make([]int32, len(definitionLevels))
+
+		numRows := int64(0)
+		for i := 0; i < len(definitionLevels); i++ {
+			table.RepetitionLevels[i] = int32(repetitionLevels[i])
+			table.DefinitionLevels[i] = int32(definitionLevels[i])
+			if table.RepetitionLevels[i] == 0 {
+				numRows++
+			}
+		}
+		page.DataTable = table
+		page.RawData = buf[len(buf)-bytesReader.Len():]
+
+		return int64(numValues), numRows, nil
+	}
+
+	return 0, 0, fmt.Errorf("Unsupported page type %v", pageType)
+}
+
+func (page *page) getValueFromRawData(columnNameIndexMap map[string]int, schemaElements []*parquet.SchemaElement) (err error) {
+	pageType := page.Header.GetType()
+	switch pageType {
+	case parquet.PageType_DICTIONARY_PAGE:
+		bytesReader := bytes.NewReader(page.RawData)
+		var values interface{}
+		values, err = readValues(bytesReader, page.DataType,
+			uint64(page.Header.DictionaryPageHeader.GetNumValues()), 0)
+		if err != nil {
+			return err
+		}
+
+		page.DataTable.Values = getTableValues(values, page.DataType)
+		return nil
+
+	case parquet.PageType_DATA_PAGE_V2:
+		if page.RawData, err = compressionCodec(page.CompressType).uncompress(page.RawData); err != nil {
+			return err
+		}
+		fallthrough
+	case parquet.PageType_DATA_PAGE:
+		encodingType := page.Header.DataPageHeader.GetEncoding()
+		bytesReader := bytes.NewReader(page.RawData)
+
+		var numNulls uint64
+		for i := 0; i < len(page.DataTable.DefinitionLevels); i++ {
+			if page.DataTable.DefinitionLevels[i] != page.DataTable.MaxDefinitionLevel {
+				numNulls++
+			}
+		}
+
+		name := strings.Join(page.DataTable.Path, ".")
+		var convertedType parquet.ConvertedType = -1
+
+		if schemaElements[columnNameIndexMap[name]].IsSetConvertedType() {
+			convertedType = schemaElements[columnNameIndexMap[name]].GetConvertedType()
+		}
+
+		values, _, err := readDataPageValues(bytesReader, encodingType, page.DataType,
+			convertedType, uint64(len(page.DataTable.DefinitionLevels))-numNulls,
+			uint64(schemaElements[columnNameIndexMap[name]].GetTypeLength()))
+		if err != nil {
+			return err
+		}
+
+		tableValues := getTableValues(values, page.DataType)
+
+		j := 0
+		for i := 0; i < len(page.DataTable.DefinitionLevels); i++ {
+			if page.DataTable.DefinitionLevels[i] == page.DataTable.MaxDefinitionLevel {
+				page.DataTable.Values[i] = tableValues[j]
+				j++
+			}
+		}
+
+		page.RawData = []byte{}
+		return nil
+	}
+
+	return fmt.Errorf("unsupported page type %v", pageType)
+}

vendor/github.com/minio/parquet-go/parquet.go

@@ -0,0 +1,162 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parquet
+
+import (
+	"encoding/binary"
+	"encoding/json"
+	"io"
+
+	"git.apache.org/thrift.git/lib/go/thrift"
+	"github.com/minio/minio-go/pkg/set"
+	"github.com/minio/parquet-go/gen-go/parquet"
+)
+
+// GetReaderFunc - function type returning io.ReadCloser for requested offset/length.
+type GetReaderFunc func(offset, length int64) (io.ReadCloser, error)
+
+func footerSize(getReaderFunc GetReaderFunc) (size int64, err error) {
+	rc, err := getReaderFunc(-8, 4)
+	if err != nil {
+		return 0, err
+	}
+	defer rc.Close()
+
+	buf := make([]byte, 4)
+	if _, err = io.ReadFull(rc, buf); err != nil {
+		return 0, err
+	}
+
+	size = int64(binary.LittleEndian.Uint32(buf))
+
+	return size, nil
+}
+
+func fileMetadata(getReaderFunc GetReaderFunc) (*parquet.FileMetaData, error) {
+	size, err := footerSize(getReaderFunc)
+	if err != nil {
+		return nil, err
+	}
+
+	rc, err := getReaderFunc(-(8 + size), size)
+	if err != nil {
+		return nil, err
+	}
+	defer rc.Close()
+
+	fileMeta := parquet.NewFileMetaData()
+
+	pf := thrift.NewTCompactProtocolFactory()
+	protocol := pf.GetProtocol(thrift.NewStreamTransportR(rc))
+	err = fileMeta.Read(protocol)
+	if err != nil {
+		return nil, err
+	}
+
+	return fileMeta, nil
+}
+
+// Value - denotes column value
+type Value struct {
+	Value interface{}
+	Type  parquet.Type
+}
+
+// MarshalJSON - encodes to JSON data
+func (value Value) MarshalJSON() (data []byte, err error) {
+	return json.Marshal(value.Value)
+}
+
+// File - denotes parquet file.
+type File struct {
+	getReaderFunc  GetReaderFunc
+	schemaElements []*parquet.SchemaElement
+	rowGroups      []*parquet.RowGroup
+	rowGroupIndex  int
+
+	columnNames set.StringSet
+	columns     map[string]*column
+	rowIndex    int64
+}
+
+// Open - opens parquet file with given column names.
+func Open(getReaderFunc GetReaderFunc, columnNames set.StringSet) (*File, error) {
+	fileMeta, err := fileMetadata(getReaderFunc)
+	if err != nil {
+		return nil, err
+	}
+
+	return &File{
+		getReaderFunc:  getReaderFunc,
+		rowGroups:      fileMeta.GetRowGroups(),
+		schemaElements: fileMeta.GetSchema(),
+		columnNames:    columnNames,
+	}, nil
+}
+
+// Read - reads single record.
+func (file *File) Read() (record map[string]Value, err error) {
+	if file.rowGroupIndex >= len(file.rowGroups) {
+		return nil, io.EOF
+	}
+
+	if file.columns == nil {
+		file.columns, err = getColumns(
+			file.rowGroups[file.rowGroupIndex],
+			file.columnNames,
+			file.schemaElements,
+			file.getReaderFunc,
+		)
+		if err != nil {
+			return nil, err
+		}
+
+		file.rowIndex = 0
+	}
+
+	if file.rowIndex >= file.rowGroups[file.rowGroupIndex].GetNumRows() {
+		file.rowGroupIndex++
+		file.Close()
+		return file.Read()
+	}
+
+	record = make(map[string]Value)
+	for name := range file.columns {
+		value, valueType := file.columns[name].read()
+		record[name] = Value{value, valueType}
+	}
+
+	file.rowIndex++
+
+	return record, nil
+}
+
+// Close - closes underneath readers.
+func (file *File) Close() (err error) {
+	if file.columns != nil {
+		return nil
+	}
+
+	for _, column := range file.columns {
+		column.close()
+	}
+
+	file.columns = nil
+	file.rowIndex = 0
+
+	return nil
+}

vendor/github.com/minio/parquet-go/parquet.thrift

@@ -0,0 +1,881 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * File format description for the parquet file format
+ */
+namespace cpp parquet
+namespace java org.apache.parquet.format
+
+/**
+ * Types supported by Parquet.  These types are intended to be used in combination
+ * with the encodings to control the on disk storage format.
+ * For example INT16 is not included as a type since a good encoding of INT32
+ * would handle this.
+ */
+enum Type {
+  BOOLEAN = 0;
+  INT32 = 1;
+  INT64 = 2;
+  INT96 = 3;  // deprecated, only used by legacy implementations.
+  FLOAT = 4;
+  DOUBLE = 5;
+  BYTE_ARRAY = 6;
+  FIXED_LEN_BYTE_ARRAY = 7;
+}
+
+/**
+ * Common types used by frameworks(e.g. hive, pig) using parquet.  This helps map
+ * between types in those frameworks to the base types in parquet.  This is only
+ * metadata and not needed to read or write the data.
+ */
+enum ConvertedType {
+  /** a BYTE_ARRAY actually contains UTF8 encoded chars */
+  UTF8 = 0;
+
+  /** a map is converted as an optional field containing a repeated key/value pair */
+  MAP = 1;
+
+  /** a key/value pair is converted into a group of two fields */
+  MAP_KEY_VALUE = 2;
+
+  /** a list is converted into an optional field containing a repeated field for its
+   * values */
+  LIST = 3;
+
+  /** an enum is converted into a binary field */
+  ENUM = 4;
+
+  /**
+   * A decimal value.
+   *
+   * This may be used to annotate binary or fixed primitive types. The
+   * underlying byte array stores the unscaled value encoded as two's
+   * complement using big-endian byte order (the most significant byte is the
+   * zeroth element). The value of the decimal is the value * 10^{-scale}.
+   *
+   * This must be accompanied by a (maximum) precision and a scale in the
+   * SchemaElement. The precision specifies the number of digits in the decimal
+   * and the scale stores the location of the decimal point. For example 1.23
+   * would have precision 3 (3 total digits) and scale 2 (the decimal point is
+   * 2 digits over).
+   */
+  DECIMAL = 5;
+
+  /**
+   * A Date
+   *
+   * Stored as days since Unix epoch, encoded as the INT32 physical type.
+   *
+   */
+  DATE = 6;
+
+  /**
+   * A time
+   *
+   * The total number of milliseconds since midnight.  The value is stored
+   * as an INT32 physical type.
+   */
+  TIME_MILLIS = 7;
+
+  /**
+   * A time.
+   *
+   * The total number of microseconds since midnight.  The value is stored as
+   * an INT64 physical type.
+   */
+  TIME_MICROS = 8;
+
+  /**
+   * A date/time combination
+   *
+   * Date and time recorded as milliseconds since the Unix epoch.  Recorded as
+   * a physical type of INT64.
+   */
+  TIMESTAMP_MILLIS = 9;
+
+  /**
+   * A date/time combination
+   *
+   * Date and time recorded as microseconds since the Unix epoch.  The value is
+   * stored as an INT64 physical type.
+   */
+  TIMESTAMP_MICROS = 10;
+
+
+  /**
+   * An unsigned integer value.
+   *
+   * The number describes the maximum number of meainful data bits in
+   * the stored value. 8, 16 and 32 bit values are stored using the
+   * INT32 physical type.  64 bit values are stored using the INT64
+   * physical type.
+   *
+   */
+  UINT_8 = 11;
+  UINT_16 = 12;
+  UINT_32 = 13;
+  UINT_64 = 14;
+
+  /**
+   * A signed integer value.
+   *
+   * The number describes the maximum number of meainful data bits in
+   * the stored value. 8, 16 and 32 bit values are stored using the
+   * INT32 physical type.  64 bit values are stored using the INT64
+   * physical type.
+   *
+   */
+  INT_8 = 15;
+  INT_16 = 16;
+  INT_32 = 17;
+  INT_64 = 18;
+
+  /**
+   * An embedded JSON document
+   *
+   * A JSON document embedded within a single UTF8 column.
+   */
+  JSON = 19;
+
+  /**
+   * An embedded BSON document
+   *
+   * A BSON document embedded within a single BINARY column.
+   */
+  BSON = 20;
+
+  /**
+   * An interval of time
+   *
+   * This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12
+   * This data is composed of three separate little endian unsigned
+   * integers.  Each stores a component of a duration of time.  The first
+   * integer identifies the number of months associated with the duration,
+   * the second identifies the number of days associated with the duration
+   * and the third identifies the number of milliseconds associated with
+   * the provided duration.  This duration of time is independent of any
+   * particular timezone or date.
+   */
+  INTERVAL = 21;
+}
+
+/**
+ * Representation of Schemas
+ */
+enum FieldRepetitionType {
+  /** This field is required (can not be null) and each record has exactly 1 value. */
+  REQUIRED = 0;
+
+  /** The field is optional (can be null) and each record has 0 or 1 values. */
+  OPTIONAL = 1;
+
+  /** The field is repeated and can contain 0 or more values */
+  REPEATED = 2;
+}
+
+/**
+ * Statistics per row group and per page
+ * All fields are optional.
+ */
+struct Statistics {
+   /**
+    * DEPRECATED: min and max value of the column. Use min_value and max_value.
+    *
+    * Values are encoded using PLAIN encoding, except that variable-length byte
+    * arrays do not include a length prefix.
+    *
+    * These fields encode min and max values determined by signed comparison
+    * only. New files should use the correct order for a column's logical type
+    * and store the values in the min_value and max_value fields.
+    *
+    * To support older readers, these may be set when the column order is
+    * signed.
+    */
+   1: optional binary max;
+   2: optional binary min;
+   /** count of null value in the column */
+   3: optional i64 null_count;
+   /** count of distinct values occurring */
+   4: optional i64 distinct_count;
+   /**
+    * Min and max values for the column, determined by its ColumnOrder.
+    *
+    * Values are encoded using PLAIN encoding, except that variable-length byte
+    * arrays do not include a length prefix.
+    */
+   5: optional binary max_value;
+   6: optional binary min_value;
+}
+
+/** Empty structs to use as logical type annotations */
+struct StringType {}  // allowed for BINARY, must be encoded with UTF-8
+struct UUIDType {}    // allowed for FIXED[16], must encoded raw UUID bytes
+struct MapType {}     // see LogicalTypes.md
+struct ListType {}    // see LogicalTypes.md
+struct EnumType {}    // allowed for BINARY, must be encoded with UTF-8
+struct DateType {}    // allowed for INT32
+
+/**
+ * Logical type to annotate a column that is always null.
+ *
+ * Sometimes when discovering the schema of existing data, values are always
+ * null and the physical type can't be determined. This annotation signals
+ * the case where the physical type was guessed from all null values.
+ */
+struct NullType {}    // allowed for any physical type, only null values stored
+
+/**
+ * Decimal logical type annotation
+ *
+ * To maintain forward-compatibility in v1, implementations using this logical
+ * type must also set scale and precision on the annotated SchemaElement.
+ *
+ * Allowed for physical types: INT32, INT64, FIXED, and BINARY
+ */
+struct DecimalType {
+  1: required i32 scale
+  2: required i32 precision
+}
+
+/** Time units for logical types */
+struct MilliSeconds {}
+struct MicroSeconds {}
+struct NanoSeconds {}
+union TimeUnit {
+  1: MilliSeconds MILLIS
+  2: MicroSeconds MICROS
+  3: NanoSeconds NANOS
+}
+
+/**
+ * Timestamp logical type annotation
+ *
+ * Allowed for physical types: INT64
+ */
+struct TimestampType {
+  1: required bool isAdjustedToUTC
+  2: required TimeUnit unit
+}
+
+/**
+ * Time logical type annotation
+ *
+ * Allowed for physical types: INT32 (millis), INT64 (micros, nanos)
+ */
+struct TimeType {
+  1: required bool isAdjustedToUTC
+  2: required TimeUnit unit
+}
+
+/**
+ * Integer logical type annotation
+ *
+ * bitWidth must be 8, 16, 32, or 64.
+ *
+ * Allowed for physical types: INT32, INT64
+ */
+struct IntType {
+  1: required byte bitWidth
+  2: required bool isSigned
+}
+
+/**
+ * Embedded JSON logical type annotation
+ *
+ * Allowed for physical types: BINARY
+ */
+struct JsonType {
+}
+
+/**
+ * Embedded BSON logical type annotation
+ *
+ * Allowed for physical types: BINARY
+ */
+struct BsonType {
+}
+
+/**
+ * LogicalType annotations to replace ConvertedType.
+ *
+ * To maintain compatibility, implementations using LogicalType for a
+ * SchemaElement must also set the corresponding ConvertedType from the
+ * following table.
+ */
+union LogicalType {
+  1:  StringType STRING       // use ConvertedType UTF8
+  2:  MapType MAP             // use ConvertedType MAP
+  3:  ListType LIST           // use ConvertedType LIST
+  4:  EnumType ENUM           // use ConvertedType ENUM
+  5:  DecimalType DECIMAL     // use ConvertedType DECIMAL
+  6:  DateType DATE           // use ConvertedType DATE
+  7:  TimeType TIME           // use ConvertedType TIME_MICROS or TIME_MILLIS
+  8:  TimestampType TIMESTAMP // use ConvertedType TIMESTAMP_MICROS or TIMESTAMP_MILLIS
+  // 9: reserved for INTERVAL
+  10: IntType INTEGER         // use ConvertedType INT_* or UINT_*
+  11: NullType UNKNOWN        // no compatible ConvertedType
+  12: JsonType JSON           // use ConvertedType JSON
+  13: BsonType BSON           // use ConvertedType BSON
+  14: UUIDType UUID
+}
+
+/**
+ * Represents a element inside a schema definition.
+ *  - if it is a group (inner node) then type is undefined and num_children is defined
+ *  - if it is a primitive type (leaf) then type is defined and num_children is undefined
+ * the nodes are listed in depth first traversal order.
+ */
+struct SchemaElement {
+  /** Data type for this field. Not set if the current element is a non-leaf node */
+  1: optional Type type;
+
+  /** If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
+   * Otherwise, if specified, this is the maximum bit length to store any of the values.
+   * (e.g. a low cardinality INT col could have this set to 3).  Note that this is
+   * in the schema, and therefore fixed for the entire file.
+   */
+  2: optional i32 type_length;
+
+  /** repetition of the field. The root of the schema does not have a repetition_type.
+   * All other nodes must have one */
+  3: optional FieldRepetitionType repetition_type;
+
+  /** Name of the field in the schema */
+  4: required string name;
+
+  /** Nested fields.  Since thrift does not support nested fields,
+   * the nesting is flattened to a single list by a depth-first traversal.
+   * The children count is used to construct the nested relationship.
+   * This field is not set when the element is a primitive type
+   */
+  5: optional i32 num_children;
+
+  /** When the schema is the result of a conversion from another model
+   * Used to record the original type to help with cross conversion.
+   */
+  6: optional ConvertedType converted_type;
+
+  /** Used when this column contains decimal data.
+   * See the DECIMAL converted type for more details.
+   */
+  7: optional i32 scale
+  8: optional i32 precision
+
+  /** When the original schema supports field ids, this will save the
+   * original field id in the parquet schema
+   */
+  9: optional i32 field_id;
+
+  /**
+   * The logical type of this SchemaElement
+   *
+   * LogicalType replaces ConvertedType, but ConvertedType is still required
+   * for some logical types to ensure forward-compatibility in format v1.
+   */
+  10: optional LogicalType logicalType
+}
+
+/**
+ * Encodings supported by Parquet.  Not all encodings are valid for all types.  These
+ * enums are also used to specify the encoding of definition and repetition levels.
+ * See the accompanying doc for the details of the more complicated encodings.
+ */
+enum Encoding {
+  /** Default encoding.
+   * BOOLEAN - 1 bit per value. 0 is false; 1 is true.
+   * INT32 - 4 bytes per value.  Stored as little-endian.
+   * INT64 - 8 bytes per value.  Stored as little-endian.
+   * FLOAT - 4 bytes per value.  IEEE. Stored as little-endian.
+   * DOUBLE - 8 bytes per value.  IEEE. Stored as little-endian.
+   * BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes.
+   * FIXED_LEN_BYTE_ARRAY - Just the bytes.
+   */
+  PLAIN = 0;
+
+  /** Group VarInt encoding for INT32/INT64.
+   * This encoding is deprecated. It was never used
+   */
+  //  GROUP_VAR_INT = 1;
+
+  /**
+   * Deprecated: Dictionary encoding. The values in the dictionary are encoded in the
+   * plain type.
+   * in a data page use RLE_DICTIONARY instead.
+   * in a Dictionary page use PLAIN instead
+   */
+  PLAIN_DICTIONARY = 2;
+
+  /** Group packed run length encoding. Usable for definition/repetition levels
+   * encoding and Booleans (on one bit: 0 is false; 1 is true.)
+   */
+  RLE = 3;
+
+  /** Bit packed encoding.  This can only be used if the data has a known max
+   * width.  Usable for definition/repetition levels encoding.
+   */
+  BIT_PACKED = 4;
+
+  /** Delta encoding for integers. This can be used for int columns and works best
+   * on sorted data
+   */
+  DELTA_BINARY_PACKED = 5;
+
+  /** Encoding for byte arrays to separate the length values and the data. The lengths
+   * are encoded using DELTA_BINARY_PACKED
+   */
+  DELTA_LENGTH_BYTE_ARRAY = 6;
+
+  /** Incremental-encoded byte array. Prefix lengths are encoded using DELTA_BINARY_PACKED.
+   * Suffixes are stored as delta length byte arrays.
+   */
+  DELTA_BYTE_ARRAY = 7;
+
+  /** Dictionary encoding: the ids are encoded using the RLE encoding
+   */
+  RLE_DICTIONARY = 8;
+}
+
+/**
+ * Supported compression algorithms.
+ *
+ * Codecs added in 2.4 can be read by readers based on 2.4 and later.
+ * Codec support may vary between readers based on the format version and
+ * libraries available at runtime. Gzip, Snappy, and LZ4 codecs are
+ * widely available, while Zstd and Brotli require additional libraries.
+ */
+enum CompressionCodec {
+  UNCOMPRESSED = 0;
+  SNAPPY = 1;
+  GZIP = 2;
+  LZO = 3;
+  BROTLI = 4; // Added in 2.4
+  LZ4 = 5;    // Added in 2.4
+  ZSTD = 6;   // Added in 2.4
+}
+
+enum PageType {
+  DATA_PAGE = 0;
+  INDEX_PAGE = 1;
+  DICTIONARY_PAGE = 2;
+  DATA_PAGE_V2 = 3;
+}
+
+/**
+ * Enum to annotate whether lists of min/max elements inside ColumnIndex
+ * are ordered and if so, in which direction.
+ */
+enum BoundaryOrder {
+  UNORDERED = 0;
+  ASCENDING = 1;
+  DESCENDING = 2;
+}
+
+/** Data page header */
+struct DataPageHeader {
+  /** Number of values, including NULLs, in this data page. **/
+  1: required i32 num_values
+
+  /** Encoding used for this data page **/
+  2: required Encoding encoding
+
+  /** Encoding used for definition levels **/
+  3: required Encoding definition_level_encoding;
+
+  /** Encoding used for repetition levels **/
+  4: required Encoding repetition_level_encoding;
+
+  /** Optional statistics for the data in this page**/
+  5: optional Statistics statistics;
+}
+
+struct IndexPageHeader {
+  /** TODO: **/
+}
+
+struct DictionaryPageHeader {
+  /** Number of values in the dictionary **/
+  1: required i32 num_values;
+
+  /** Encoding using this dictionary page **/
+  2: required Encoding encoding
+
+  /** If true, the entries in the dictionary are sorted in ascending order **/
+  3: optional bool is_sorted;
+}
+
+/**
+ * New page format allowing reading levels without decompressing the data
+ * Repetition and definition levels are uncompressed
+ * The remaining section containing the data is compressed if is_compressed is true
+ **/
+struct DataPageHeaderV2 {
+  /** Number of values, including NULLs, in this data page. **/
+  1: required i32 num_values
+  /** Number of NULL values, in this data page.
+      Number of non-null = num_values - num_nulls which is also the number of values in the data section **/
+  2: required i32 num_nulls
+  /** Number of rows in this data page. which means pages change on record boundaries (r = 0) **/
+  3: required i32 num_rows
+  /** Encoding used for data in this page **/
+  4: required Encoding encoding
+
+  // repetition levels and definition levels are always using RLE (without size in it)
+
+  /** length of the definition levels */
+  5: required i32 definition_levels_byte_length;
+  /** length of the repetition levels */
+  6: required i32 repetition_levels_byte_length;
+
+  /**  whether the values are compressed.
+  Which means the section of the page between
+  definition_levels_byte_length + repetition_levels_byte_length + 1 and compressed_page_size (included)
+  is compressed with the compression_codec.
+  If missing it is considered compressed */
+  7: optional bool is_compressed = 1;
+
+  /** optional statistics for this column chunk */
+  8: optional Statistics statistics;
+}
+
+struct PageHeader {
+  /** the type of the page: indicates which of the *_header fields is set **/
+  1: required PageType type
+
+  /** Uncompressed page size in bytes (not including this header) **/
+  2: required i32 uncompressed_page_size
+
+  /** Compressed page size in bytes (not including this header) **/
+  3: required i32 compressed_page_size
+
+  /** 32bit crc for the data below. This allows for disabling checksumming in HDFS
+   *  if only a few pages needs to be read
+   **/
+  4: optional i32 crc
+
+  // Headers for page specific data.  One only will be set.
+  5: optional DataPageHeader data_page_header;
+  6: optional IndexPageHeader index_page_header;
+  7: optional DictionaryPageHeader dictionary_page_header;
+  8: optional DataPageHeaderV2 data_page_header_v2;
+}
+
+/**
+ * Wrapper struct to store key values
+ */
+ struct KeyValue {
+  1: required string key
+  2: optional string value
+}
+
+/**
+ * Wrapper struct to specify sort order
+ */
+struct SortingColumn {
+  /** The column index (in this row group) **/
+  1: required i32 column_idx
+
+  /** If true, indicates this column is sorted in descending order. **/
+  2: required bool descending
+
+  /** If true, nulls will come before non-null values, otherwise,
+   * nulls go at the end. */
+  3: required bool nulls_first
+}
+
+/**
+ * statistics of a given page type and encoding
+ */
+struct PageEncodingStats {
+
+  /** the page type (data/dic/...) **/
+  1: required PageType page_type;
+
+  /** encoding of the page **/
+  2: required Encoding encoding;
+
+  /** number of pages of this type with this encoding **/
+  3: required i32 count;
+
+}
+
+/**
+ * Description for column metadata
+ */
+struct ColumnMetaData {
+  /** Type of this column **/
+  1: required Type type
+
+  /** Set of all encodings used for this column. The purpose is to validate
+   * whether we can decode those pages. **/
+  2: required list<Encoding> encodings
+
+  /** Path in schema **/
+  3: required list<string> path_in_schema
+
+  /** Compression codec **/
+  4: required CompressionCodec codec
+
+  /** Number of values in this column **/
+  5: required i64 num_values
+
+  /** total byte size of all uncompressed pages in this column chunk (including the headers) **/
+  6: required i64 total_uncompressed_size
+
+  /** total byte size of all compressed pages in this column chunk (including the headers) **/
+  7: required i64 total_compressed_size
+
+  /** Optional key/value metadata **/
+  8: optional list<KeyValue> key_value_metadata
+
+  /** Byte offset from beginning of file to first data page **/
+  9: required i64 data_page_offset
+
+  /** Byte offset from beginning of file to root index page **/
+  10: optional i64 index_page_offset
+
+  /** Byte offset from the beginning of file to first (only) dictionary page **/
+  11: optional i64 dictionary_page_offset
+
+  /** optional statistics for this column chunk */
+  12: optional Statistics statistics;
+
+  /** Set of all encodings used for pages in this column chunk.
+   * This information can be used to determine if all data pages are
+   * dictionary encoded for example **/
+  13: optional list<PageEncodingStats> encoding_stats;
+}
+
+struct ColumnChunk {
+  /** File where column data is stored.  If not set, assumed to be same file as
+    * metadata.  This path is relative to the current file.
+    **/
+  1: optional string file_path
+
+  /** Byte offset in file_path to the ColumnMetaData **/
+  2: required i64 file_offset
+
+  /** Column metadata for this chunk. This is the same content as what is at
+   * file_path/file_offset.  Having it here has it replicated in the file
+   * metadata.
+   **/
+  3: optional ColumnMetaData meta_data
+
+  /** File offset of ColumnChunk's OffsetIndex **/
+  4: optional i64 offset_index_offset
+
+  /** Size of ColumnChunk's OffsetIndex, in bytes **/
+  5: optional i32 offset_index_length
+
+  /** File offset of ColumnChunk's ColumnIndex **/
+  6: optional i64 column_index_offset
+
+  /** Size of ColumnChunk's ColumnIndex, in bytes **/
+  7: optional i32 column_index_length
+}
+
+struct RowGroup {
+  /** Metadata for each column chunk in this row group.
+   * This list must have the same order as the SchemaElement list in FileMetaData.
+   **/
+  1: required list<ColumnChunk> columns
+
+  /** Total byte size of all the uncompressed column data in this row group **/
+  2: required i64 total_byte_size
+
+  /** Number of rows in this row group **/
+  3: required i64 num_rows
+
+  /** If set, specifies a sort ordering of the rows in this RowGroup.
+   * The sorting columns can be a subset of all the columns.
+   */
+  4: optional list<SortingColumn> sorting_columns
+}
+
+/** Empty struct to signal the order defined by the physical or logical type */
+struct TypeDefinedOrder {}
+
+/**
+ * Union to specify the order used for the min_value and max_value fields for a
+ * column. This union takes the role of an enhanced enum that allows rich
+ * elements (which will be needed for a collation-based ordering in the future).
+ *
+ * Possible values are:
+ * * TypeDefinedOrder - the column uses the order defined by its logical or
+ *                      physical type (if there is no logical type).
+ *
+ * If the reader does not support the value of this union, min and max stats
+ * for this column should be ignored.
+ */
+union ColumnOrder {
+
+  /**
+   * The sort orders for logical types are:
+   *   UTF8 - unsigned byte-wise comparison
+   *   INT8 - signed comparison
+   *   INT16 - signed comparison
+   *   INT32 - signed comparison
+   *   INT64 - signed comparison
+   *   UINT8 - unsigned comparison
+   *   UINT16 - unsigned comparison
+   *   UINT32 - unsigned comparison
+   *   UINT64 - unsigned comparison
+   *   DECIMAL - signed comparison of the represented value
+   *   DATE - signed comparison
+   *   TIME_MILLIS - signed comparison
+   *   TIME_MICROS - signed comparison
+   *   TIMESTAMP_MILLIS - signed comparison
+   *   TIMESTAMP_MICROS - signed comparison
+   *   INTERVAL - unsigned comparison
+   *   JSON - unsigned byte-wise comparison
+   *   BSON - unsigned byte-wise comparison
+   *   ENUM - unsigned byte-wise comparison
+   *   LIST - undefined
+   *   MAP - undefined
+   *
+   * In the absence of logical types, the sort order is determined by the physical type:
+   *   BOOLEAN - false, true
+   *   INT32 - signed comparison
+   *   INT64 - signed comparison
+   *   INT96 (only used for legacy timestamps) - undefined
+   *   FLOAT - signed comparison of the represented value (*)
+   *   DOUBLE - signed comparison of the represented value (*)
+   *   BYTE_ARRAY - unsigned byte-wise comparison
+   *   FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison
+   *
+   * (*) Because the sorting order is not specified properly for floating
+   *     point values (relations vs. total ordering) the following
+   *     compatibility rules should be applied when reading statistics:
+   *     - If the min is a NaN, it should be ignored.
+   *     - If the max is a NaN, it should be ignored.
+   *     - If the min is +0, the row group may contain -0 values as well.
+   *     - If the max is -0, the row group may contain +0 values as well.
+   *     - When looking for NaN values, min and max should be ignored.
+   */
+  1: TypeDefinedOrder TYPE_ORDER;
+}
+
+struct PageLocation {
+  /** Offset of the page in the file **/
+  1: required i64 offset
+
+  /**
+   * Size of the page, including header. Sum of compressed_page_size and header
+   * length
+   */
+  2: required i32 compressed_page_size
+
+  /**
+   * Index within the RowGroup of the first row of the page; this means pages
+   * change on record boundaries (r = 0).
+   */
+  3: required i64 first_row_index
+}
+
+struct OffsetIndex {
+  /**
+   * PageLocations, ordered by increasing PageLocation.offset. It is required
+   * that page_locations[i].first_row_index < page_locations[i+1].first_row_index.
+   */
+  1: required list<PageLocation> page_locations
+}
+
+/**
+ * Description for ColumnIndex.
+ * Each <array-field>[i] refers to the page at OffsetIndex.page_locations[i]
+ */
+struct ColumnIndex {
+  /**
+   * A list of Boolean values to determine the validity of the corresponding
+   * min and max values. If true, a page contains only null values, and writers
+   * have to set the corresponding entries in min_values and max_values to
+   * byte[0], so that all lists have the same length. If false, the
+   * corresponding entries in min_values and max_values must be valid.
+   */
+  1: required list<bool> null_pages
+
+  /**
+   * Two lists containing lower and upper bounds for the values of each page.
+   * These may be the actual minimum and maximum values found on a page, but
+   * can also be (more compact) values that do not exist on a page. For
+   * example, instead of storing ""Blart Versenwald III", a writer may set
+   * min_values[i]="B", max_values[i]="C". Such more compact values must still
+   * be valid values within the column's logical type. Readers must make sure
+   * that list entries are populated before using them by inspecting null_pages.
+   */
+  2: required list<binary> min_values
+  3: required list<binary> max_values
+
+  /**
+   * Stores whether both min_values and max_values are orderd and if so, in
+   * which direction. This allows readers to perform binary searches in both
+   * lists. Readers cannot assume that max_values[i] <= min_values[i+1], even
+   * if the lists are ordered.
+   */
+  4: required BoundaryOrder boundary_order
+
+  /** A list containing the number of null values for each page **/
+  5: optional list<i64> null_counts
+}
+
+/**
+ * Description for file metadata
+ */
+struct FileMetaData {
+  /** Version of this file **/
+  1: required i32 version
+
+  /** Parquet schema for this file.  This schema contains metadata for all the columns.
+   * The schema is represented as a tree with a single root.  The nodes of the tree
+   * are flattened to a list by doing a depth-first traversal.
+   * The column metadata contains the path in the schema for that column which can be
+   * used to map columns to nodes in the schema.
+   * The first element is the root **/
+  2: required list<SchemaElement> schema;
+
+  /** Number of rows in this file **/
+  3: required i64 num_rows
+
+  /** Row groups in this file **/
+  4: required list<RowGroup> row_groups
+
+  /** Optional key/value metadata **/
+  5: optional list<KeyValue> key_value_metadata
+
+  /** String for application that wrote this file.  This should be in the format
+   * <Application> version <App Version> (build <App Build Hash>).
+   * e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
+   **/
+  6: optional string created_by
+
+  /**
+   * Sort order used for the min_value and max_value fields of each column in
+   * this file. Each sort order corresponds to one column, determined by its
+   * position in the list, matching the position of the column in the schema.
+   *
+   * Without column_orders, the meaning of the min_value and max_value fields is
+   * undefined. To ensure well-defined behaviour, if min_value and max_value are
+   * written to a Parquet file, column_orders must be written as well.
+   *
+   * The obsolete min and max fields are always sorted by signed comparison
+   * regardless of column_orders.
+   */
+  7: optional list<ColumnOrder> column_orders;
+}
+

vendor/github.com/minio/parquet-go/table.go

@@ -0,0 +1,124 @@
+/*
+ * Minio Cloud Storage, (C) 2018 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package parquet
+
+import "github.com/minio/parquet-go/gen-go/parquet"
+
+func getTableValues(values interface{}, valueType parquet.Type) (tableValues []interface{}) {
+	switch valueType {
+	case parquet.Type_BOOLEAN:
+		for _, v := range values.([]bool) {
+			tableValues = append(tableValues, v)
+		}
+	case parquet.Type_INT32:
+		for _, v := range values.([]int32) {
+			tableValues = append(tableValues, v)
+		}
+	case parquet.Type_INT64:
+		for _, v := range values.([]int64) {
+			tableValues = append(tableValues, v)
+		}
+	case parquet.Type_FLOAT:
+		for _, v := range values.([]float32) {
+			tableValues = append(tableValues, v)
+		}
+	case parquet.Type_DOUBLE:
+		for _, v := range values.([]float64) {
+			tableValues = append(tableValues, v)
+		}
+	case parquet.Type_INT96, parquet.Type_BYTE_ARRAY, parquet.Type_FIXED_LEN_BYTE_ARRAY:
+		for _, v := range values.([][]byte) {
+			tableValues = append(tableValues, v)
+		}
+	}
+
+	return tableValues
+}
+
+type table struct {
+	RepetitionType     parquet.FieldRepetitionType
+	Type               parquet.Type
+	MaxDefinitionLevel int32
+	MaxRepetitionLevel int32
+	Path               []string      // Path of this column
+	Values             []interface{} // Parquet values
+	DefinitionLevels   []int32       // Definition Levels slice
+	RepetitionLevels   []int32       // Repetition Levels slice
+}
+
+func newTableFromTable(srcTable *table) *table {
+	if srcTable == nil {
+		return nil
+	}
+
+	return &table{
+		Type: srcTable.Type,
+		Path: append([]string{}, srcTable.Path...),
+	}
+}
+
+func (table *table) Merge(tables ...*table) {
+	for i := 0; i < len(tables); i++ {
+		if tables[i] == nil {
+			continue
+		}
+
+		table.Values = append(table.Values, tables[i].Values...)
+		table.RepetitionLevels = append(table.RepetitionLevels, tables[i].RepetitionLevels...)
+		table.DefinitionLevels = append(table.DefinitionLevels, tables[i].DefinitionLevels...)
+
+		if table.MaxDefinitionLevel < tables[i].MaxDefinitionLevel {
+			table.MaxDefinitionLevel = tables[i].MaxDefinitionLevel
+		}
+
+		if table.MaxRepetitionLevel < tables[i].MaxRepetitionLevel {
+			table.MaxRepetitionLevel = tables[i].MaxRepetitionLevel
+		}
+	}
+}
+
+func (table *table) Pop(numRows int64) *table {
+	result := newTableFromTable(table)
+	var i, num int64
+	for i = int64(0); i < int64(len(table.Values)); i++ {
+		if table.RepetitionLevels[i] == 0 {
+			if num >= numRows {
+				break
+			}
+
+			num++
+		}
+
+		if result.MaxRepetitionLevel < table.RepetitionLevels[i] {
+			result.MaxRepetitionLevel = table.RepetitionLevels[i]
+		}
+
+		if result.MaxDefinitionLevel < table.DefinitionLevels[i] {
+			result.MaxDefinitionLevel = table.DefinitionLevels[i]
+		}
+	}
+
+	result.RepetitionLevels = table.RepetitionLevels[:i]
+	result.DefinitionLevels = table.DefinitionLevels[:i]
+	result.Values = table.Values[:i]
+
+	table.RepetitionLevels = table.RepetitionLevels[i:]
+	table.DefinitionLevels = table.DefinitionLevels[i:]
+	table.Values = table.Values[i:]
+
+	return result
+}

vendor/github.com/pierrec/lz4/LICENSE

@@ -0,0 +1,28 @@
+Copyright (c) 2015, Pierre Curto
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of xxHash nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+

vendor/github.com/pierrec/lz4/README.md

@@ -0,0 +1,24 @@
+[![godoc](https://godoc.org/github.com/pierrec/lz4?status.png)](https://godoc.org/github.com/pierrec/lz4)
+
+# lz4
+LZ4 compression and decompression in pure Go.
+
+## Usage
+
+```go
+import "github.com/pierrec/lz4/v2"
+```
+
+## Description
+Package lz4 implements reading and writing lz4 compressed data (a frame),
+as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html.
+
+This package is **compatible with the LZ4 frame format** although the block level compression 
+and decompression functions are exposed and are fully compatible with the lz4 block format 
+definition, they are low level and should not be used directly.
+
+For a complete description of an lz4 compressed block, see:
+http://fastcompression.blogspot.fr/2011/05/lz4-explained.html
+
+See https://github.com/Cyan4973/lz4 for the reference C implementation.
+

vendor/github.com/pierrec/lz4/block.go

@@ -0,0 +1,397 @@
+package lz4
+
+import (
+	"encoding/binary"
+	"errors"
+)
+
+var (
+	// ErrInvalidSourceShortBuffer is returned by UncompressBlock or CompressBLock when a compressed
+	// block is corrupted or the destination buffer is not large enough for the uncompressed data.
+	ErrInvalidSourceShortBuffer = errors.New("lz4: invalid source or destination buffer too short")
+	// ErrInvalid is returned when reading an invalid LZ4 archive.
+	ErrInvalid = errors.New("lz4: bad magic number")
+)
+
+// blockHash hashes 4 bytes into a value < winSize.
+func blockHash(x uint32) uint32 {
+	const hasher uint32 = 2654435761 // Knuth multiplicative hash.
+	return x * hasher >> hashShift
+}
+
+// CompressBlockBound returns the maximum size of a given buffer of size n, when not compressible.
+func CompressBlockBound(n int) int {
+	return n + n/255 + 16
+}
+
+// UncompressBlock uncompresses the source buffer into the destination one,
+// and returns the uncompressed size.
+//
+// The destination buffer must be sized appropriately.
+//
+// An error is returned if the source data is invalid or the destination buffer is too small.
+func UncompressBlock(src, dst []byte) (si int, err error) {
+	defer func() {
+		// It is now faster to let the runtime panic and recover on out of bound slice access
+		// than checking indices as we go along.
+		if recover() != nil {
+			err = ErrInvalidSourceShortBuffer
+		}
+	}()
+	sn := len(src)
+	if sn == 0 {
+		return 0, nil
+	}
+	var di int
+
+	for {
+		// Literals and match lengths (token).
+		b := int(src[si])
+		si++
+
+		// Literals.
+		if lLen := b >> 4; lLen > 0 {
+			if lLen == 0xF {
+				for src[si] == 0xFF {
+					lLen += 0xFF
+					si++
+				}
+				lLen += int(src[si])
+				si++
+			}
+			i := si
+			si += lLen
+			di += copy(dst[di:], src[i:si])
+
+			if si >= sn {
+				return di, nil
+			}
+		}
+
+		si++
+		_ = src[si] // Bound check elimination.
+		offset := int(src[si-1]) | int(src[si])<<8
+		si++
+
+		// Match.
+		mLen := b & 0xF
+		if mLen == 0xF {
+			for src[si] == 0xFF {
+				mLen += 0xFF
+				si++
+			}
+			mLen += int(src[si])
+			si++
+		}
+		mLen += minMatch
+
+		// Copy the match.
+		i := di - offset
+		if offset > 0 && mLen >= offset {
+			// Efficiently copy the match dst[di-offset:di] into the dst slice.
+			bytesToCopy := offset * (mLen / offset)
+			expanded := dst[i:]
+			for n := offset; n <= bytesToCopy+offset; n *= 2 {
+				copy(expanded[n:], expanded[:n])
+			}
+			di += bytesToCopy
+			mLen -= bytesToCopy
+		}
+		di += copy(dst[di:], dst[i:i+mLen])
+	}
+}
+
+// CompressBlock compresses the source buffer into the destination one.
+// This is the fast version of LZ4 compression and also the default one.
+// The size of hashTable must be at least 64Kb.
+//
+// The size of the compressed data is returned. If it is 0 and no error, then the data is incompressible.
+//
+// An error is returned if the destination buffer is too small.
+func CompressBlock(src, dst []byte, hashTable []int) (di int, err error) {
+	defer func() {
+		if recover() != nil {
+			err = ErrInvalidSourceShortBuffer
+		}
+	}()
+
+	sn, dn := len(src)-mfLimit, len(dst)
+	if sn <= 0 || dn == 0 {
+		return 0, nil
+	}
+	var si int
+
+	// Fast scan strategy: the hash table only stores the last 4 bytes sequences.
+	// const accInit = 1 << skipStrength
+
+	anchor := si // Position of the current literals.
+	// acc := accInit // Variable step: improves performance on non-compressible data.
+
+	for si < sn {
+		// Hash the next 4 bytes (sequence)...
+		match := binary.LittleEndian.Uint32(src[si:])
+		h := blockHash(match)
+
+		ref := hashTable[h]
+		hashTable[h] = si
+		if ref >= sn { // Invalid reference (dirty hashtable).
+			si++
+			continue
+		}
+		offset := si - ref
+		if offset <= 0 || offset >= winSize || // Out of window.
+			match != binary.LittleEndian.Uint32(src[ref:]) { // Hash collision on different matches.
+			// si += acc >> skipStrength
+			// acc++
+			si++
+			continue
+		}
+
+		// Match found.
+		// acc = accInit
+		lLen := si - anchor // Literal length.
+
+		// Encode match length part 1.
+		si += minMatch
+		mLen := si // Match length has minMatch already.
+		// Find the longest match, first looking by batches of 8 bytes.
+		for si < sn && binary.LittleEndian.Uint64(src[si:]) == binary.LittleEndian.Uint64(src[si-offset:]) {
+			si += 8
+		}
+		// Then byte by byte.
+		for si < sn && src[si] == src[si-offset] {
+			si++
+		}
+
+		mLen = si - mLen
+		if mLen < 0xF {
+			dst[di] = byte(mLen)
+		} else {
+			dst[di] = 0xF
+		}
+
+		// Encode literals length.
+		if lLen < 0xF {
+			dst[di] |= byte(lLen << 4)
+		} else {
+			dst[di] |= 0xF0
+			di++
+			l := lLen - 0xF
+			for ; l >= 0xFF; l -= 0xFF {
+				dst[di] = 0xFF
+				di++
+			}
+			dst[di] = byte(l)
+		}
+		di++
+
+		// Literals.
+		copy(dst[di:], src[anchor:anchor+lLen])
+		di += lLen + 2
+		anchor = si
+
+		// Encode offset.
+		_ = dst[di] // Bound check elimination.
+		dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)
+
+		// Encode match length part 2.
+		if mLen >= 0xF {
+			for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
+				dst[di] = 0xFF
+				di++
+			}
+			dst[di] = byte(mLen)
+			di++
+		}
+	}
+
+	if anchor == 0 {
+		// Incompressible.
+		return 0, nil
+	}
+
+	// Last literals.
+	lLen := len(src) - anchor
+	if lLen < 0xF {
+		dst[di] = byte(lLen << 4)
+	} else {
+		dst[di] = 0xF0
+		di++
+		for lLen -= 0xF; lLen >= 0xFF; lLen -= 0xFF {
+			dst[di] = 0xFF
+			di++
+		}
+		dst[di] = byte(lLen)
+	}
+	di++
+
+	// Write the last literals.
+	if di >= anchor {
+		// Incompressible.
+		return 0, nil
+	}
+	di += copy(dst[di:], src[anchor:])
+	return di, nil
+}
+
+// CompressBlockHC compresses the source buffer src into the destination dst
+// with max search depth (use 0 or negative value for no max).
+//
+// CompressBlockHC compression ratio is better than CompressBlock but it is also slower.
+//
+// The size of the compressed data is returned. If it is 0 and no error, then the data is not compressible.
+//
+// An error is returned if the destination buffer is too small.
+func CompressBlockHC(src, dst []byte, depth int) (di int, err error) {
+	defer func() {
+		if recover() != nil {
+			err = ErrInvalidSourceShortBuffer
+		}
+	}()
+
+	sn, dn := len(src)-mfLimit, len(dst)
+	if sn <= 0 || dn == 0 {
+		return 0, nil
+	}
+	var si int
+
+	// hashTable: stores the last position found for a given hash
+	// chaingTable: stores previous positions for a given hash
+	var hashTable, chainTable [winSize]int
+
+	if depth <= 0 {
+		depth = winSize
+	}
+
+	anchor := si
+	for si < sn {
+		// Hash the next 4 bytes (sequence).
+		match := binary.LittleEndian.Uint32(src[si:])
+		h := blockHash(match)
+
+		// Follow the chain until out of window and give the longest match.
+		mLen := 0
+		offset := 0
+		for next, try := hashTable[h], depth; try > 0 && next > 0 && si-next < winSize; next = chainTable[next&winMask] {
+			// The first (mLen==0) or next byte (mLen>=minMatch) at current match length
+			// must match to improve on the match length.
+			if src[next+mLen] != src[si+mLen] {
+				continue
+			}
+			ml := 0
+			// Compare the current position with a previous with the same hash.
+			for ml < sn-si && binary.LittleEndian.Uint64(src[next+ml:]) == binary.LittleEndian.Uint64(src[si+ml:]) {
+				ml += 8
+			}
+			for ml < sn-si && src[next+ml] == src[si+ml] {
+				ml++
+			}
+			if ml+1 < minMatch || ml <= mLen {
+				// Match too small (<minMath) or smaller than the current match.
+				continue
+			}
+			// Found a longer match, keep its position and length.
+			mLen = ml
+			offset = si - next
+			// Try another previous position with the same hash.
+			try--
+		}
+		chainTable[si&winMask] = hashTable[h]
+		hashTable[h] = si
+
+		// No match found.
+		if mLen == 0 {
+			si++
+			continue
+		}
+
+		// Match found.
+		// Update hash/chain tables with overlapping bytes:
+		// si already hashed, add everything from si+1 up to the match length.
+		winStart := si + 1
+		if ws := si + mLen - winSize; ws > winStart {
+			winStart = ws
+		}
+		for si, ml := winStart, si+mLen; si < ml; {
+			match >>= 8
+			match |= uint32(src[si+3]) << 24
+			h := blockHash(match)
+			chainTable[si&winMask] = hashTable[h]
+			hashTable[h] = si
+			si++
+		}
+
+		lLen := si - anchor
+		si += mLen
+		mLen -= minMatch // Match length does not include minMatch.
+
+		if mLen < 0xF {
+			dst[di] = byte(mLen)
+		} else {
+			dst[di] = 0xF
+		}
+
+		// Encode literals length.
+		if lLen < 0xF {
+			dst[di] |= byte(lLen << 4)
+		} else {
+			dst[di] |= 0xF0
+			di++
+			l := lLen - 0xF
+			for ; l >= 0xFF; l -= 0xFF {
+				dst[di] = 0xFF
+				di++
+			}
+			dst[di] = byte(l)
+		}
+		di++
+
+		// Literals.
+		copy(dst[di:], src[anchor:anchor+lLen])
+		di += lLen
+		anchor = si
+
+		// Encode offset.
+		di += 2
+		dst[di-2], dst[di-1] = byte(offset), byte(offset>>8)
+
+		// Encode match length part 2.
+		if mLen >= 0xF {
+			for mLen -= 0xF; mLen >= 0xFF; mLen -= 0xFF {
+				dst[di] = 0xFF
+				di++
+			}
+			dst[di] = byte(mLen)
+			di++
+		}
+	}
+
+	if anchor == 0 {
+		// Incompressible.
+		return 0, nil
+	}
+
+	// Last literals.
+	lLen := len(src) - anchor
+	if lLen < 0xF {
+		dst[di] = byte(lLen << 4)
+	} else {
+		dst[di] = 0xF0
+		di++
+		lLen -= 0xF
+		for ; lLen >= 0xFF; lLen -= 0xFF {
+			dst[di] = 0xFF
+			di++
+		}
+		dst[di] = byte(lLen)
+	}
+	di++
+
+	// Write the last literals.
+	if di >= anchor {
+		// Incompressible.
+		return 0, nil
+	}
+	di += copy(dst[di:], src[anchor:])
+	return di, nil
+}

vendor/github.com/pierrec/lz4/debug.go

@@ -0,0 +1,23 @@
+// +build lz4debug
+
+package lz4
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+)
+
+const debugFlag = true
+
+func debug(args ...interface{}) {
+	_, file, line, _ := runtime.Caller(1)
+	file = filepath.Base(file)
+
+	f := fmt.Sprintf("LZ4: %s:%d %s", file, line, args[0])
+	if f[len(f)-1] != '\n' {
+		f += "\n"
+	}
+	fmt.Fprintf(os.Stderr, f, args[1:]...)
+}

vendor/github.com/pierrec/lz4/debug_stub.go

@@ -0,0 +1,7 @@
+// +build !lz4debug
+
+package lz4
+
+const debugFlag = false
+
+func debug(args ...interface{}) {}

vendor/github.com/pierrec/lz4/go.mod

@@ -0,0 +1,3 @@
+module github.com/pierrec/lz4
+
+require github.com/pkg/profile v1.2.1

vendor/github.com/pierrec/lz4/go.sum

@@ -0,0 +1,2 @@
+github.com/pkg/profile v1.2.1 h1:F++O52m40owAmADcojzM+9gyjmMOY/T4oYJkgFDH8RE=
+github.com/pkg/profile v1.2.1/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA=

vendor/github.com/pierrec/lz4/internal/xxh32/xxh32zero.go

@@ -0,0 +1,222 @@
+// Package xxh32 implements the very fast XXH hashing algorithm (32 bits version).
+// (https://github.com/Cyan4973/XXH/)
+package xxh32
+
+import (
+	"encoding/binary"
+)
+
+const (
+	prime32_1 uint32 = 2654435761
+	prime32_2 uint32 = 2246822519
+	prime32_3 uint32 = 3266489917
+	prime32_4 uint32 = 668265263
+	prime32_5 uint32 = 374761393
+
+	prime32_1plus2 uint32 = 606290984
+	prime32_minus1 uint32 = 1640531535
+)
+
+// XXHZero represents an xxhash32 object with seed 0.
+type XXHZero struct {
+	v1       uint32
+	v2       uint32
+	v3       uint32
+	v4       uint32
+	totalLen uint64
+	buf      [16]byte
+	bufused  int
+}
+
+// Sum appends the current hash to b and returns the resulting slice.
+// It does not change the underlying hash state.
+func (xxh XXHZero) Sum(b []byte) []byte {
+	h32 := xxh.Sum32()
+	return append(b, byte(h32), byte(h32>>8), byte(h32>>16), byte(h32>>24))
+}
+
+// Reset resets the Hash to its initial state.
+func (xxh *XXHZero) Reset() {
+	xxh.v1 = prime32_1plus2
+	xxh.v2 = prime32_2
+	xxh.v3 = 0
+	xxh.v4 = prime32_minus1
+	xxh.totalLen = 0
+	xxh.bufused = 0
+}
+
+// Size returns the number of bytes returned by Sum().
+func (xxh *XXHZero) Size() int {
+	return 4
+}
+
+// BlockSize gives the minimum number of bytes accepted by Write().
+func (xxh *XXHZero) BlockSize() int {
+	return 1
+}
+
+// Write adds input bytes to the Hash.
+// It never returns an error.
+func (xxh *XXHZero) Write(input []byte) (int, error) {
+	if xxh.totalLen == 0 {
+		xxh.Reset()
+	}
+	n := len(input)
+	m := xxh.bufused
+
+	xxh.totalLen += uint64(n)
+
+	r := len(xxh.buf) - m
+	if n < r {
+		copy(xxh.buf[m:], input)
+		xxh.bufused += len(input)
+		return n, nil
+	}
+
+	p := 0
+	// Causes compiler to work directly from registers instead of stack:
+	v1, v2, v3, v4 := xxh.v1, xxh.v2, xxh.v3, xxh.v4
+	if m > 0 {
+		// some data left from previous update
+		copy(xxh.buf[xxh.bufused:], input[:r])
+		xxh.bufused += len(input) - r
+
+		// fast rotl(13)
+		buf := xxh.buf[:16] // BCE hint.
+		v1 = rol13(v1+binary.LittleEndian.Uint32(buf[:])*prime32_2) * prime32_1
+		v2 = rol13(v2+binary.LittleEndian.Uint32(buf[4:])*prime32_2) * prime32_1
+		v3 = rol13(v3+binary.LittleEndian.Uint32(buf[8:])*prime32_2) * prime32_1
+		v4 = rol13(v4+binary.LittleEndian.Uint32(buf[12:])*prime32_2) * prime32_1
+		p = r
+		xxh.bufused = 0
+	}
+
+	for n := n - 16; p <= n; p += 16 {
+		sub := input[p:][:16] //BCE hint for compiler
+		v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime32_2) * prime32_1
+		v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime32_2) * prime32_1
+		v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime32_2) * prime32_1
+		v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime32_2) * prime32_1
+	}
+	xxh.v1, xxh.v2, xxh.v3, xxh.v4 = v1, v2, v3, v4
+
+	copy(xxh.buf[xxh.bufused:], input[p:])
+	xxh.bufused += len(input) - p
+
+	return n, nil
+}
+
+// Sum32 returns the 32 bits Hash value.
+func (xxh *XXHZero) Sum32() uint32 {
+	h32 := uint32(xxh.totalLen)
+	if h32 >= 16 {
+		h32 += rol1(xxh.v1) + rol7(xxh.v2) + rol12(xxh.v3) + rol18(xxh.v4)
+	} else {
+		h32 += prime32_5
+	}
+
+	p := 0
+	n := xxh.bufused
+	buf := xxh.buf
+	for n := n - 4; p <= n; p += 4 {
+		h32 += binary.LittleEndian.Uint32(buf[p:p+4]) * prime32_3
+		h32 = rol17(h32) * prime32_4
+	}
+	for ; p < n; p++ {
+		h32 += uint32(buf[p]) * prime32_5
+		h32 = rol11(h32) * prime32_1
+	}
+
+	h32 ^= h32 >> 15
+	h32 *= prime32_2
+	h32 ^= h32 >> 13
+	h32 *= prime32_3
+	h32 ^= h32 >> 16
+
+	return h32
+}
+
+// ChecksumZero returns the 32bits Hash value.
+func ChecksumZero(input []byte) uint32 {
+	n := len(input)
+	h32 := uint32(n)
+
+	if n < 16 {
+		h32 += prime32_5
+	} else {
+		v1 := prime32_1plus2
+		v2 := prime32_2
+		v3 := uint32(0)
+		v4 := prime32_minus1
+		p := 0
+		for n := n - 16; p <= n; p += 16 {
+			sub := input[p:][:16] //BCE hint for compiler
+			v1 = rol13(v1+binary.LittleEndian.Uint32(sub[:])*prime32_2) * prime32_1
+			v2 = rol13(v2+binary.LittleEndian.Uint32(sub[4:])*prime32_2) * prime32_1
+			v3 = rol13(v3+binary.LittleEndian.Uint32(sub[8:])*prime32_2) * prime32_1
+			v4 = rol13(v4+binary.LittleEndian.Uint32(sub[12:])*prime32_2) * prime32_1
+		}
+		input = input[p:]
+		n -= p
+		h32 += rol1(v1) + rol7(v2) + rol12(v3) + rol18(v4)
+	}
+
+	p := 0
+	for n := n - 4; p <= n; p += 4 {
+		h32 += binary.LittleEndian.Uint32(input[p:p+4]) * prime32_3
+		h32 = rol17(h32) * prime32_4
+	}
+	for p < n {
+		h32 += uint32(input[p]) * prime32_5
+		h32 = rol11(h32) * prime32_1
+		p++
+	}
+
+	h32 ^= h32 >> 15
+	h32 *= prime32_2
+	h32 ^= h32 >> 13
+	h32 *= prime32_3
+	h32 ^= h32 >> 16
+
+	return h32
+}
+
+// Uint32Zero hashes x with seed 0.
+func Uint32Zero(x uint32) uint32 {
+	h := prime32_5 + 4 + x*prime32_3
+	h = rol17(h) * prime32_4
+	h ^= h >> 15
+	h *= prime32_2
+	h ^= h >> 13
+	h *= prime32_3
+	h ^= h >> 16
+	return h
+}
+
+func rol1(u uint32) uint32 {
+	return u<<1 | u>>31
+}
+
+func rol7(u uint32) uint32 {
+	return u<<7 | u>>25
+}
+
+func rol11(u uint32) uint32 {
+	return u<<11 | u>>21
+}
+
+func rol12(u uint32) uint32 {
+	return u<<12 | u>>20
+}
+
+func rol13(u uint32) uint32 {
+	return u<<13 | u>>19
+}
+
+func rol17(u uint32) uint32 {
+	return u<<17 | u>>15
+}
+
+func rol18(u uint32) uint32 {
+	return u<<18 | u>>14
+}

vendor/github.com/pierrec/lz4/lz4.go

@@ -0,0 +1,68 @@
+// Package lz4 implements reading and writing lz4 compressed data (a frame),
+// as specified in http://fastcompression.blogspot.fr/2013/04/lz4-streaming-format-final.html.
+//
+// Although the block level compression and decompression functions are exposed and are fully compatible
+// with the lz4 block format definition, they are low level and should not be used directly.
+// For a complete description of an lz4 compressed block, see:
+// http://fastcompression.blogspot.fr/2011/05/lz4-explained.html
+//
+// See https://github.com/Cyan4973/lz4 for the reference C implementation.
+//
+package lz4
+
+const (
+	// Extension is the LZ4 frame file name extension
+	Extension = ".lz4"
+	// Version is the LZ4 frame format version
+	Version = 1
+
+	frameMagic     uint32 = 0x184D2204
+	frameSkipMagic uint32 = 0x184D2A50
+
+	// The following constants are used to setup the compression algorithm.
+	minMatch            = 4  // the minimum size of the match sequence size (4 bytes)
+	winSizeLog          = 16 // LZ4 64Kb window size limit
+	winSize             = 1 << winSizeLog
+	winMask             = winSize - 1 // 64Kb window of previous data for dependent blocks
+	compressedBlockFlag = 1 << 31
+	compressedBlockMask = compressedBlockFlag - 1
+
+	// hashLog determines the size of the hash table used to quickly find a previous match position.
+	// Its value influences the compression speed and memory usage, the lower the faster,
+	// but at the expense of the compression ratio.
+	// 16 seems to be the best compromise.
+	hashLog       = 16
+	hashTableSize = 1 << hashLog
+	hashShift     = uint((minMatch * 8) - hashLog)
+
+	mfLimit      = 8 + minMatch // The last match cannot start within the last 12 bytes.
+	skipStrength = 6            // variable step for fast scan
+)
+
+// map the block max size id with its value in bytes: 64Kb, 256Kb, 1Mb and 4Mb.
+var (
+	bsMapID    = map[byte]int{4: 64 << 10, 5: 256 << 10, 6: 1 << 20, 7: 4 << 20}
+	bsMapValue = make(map[int]byte, len(bsMapID))
+)
+
+// Reversed.
+func init() {
+	for i, v := range bsMapID {
+		bsMapValue[v] = i
+	}
+}
+
+// Header describes the various flags that can be set on a Writer or obtained from a Reader.
+// The default values match those of the LZ4 frame format definition
+// (http://fastcompression.blogspot.com/2013/04/lz4-streaming-format-final.html).
+//
+// NB. in a Reader, in case of concatenated frames, the Header values may change between Read() calls.
+// It is the caller responsibility to check them if necessary.
+type Header struct {
+	BlockChecksum    bool   // Compressed blocks checksum flag.
+	NoChecksum       bool   // Frame checksum flag.
+	BlockMaxSize     int    // Size of the uncompressed data block (one of [64KB, 256KB, 1MB, 4MB]). Default=4MB.
+	Size             uint64 // Frame total size. It is _not_ computed by the Writer.
+	CompressionLevel int    // Compression level (higher is better, use 0 for fastest compression).
+	done             bool   // Header processed flag (Read or Write and checked).
+}

vendor/github.com/pierrec/lz4/lz4_go1.10.go

@@ -0,0 +1,29 @@
+//+build go1.10
+
+package lz4
+
+import (
+	"fmt"
+	"strings"
+)
+
+func (h Header) String() string {
+	var s strings.Builder
+
+	s.WriteString(fmt.Sprintf("%T{", h))
+	if h.BlockChecksum {
+		s.WriteString("BlockChecksum: true ")
+	}
+	if h.NoChecksum {
+		s.WriteString("NoChecksum: true ")
+	}
+	if bs := h.BlockMaxSize; bs != 0 && bs != 4<<20 {
+		s.WriteString(fmt.Sprintf("BlockMaxSize: %d ", bs))
+	}
+	if l := h.CompressionLevel; l != 0 {
+		s.WriteString(fmt.Sprintf("CompressionLevel: %d ", l))
+	}
+	s.WriteByte('}')
+
+	return s.String()
+}

vendor/github.com/pierrec/lz4/lz4_notgo1.10.go

@@ -0,0 +1,29 @@
+//+build !go1.10
+
+package lz4
+
+import (
+	"bytes"
+	"fmt"
+)
+
+func (h Header) String() string {
+	var s bytes.Buffer
+
+	s.WriteString(fmt.Sprintf("%T{", h))
+	if h.BlockChecksum {
+		s.WriteString("BlockChecksum: true ")
+	}
+	if h.NoChecksum {
+		s.WriteString("NoChecksum: true ")
+	}
+	if bs := h.BlockMaxSize; bs != 0 && bs != 4<<20 {
+		s.WriteString(fmt.Sprintf("BlockMaxSize: %d ", bs))
+	}
+	if l := h.CompressionLevel; l != 0 {
+		s.WriteString(fmt.Sprintf("CompressionLevel: %d ", l))
+	}
+	s.WriteByte('}')
+
+	return s.String()
+}

vendor/github.com/pierrec/lz4/reader.go

@@ -0,0 +1,295 @@
+package lz4
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/pierrec/lz4/internal/xxh32"
+)
+
+// Reader implements the LZ4 frame decoder.
+// The Header is set after the first call to Read().
+// The Header may change between Read() calls in case of concatenated frames.
+type Reader struct {
+	Header
+
+	buf      [8]byte       // Scrap buffer.
+	pos      int64         // Current position in src.
+	src      io.Reader     // Source.
+	zdata    []byte        // Compressed data.
+	data     []byte        // Uncompressed data.
+	idx      int           // Index of unread bytes into data.
+	checksum xxh32.XXHZero // Frame hash.
+}
+
+// NewReader returns a new LZ4 frame decoder.
+// No access to the underlying io.Reader is performed.
+func NewReader(src io.Reader) *Reader {
+	r := &Reader{src: src}
+	return r
+}
+
+// readHeader checks the frame magic number and parses the frame descriptoz.
+// Skippable frames are supported even as a first frame although the LZ4
+// specifications recommends skippable frames not to be used as first frames.
+func (z *Reader) readHeader(first bool) error {
+	defer z.checksum.Reset()
+
+	buf := z.buf[:]
+	for {
+		magic, err := z.readUint32()
+		if err != nil {
+			z.pos += 4
+			if !first && err == io.ErrUnexpectedEOF {
+				return io.EOF
+			}
+			return err
+		}
+		if magic == frameMagic {
+			break
+		}
+		if magic>>8 != frameSkipMagic>>8 {
+			return ErrInvalid
+		}
+		skipSize, err := z.readUint32()
+		if err != nil {
+			return err
+		}
+		z.pos += 4
+		m, err := io.CopyN(ioutil.Discard, z.src, int64(skipSize))
+		if err != nil {
+			return err
+		}
+		z.pos += m
+	}
+
+	// Header.
+	if _, err := io.ReadFull(z.src, buf[:2]); err != nil {
+		return err
+	}
+	z.pos += 8
+
+	b := buf[0]
+	if v := b >> 6; v != Version {
+		return fmt.Errorf("lz4: invalid version: got %d; expected %d", v, Version)
+	}
+	if b>>5&1 == 0 {
+		return fmt.Errorf("lz4: block dependency not supported")
+	}
+	z.BlockChecksum = b>>4&1 > 0
+	frameSize := b>>3&1 > 0
+	z.NoChecksum = b>>2&1 == 0
+
+	bmsID := buf[1] >> 4 & 0x7
+	bSize, ok := bsMapID[bmsID]
+	if !ok {
+		return fmt.Errorf("lz4: invalid block max size ID: %d", bmsID)
+	}
+	z.BlockMaxSize = bSize
+
+	// Allocate the compressed/uncompressed buffers.
+	// The compressed buffer cannot exceed the uncompressed one.
+	if n := 2 * bSize; cap(z.zdata) < n {
+		z.zdata = make([]byte, n, n)
+	}
+	if debugFlag {
+		debug("header block max size id=%d size=%d", bmsID, bSize)
+	}
+	z.zdata = z.zdata[:bSize]
+	z.data = z.zdata[:cap(z.zdata)][bSize:]
+	z.idx = len(z.data)
+
+	z.checksum.Write(buf[0:2])
+
+	if frameSize {
+		buf := buf[:8]
+		if _, err := io.ReadFull(z.src, buf); err != nil {
+			return err
+		}
+		z.Size = binary.LittleEndian.Uint64(buf)
+		z.pos += 8
+		z.checksum.Write(buf)
+	}
+
+	// Header checksum.
+	if _, err := io.ReadFull(z.src, buf[:1]); err != nil {
+		return err
+	}
+	z.pos++
+	if h := byte(z.checksum.Sum32() >> 8 & 0xFF); h != buf[0] {
+		return fmt.Errorf("lz4: invalid header checksum: got %x; expected %x", buf[0], h)
+	}
+
+	z.Header.done = true
+	if debugFlag {
+		debug("header read: %v", z.Header)
+	}
+
+	return nil
+}
+
+// Read decompresses data from the underlying source into the supplied buffer.
+//
+// Since there can be multiple streams concatenated, Header values may
+// change between calls to Read(). If that is the case, no data is actually read from
+// the underlying io.Reader, to allow for potential input buffer resizing.
+func (z *Reader) Read(buf []byte) (int, error) {
+	if debugFlag {
+		debug("Read buf len=%d", len(buf))
+	}
+	if !z.Header.done {
+		if err := z.readHeader(true); err != nil {
+			return 0, err
+		}
+		if debugFlag {
+			debug("header read OK compressed buffer %d / %d uncompressed buffer %d : %d index=%d",
+				len(z.zdata), cap(z.zdata), len(z.data), cap(z.data), z.idx)
+		}
+	}
+
+	if len(buf) == 0 {
+		return 0, nil
+	}
+
+	if z.idx == len(z.data) {
+		// No data ready for reading, process the next block.
+		if debugFlag {
+			debug("reading block from writer")
+		}
+		// Block length: 0 = end of frame, highest bit set: uncompressed.
+		bLen, err := z.readUint32()
+		if err != nil {
+			return 0, err
+		}
+		z.pos += 4
+
+		if bLen == 0 {
+			// End of frame reached.
+			if !z.NoChecksum {
+				// Validate the frame checksum.
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				if debugFlag {
+					debug("frame checksum got=%x / want=%x", z.checksum.Sum32(), checksum)
+				}
+				z.pos += 4
+				if h := z.checksum.Sum32(); checksum != h {
+					return 0, fmt.Errorf("lz4: invalid frame checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+			// Get ready for the next concatenated frame and keep the position.
+			pos := z.pos
+			z.Reset(z.src)
+			z.pos = pos
+
+			// Since multiple frames can be concatenated, check for more.
+			return 0, z.readHeader(false)
+		}
+
+		if debugFlag {
+			debug("raw block size %d", bLen)
+		}
+		if bLen&compressedBlockFlag > 0 {
+			// Uncompressed block.
+			bLen &= compressedBlockMask
+			if debugFlag {
+				debug("uncompressed block size %d", bLen)
+			}
+			if int(bLen) > cap(z.data) {
+				return 0, fmt.Errorf("lz4: invalid block size: %d", bLen)
+			}
+			z.data = z.data[:bLen]
+			if _, err := io.ReadFull(z.src, z.data); err != nil {
+				return 0, err
+			}
+			z.pos += int64(bLen)
+
+			if z.BlockChecksum {
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				z.pos += 4
+
+				if h := xxh32.ChecksumZero(z.data); h != checksum {
+					return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+		} else {
+			// Compressed block.
+			if debugFlag {
+				debug("compressed block size %d", bLen)
+			}
+			if int(bLen) > cap(z.data) {
+				return 0, fmt.Errorf("lz4: invalid block size: %d", bLen)
+			}
+			zdata := z.zdata[:bLen]
+			if _, err := io.ReadFull(z.src, zdata); err != nil {
+				return 0, err
+			}
+			z.pos += int64(bLen)
+
+			if z.BlockChecksum {
+				checksum, err := z.readUint32()
+				if err != nil {
+					return 0, err
+				}
+				z.pos += 4
+
+				if h := xxh32.ChecksumZero(zdata); h != checksum {
+					return 0, fmt.Errorf("lz4: invalid block checksum: got %x; expected %x", h, checksum)
+				}
+			}
+
+			n, err := UncompressBlock(zdata, z.data)
+			if err != nil {
+				return 0, err
+			}
+			z.data = z.data[:n]
+		}
+
+		if !z.NoChecksum {
+			z.checksum.Write(z.data)
+			if debugFlag {
+				debug("current frame checksum %x", z.checksum.Sum32())
+			}
+		}
+		z.idx = 0
+	}
+
+	n := copy(buf, z.data[z.idx:])
+	z.idx += n
+	if debugFlag {
+		debug("copied %d bytes to input", n)
+	}
+
+	return n, nil
+}
+
+// Reset discards the Reader's state and makes it equivalent to the
+// result of its original state from NewReader, but reading from r instead.
+// This permits reusing a Reader rather than allocating a new one.
+func (z *Reader) Reset(r io.Reader) {
+	z.Header = Header{}
+	z.pos = 0
+	z.src = r
+	z.zdata = z.zdata[:0]
+	z.data = z.data[:0]
+	z.idx = 0
+	z.checksum.Reset()
+}
+
+// readUint32 reads an uint32 into the supplied buffer.
+// The idea is to make use of the already allocated buffers avoiding additional allocations.
+func (z *Reader) readUint32() (uint32, error) {
+	buf := z.buf[:4]
+	_, err := io.ReadFull(z.src, buf)
+	x := binary.LittleEndian.Uint32(buf)
+	return x, err
+}

vendor/github.com/pierrec/lz4/writer.go

@@ -0,0 +1,267 @@
+package lz4
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+
+	"github.com/pierrec/lz4/internal/xxh32"
+)
+
+// Writer implements the LZ4 frame encoder.
+type Writer struct {
+	Header
+
+	buf       [19]byte      // magic number(4) + header(flags(2)+[Size(8)+DictID(4)]+checksum(1)) does not exceed 19 bytes
+	dst       io.Writer     // Destination.
+	checksum  xxh32.XXHZero // Frame checksum.
+	zdata     []byte        // Compressed data.
+	data      []byte        // Data to be compressed.
+	idx       int           // Index into data.
+	hashtable [winSize]int  // Hash table used in CompressBlock().
+}
+
+// NewWriter returns a new LZ4 frame encoder.
+// No access to the underlying io.Writer is performed.
+// The supplied Header is checked at the first Write.
+// It is ok to change it before the first Write but then not until a Reset() is performed.
+func NewWriter(dst io.Writer) *Writer {
+	return &Writer{dst: dst}
+}
+
+// writeHeader builds and writes the header (magic+header) to the underlying io.Writer.
+func (z *Writer) writeHeader() error {
+	// Default to 4Mb if BlockMaxSize is not set.
+	if z.Header.BlockMaxSize == 0 {
+		z.Header.BlockMaxSize = bsMapID[7]
+	}
+	// The only option that needs to be validated.
+	bSize := z.Header.BlockMaxSize
+	bSizeID, ok := bsMapValue[bSize]
+	if !ok {
+		return fmt.Errorf("lz4: invalid block max size: %d", bSize)
+	}
+	// Allocate the compressed/uncompressed buffers.
+	// The compressed buffer cannot exceed the uncompressed one.
+	if n := 2 * bSize; cap(z.zdata) < n {
+		z.zdata = make([]byte, n, n)
+	}
+	z.zdata = z.zdata[:bSize]
+	z.data = z.zdata[:cap(z.zdata)][bSize:]
+	z.idx = 0
+
+	// Size is optional.
+	buf := z.buf[:]
+
+	// Set the fixed size data: magic number, block max size and flags.
+	binary.LittleEndian.PutUint32(buf[0:], frameMagic)
+	flg := byte(Version << 6)
+	flg |= 1 << 5 // No block dependency.
+	if z.Header.BlockChecksum {
+		flg |= 1 << 4
+	}
+	if z.Header.Size > 0 {
+		flg |= 1 << 3
+	}
+	if !z.Header.NoChecksum {
+		flg |= 1 << 2
+	}
+	buf[4] = flg
+	buf[5] = bSizeID << 4
+
+	// Current buffer size: magic(4) + flags(1) + block max size (1).
+	n := 6
+	// Optional items.
+	if z.Header.Size > 0 {
+		binary.LittleEndian.PutUint64(buf[n:], z.Header.Size)
+		n += 8
+	}
+
+	// The header checksum includes the flags, block max size and optional Size.
+	buf[n] = byte(xxh32.ChecksumZero(buf[4:n]) >> 8 & 0xFF)
+	z.checksum.Reset()
+
+	// Header ready, write it out.
+	if _, err := z.dst.Write(buf[0 : n+1]); err != nil {
+		return err
+	}
+	z.Header.done = true
+	if debugFlag {
+		debug("wrote header %v", z.Header)
+	}
+
+	return nil
+}
+
+// Write compresses data from the supplied buffer into the underlying io.Writer.
+// Write does not return until the data has been written.
+func (z *Writer) Write(buf []byte) (int, error) {
+	if !z.Header.done {
+		if err := z.writeHeader(); err != nil {
+			return 0, err
+		}
+	}
+	if debugFlag {
+		debug("input buffer len=%d index=%d", len(buf), z.idx)
+	}
+
+	zn := len(z.data)
+	var n int
+	for len(buf) > 0 {
+		if z.idx == 0 && len(buf) >= zn {
+			// Avoid a copy as there is enough data for a block.
+			if err := z.compressBlock(buf[:zn]); err != nil {
+				return n, err
+			}
+			n += zn
+			buf = buf[zn:]
+			continue
+		}
+		// Accumulate the data to be compressed.
+		m := copy(z.data[z.idx:], buf)
+		n += m
+		z.idx += m
+		buf = buf[m:]
+		if debugFlag {
+			debug("%d bytes copied to buf, current index %d", n, z.idx)
+		}
+
+		if z.idx < len(z.data) {
+			// Buffer not filled.
+			if debugFlag {
+				debug("need more data for compression")
+			}
+			return n, nil
+		}
+
+		// Buffer full.
+		if err := z.compressBlock(z.data); err != nil {
+			return n, err
+		}
+		z.idx = 0
+	}
+
+	return n, nil
+}
+
+// compressBlock compresses a block.
+func (z *Writer) compressBlock(data []byte) error {
+	if !z.NoChecksum {
+		z.checksum.Write(data)
+	}
+
+	// The compressed block size cannot exceed the input's.
+	var zn int
+	var err error
+
+	if level := z.Header.CompressionLevel; level != 0 {
+		zn, err = CompressBlockHC(data, z.zdata, level)
+	} else {
+		zn, err = CompressBlock(data, z.zdata, z.hashtable[:])
+	}
+
+	var zdata []byte
+	var bLen uint32
+	if debugFlag {
+		debug("block compression %d => %d", len(data), zn)
+	}
+	if err == nil && zn > 0 && zn < len(data) {
+		// Compressible and compressed size smaller than uncompressed: ok!
+		bLen = uint32(zn)
+		zdata = z.zdata[:zn]
+	} else {
+		// Uncompressed block.
+		bLen = uint32(len(data)) | compressedBlockFlag
+		zdata = data
+	}
+	if debugFlag {
+		debug("block compression to be written len=%d data len=%d", bLen, len(zdata))
+	}
+
+	// Write the block.
+	if err := z.writeUint32(bLen); err != nil {
+		return err
+	}
+	if _, err := z.dst.Write(zdata); err != nil {
+		return err
+	}
+
+	if z.BlockChecksum {
+		checksum := xxh32.ChecksumZero(zdata)
+		if debugFlag {
+			debug("block checksum %x", checksum)
+		}
+		if err := z.writeUint32(checksum); err != nil {
+			return err
+		}
+	}
+	if debugFlag {
+		debug("current frame checksum %x", z.checksum.Sum32())
+	}
+
+	return nil
+}
+
+// Flush flushes any pending compressed data to the underlying writer.
+// Flush does not return until the data has been written.
+// If the underlying writer returns an error, Flush returns that error.
+func (z *Writer) Flush() error {
+	if debugFlag {
+		debug("flush with index %d", z.idx)
+	}
+	if z.idx == 0 {
+		return nil
+	}
+
+	return z.compressBlock(z.data[:z.idx])
+}
+
+// Close closes the Writer, flushing any unwritten data to the underlying io.Writer, but does not close the underlying io.Writer.
+func (z *Writer) Close() error {
+	if !z.Header.done {
+		if err := z.writeHeader(); err != nil {
+			return err
+		}
+	}
+
+	if err := z.Flush(); err != nil {
+		return err
+	}
+
+	if debugFlag {
+		debug("writing last empty block")
+	}
+	if err := z.writeUint32(0); err != nil {
+		return err
+	}
+	if !z.NoChecksum {
+		checksum := z.checksum.Sum32()
+		if debugFlag {
+			debug("stream checksum %x", checksum)
+		}
+		if err := z.writeUint32(checksum); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// Reset clears the state of the Writer z such that it is equivalent to its
+// initial state from NewWriter, but instead writing to w.
+// No access to the underlying io.Writer is performed.
+func (z *Writer) Reset(w io.Writer) {
+	z.Header = Header{}
+	z.dst = w
+	z.checksum.Reset()
+	z.zdata = z.zdata[:0]
+	z.data = z.data[:0]
+	z.idx = 0
+}
+
+// writeUint32 writes a uint32 to the underlying writer.
+func (z *Writer) writeUint32(x uint32) error {
+	buf := z.buf[:4]
+	binary.LittleEndian.PutUint32(buf, x)
+	_, err := z.dst.Write(buf)
+	return err
+}

vendor/github.com/rasky/go-lzo/LICENSE.gpl

@@ -0,0 +1,339 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 2, June 1991
+
+ Copyright (C) 1989, 1991 Free Software Foundation, Inc.,
+ 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The licenses for most software are designed to take away your
+freedom to share and change it.  By contrast, the GNU General Public
+License is intended to guarantee your freedom to share and change free
+software--to make sure the software is free for all its users.  This
+General Public License applies to most of the Free Software
+Foundation's software and to any other program whose authors commit to
+using it.  (Some other Free Software Foundation software is covered by
+the GNU Lesser General Public License instead.)  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+this service if you wish), that you receive source code or can get it
+if you want it, that you can change the software or use pieces of it
+in new free programs; and that you know you can do these things.
+
+  To protect your rights, we need to make restrictions that forbid
+anyone to deny you these rights or to ask you to surrender the rights.
+These restrictions translate to certain responsibilities for you if you
+distribute copies of the software, or if you modify it.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must give the recipients all the rights that
+you have.  You must make sure that they, too, receive or can get the
+source code.  And you must show them these terms so they know their
+rights.
+
+  We protect your rights with two steps: (1) copyright the software, and
+(2) offer you this license which gives you legal permission to copy,
+distribute and/or modify the software.
+
+  Also, for each author's protection and ours, we want to make certain
+that everyone understands that there is no warranty for this free
+software.  If the software is modified by someone else and passed on, we
+want its recipients to know that what they have is not the original, so
+that any problems introduced by others will not reflect on the original
+authors' reputations.
+
+  Finally, any free program is threatened constantly by software
+patents.  We wish to avoid the danger that redistributors of a free
+program will individually obtain patent licenses, in effect making the
+program proprietary.  To prevent this, we have made it clear that any
+patent must be licensed for everyone's free use or not licensed at all.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                    GNU GENERAL PUBLIC LICENSE
+   TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
+
+  0. This License applies to any program or other work which contains
+a notice placed by the copyright holder saying it may be distributed
+under the terms of this General Public License.  The "Program", below,
+refers to any such program or work, and a "work based on the Program"
+means either the Program or any derivative work under copyright law:
+that is to say, a work containing the Program or a portion of it,
+either verbatim or with modifications and/or translated into another
+language.  (Hereinafter, translation is included without limitation in
+the term "modification".)  Each licensee is addressed as "you".
+
+Activities other than copying, distribution and modification are not
+covered by this License; they are outside its scope.  The act of
+running the Program is not restricted, and the output from the Program
+is covered only if its contents constitute a work based on the
+Program (independent of having been made by running the Program).
+Whether that is true depends on what the Program does.
+
+  1. You may copy and distribute verbatim copies of the Program's
+source code as you receive it, in any medium, provided that you
+conspicuously and appropriately publish on each copy an appropriate
+copyright notice and disclaimer of warranty; keep intact all the
+notices that refer to this License and to the absence of any warranty;
+and give any other recipients of the Program a copy of this License
+along with the Program.
+
+You may charge a fee for the physical act of transferring a copy, and
+you may at your option offer warranty protection in exchange for a fee.
+
+  2. You may modify your copy or copies of the Program or any portion
+of it, thus forming a work based on the Program, and copy and
+distribute such modifications or work under the terms of Section 1
+above, provided that you also meet all of these conditions:
+
+    a) You must cause the modified files to carry prominent notices
+    stating that you changed the files and the date of any change.
+
+    b) You must cause any work that you distribute or publish, that in
+    whole or in part contains or is derived from the Program or any
+    part thereof, to be licensed as a whole at no charge to all third
+    parties under the terms of this License.
+
+    c) If the modified program normally reads commands interactively
+    when run, you must cause it, when started running for such
+    interactive use in the most ordinary way, to print or display an
+    announcement including an appropriate copyright notice and a
+    notice that there is no warranty (or else, saying that you provide
+    a warranty) and that users may redistribute the program under
+    these conditions, and telling the user how to view a copy of this
+    License.  (Exception: if the Program itself is interactive but
+    does not normally print such an announcement, your work based on
+    the Program is not required to print an announcement.)
+
+These requirements apply to the modified work as a whole.  If
+identifiable sections of that work are not derived from the Program,
+and can be reasonably considered independent and separate works in
+themselves, then this License, and its terms, do not apply to those
+sections when you distribute them as separate works.  But when you
+distribute the same sections as part of a whole which is a work based
+on the Program, the distribution of the whole must be on the terms of
+this License, whose permissions for other licensees extend to the
+entire whole, and thus to each and every part regardless of who wrote it.
+
+Thus, it is not the intent of this section to claim rights or contest
+your rights to work written entirely by you; rather, the intent is to
+exercise the right to control the distribution of derivative or
+collective works based on the Program.
+
+In addition, mere aggregation of another work not based on the Program
+with the Program (or with a work based on the Program) on a volume of
+a storage or distribution medium does not bring the other work under
+the scope of this License.
+
+  3. You may copy and distribute the Program (or a work based on it,
+under Section 2) in object code or executable form under the terms of
+Sections 1 and 2 above provided that you also do one of the following:
+
+    a) Accompany it with the complete corresponding machine-readable
+    source code, which must be distributed under the terms of Sections
+    1 and 2 above on a medium customarily used for software interchange; or,
+
+    b) Accompany it with a written offer, valid for at least three
+    years, to give any third party, for a charge no more than your
+    cost of physically performing source distribution, a complete
+    machine-readable copy of the corresponding source code, to be
+    distributed under the terms of Sections 1 and 2 above on a medium
+    customarily used for software interchange; or,
+
+    c) Accompany it with the information you received as to the offer
+    to distribute corresponding source code.  (This alternative is
+    allowed only for noncommercial distribution and only if you
+    received the program in object code or executable form with such
+    an offer, in accord with Subsection b above.)
+
+The source code for a work means the preferred form of the work for
+making modifications to it.  For an executable work, complete source
+code means all the source code for all modules it contains, plus any
+associated interface definition files, plus the scripts used to
+control compilation and installation of the executable.  However, as a
+special exception, the source code distributed need not include
+anything that is normally distributed (in either source or binary
+form) with the major components (compiler, kernel, and so on) of the
+operating system on which the executable runs, unless that component
+itself accompanies the executable.
+
+If distribution of executable or object code is made by offering
+access to copy from a designated place, then offering equivalent
+access to copy the source code from the same place counts as
+distribution of the source code, even though third parties are not
+compelled to copy the source along with the object code.
+
+  4. You may not copy, modify, sublicense, or distribute the Program
+except as expressly provided under this License.  Any attempt
+otherwise to copy, modify, sublicense or distribute the Program is
+void, and will automatically terminate your rights under this License.
+However, parties who have received copies, or rights, from you under
+this License will not have their licenses terminated so long as such
+parties remain in full compliance.
+
+  5. You are not required to accept this License, since you have not
+signed it.  However, nothing else grants you permission to modify or
+distribute the Program or its derivative works.  These actions are
+prohibited by law if you do not accept this License.  Therefore, by
+modifying or distributing the Program (or any work based on the
+Program), you indicate your acceptance of this License to do so, and
+all its terms and conditions for copying, distributing or modifying
+the Program or works based on it.
+
+  6. Each time you redistribute the Program (or any work based on the
+Program), the recipient automatically receives a license from the
+original licensor to copy, distribute or modify the Program subject to
+these terms and conditions.  You may not impose any further
+restrictions on the recipients' exercise of the rights granted herein.
+You are not responsible for enforcing compliance by third parties to
+this License.
+
+  7. If, as a consequence of a court judgment or allegation of patent
+infringement or for any other reason (not limited to patent issues),
+conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot
+distribute so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you
+may not distribute the Program at all.  For example, if a patent
+license would not permit royalty-free redistribution of the Program by
+all those who receive copies directly or indirectly through you, then
+the only way you could satisfy both it and this License would be to
+refrain entirely from distribution of the Program.
+
+If any portion of this section is held invalid or unenforceable under
+any particular circumstance, the balance of the section is intended to
+apply and the section as a whole is intended to apply in other
+circumstances.
+
+It is not the purpose of this section to induce you to infringe any
+patents or other property right claims or to contest validity of any
+such claims; this section has the sole purpose of protecting the
+integrity of the free software distribution system, which is
+implemented by public license practices.  Many people have made
+generous contributions to the wide range of software distributed
+through that system in reliance on consistent application of that
+system; it is up to the author/donor to decide if he or she is willing
+to distribute software through any other system and a licensee cannot
+impose that choice.
+
+This section is intended to make thoroughly clear what is believed to
+be a consequence of the rest of this License.
+
+  8. If the distribution and/or use of the Program is restricted in
+certain countries either by patents or by copyrighted interfaces, the
+original copyright holder who places the Program under this License
+may add an explicit geographical distribution limitation excluding
+those countries, so that distribution is permitted only in or among
+countries not thus excluded.  In such case, this License incorporates
+the limitation as if written in the body of this License.
+
+  9. The Free Software Foundation may publish revised and/or new versions
+of the General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+Each version is given a distinguishing version number.  If the Program
+specifies a version number of this License which applies to it and "any
+later version", you have the option of following the terms and conditions
+either of that version or of any later version published by the Free
+Software Foundation.  If the Program does not specify a version number of
+this License, you may choose any version ever published by the Free Software
+Foundation.
+
+  10. If you wish to incorporate parts of the Program into other free
+programs whose distribution conditions are different, write to the author
+to ask for permission.  For software which is copyrighted by the Free
+Software Foundation, write to the Free Software Foundation; we sometimes
+make exceptions for this.  Our decision will be guided by the two goals
+of preserving the free status of all derivatives of our free software and
+of promoting the sharing and reuse of software generally.
+
+                            NO WARRANTY
+
+  11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY
+FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW.  EXCEPT WHEN
+OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES
+PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED
+OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.  THE ENTIRE RISK AS
+TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU.  SHOULD THE
+PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING,
+REPAIR OR CORRECTION.
+
+  12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR
+REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES,
+INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING
+OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED
+TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY
+YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER
+PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE
+POSSIBILITY OF SUCH DAMAGES.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+convey the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software; you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation; either version 2 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License along
+    with this program; if not, write to the Free Software Foundation, Inc.,
+    51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+Also add information on how to contact you by electronic and paper mail.
+
+If the program is interactive, make it output a short notice like this
+when it starts in an interactive mode:
+
+    Gnomovision version 69, Copyright (C) year name of author
+    Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, the commands you use may
+be called something other than `show w' and `show c'; they could even be
+mouse-clicks or menu items--whatever suits your program.
+
+You should also get your employer (if you work as a programmer) or your
+school, if any, to sign a "copyright disclaimer" for the program, if
+necessary.  Here is a sample; alter the names:
+
+  Yoyodyne, Inc., hereby disclaims all copyright interest in the program
+  `Gnomovision' (which makes passes at compilers) written by James Hacker.
+
+  <signature of Ty Coon>, 1 April 1989
+  Ty Coon, President of Vice
+
+This General Public License does not permit incorporating your program into
+proprietary programs.  If your program is a subroutine library, you may
+consider it more useful to permit linking proprietary applications with the
+library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.

vendor/github.com/rasky/go-lzo/README.md

@@ -0,0 +1,36 @@
+# go-lzo
+
+[![Build status](https://travis-ci.org/rasky/go-lzo.svg)](https://travis-ci.org/rasky/go-lzo)
+[![Coverage Status](https://coveralls.io/repos/rasky/go-lzo/badge.svg?branch=master&service=github)](https://coveralls.io/github/rasky/go-lzo?branch=master)
+
+Native LZO1X implementation in Golang
+
+This code has been written using the original LZO1X source code as a reference,
+to study and understand the algorithms. Both the LZO1X-1 and LZO1X-999
+algorithms are implemented. These are the most popular of the whole LZO suite
+of algorithms.
+
+Being a straightforward port of the original source code, it shares the same
+license (GPLv2) as I can't possibly claim any copyright on it.
+
+I plan to eventually reimplement LZO1X-1 from scratch. At that point, I will be
+also changing license.
+
+# Benchmarks
+
+These are the benchmarks obtained running the testsuite over the Canterbury
+corpus for the available compressor levels:
+
+Compressor | Level | Original | Compressed | Factor | Time | Speed
+-----------|-------|----------|------------|--------|------|------
+LZO1X-1   | - | 18521760 | 8957481 | 51.6% | 0.16s | 109MiB/s
+LZO1X-999 | 1 | 18521760 | 8217347 | 55.6% | 1.38s | 13MiB/s
+LZO1X-999 | 2 | 18521760 | 7724879 | 58.3% | 1.50s | 12MiB/s
+LZO1X-999 | 3 | 18521760 | 7384377 | 60.1% | 1.68s | 10MiB/s
+LZO1X-999 | 4 | 18521760 | 7266674 | 60.8% | 1.69s | 10MiB/s
+LZO1X-999 | 5 | 18521760 | 6979879 | 62.3% | 2.75s | 6.4MiB/s
+LZO1X-999 | 6 | 18521760 | 6938593 | 62.5% | 4.53s | 3.9MiB/s
+LZO1X-999 | 7 | 18521760 | 6905362 | 62.7% | 6.94s | 2.5MiB/s
+LZO1X-999 | 8 | 18521760 | 6713477 | 63.8% | 20.96s | 863KiB/s
+LZO1X-999 | 9 | 18521760 | 6712069 | 63.8% | 22.82s | 792KiB/s
+

vendor/github.com/rasky/go-lzo/compress.go

@@ -0,0 +1,178 @@
+package lzo
+
+func appendMulti(out []byte, t int) []byte {
+	for t > 255 {
+		out = append(out, 0)
+		t -= 255
+	}
+	out = append(out, byte(t))
+	return out
+}
+
+func compress(in []byte) (out []byte, sz int) {
+	var m_off int
+	in_len := len(in)
+	ip_len := in_len - m2_MAX_LEN - 5
+	dict := make([]int32, 1<<d_BITS)
+	ii := 0
+	ip := 4
+	for {
+		key := int(in[ip+3])
+		key = (key << 6) ^ int(in[ip+2])
+		key = (key << 5) ^ int(in[ip+1])
+		key = (key << 5) ^ int(in[ip+0])
+		dindex := ((0x21 * key) >> 5) & d_MASK
+		m_pos := int(dict[dindex]) - 1
+		if m_pos < 0 {
+			goto literal
+		}
+		if ip == m_pos || (ip-m_pos) > m4_MAX_OFFSET {
+			goto literal
+		}
+		m_off = ip - m_pos
+		if m_off <= m2_MAX_OFFSET || in[m_pos+3] == in[ip+3] {
+			goto try_match
+		}
+
+		dindex = (dindex & (d_MASK & 0x7ff)) ^ (d_HIGH | 0x1f)
+		m_pos = int(dict[dindex]) - 1
+		if m_pos < 0 {
+			goto literal
+		}
+		if ip == m_pos || (ip-m_pos) > m4_MAX_OFFSET {
+			goto literal
+		}
+		m_off = ip - m_pos
+		if m_off <= m2_MAX_OFFSET || in[m_pos+3] == in[ip+3] {
+			goto try_match
+		}
+
+		goto literal
+
+	try_match:
+		if in[m_pos] == in[ip] && in[m_pos+1] == in[ip+1] && in[m_pos+2] == in[ip+2] {
+			goto match
+		}
+
+	literal:
+		dict[dindex] = int32(ip + 1)
+		ip += 1 + (ip-ii)>>5
+		if ip >= ip_len {
+			break
+		}
+		continue
+
+	match:
+		dict[dindex] = int32(ip + 1)
+		if ip != ii {
+			t := ip - ii
+			if t <= 3 {
+				out[len(out)-2] |= byte(t)
+			} else if t <= 18 {
+				out = append(out, byte(t-3))
+			} else {
+				out = append(out, 0)
+				out = appendMulti(out, t-18)
+			}
+
+			out = append(out, in[ii:ii+t]...)
+			ii += t
+		}
+
+		var i int
+		ip += 3
+		for i = 3; i < 9; i++ {
+			ip++
+			if in[m_pos+i] != in[ip-1] {
+				break
+			}
+		}
+		if i < 9 {
+			ip--
+			m_len := ip - ii
+			if m_off <= m2_MAX_OFFSET {
+				m_off -= 1
+				out = append(out,
+					byte((((m_len - 1) << 5) | ((m_off & 7) << 2))),
+					byte((m_off >> 3)))
+			} else if m_off <= m3_MAX_OFFSET {
+				m_off -= 1
+				out = append(out,
+					byte(m3_MARKER|(m_len-2)),
+					byte((m_off&63)<<2),
+					byte(m_off>>6))
+			} else {
+				m_off -= 0x4000
+				out = append(out,
+					byte(m4_MARKER|((m_off&0x4000)>>11)|(m_len-2)),
+					byte((m_off&63)<<2),
+					byte(m_off>>6))
+			}
+		} else {
+			m := m_pos + m2_MAX_LEN + 1
+			for ip < in_len && in[m] == in[ip] {
+				m++
+				ip++
+			}
+			m_len := ip - ii
+			if m_off <= m3_MAX_OFFSET {
+				m_off -= 1
+				if m_len <= 33 {
+					out = append(out, byte(m3_MARKER|(m_len-2)))
+				} else {
+					m_len -= 33
+					out = append(out, byte(m3_MARKER|0))
+					out = appendMulti(out, m_len)
+				}
+			} else {
+				m_off -= 0x4000
+				if m_len <= m4_MAX_LEN {
+					out = append(out, byte(m4_MARKER|((m_off&0x4000)>>11)|(m_len-2)))
+				} else {
+					m_len -= m4_MAX_LEN
+					out = append(out, byte(m4_MARKER|((m_off&0x4000)>>11)))
+					out = appendMulti(out, m_len)
+				}
+			}
+			out = append(out, byte((m_off&63)<<2), byte(m_off>>6))
+		}
+
+		ii = ip
+		if ip >= ip_len {
+			break
+		}
+	}
+
+	sz = in_len - ii
+	return
+}
+
+// Compress an input buffer with LZO1X
+func Compress1X(in []byte) (out []byte) {
+	var t int
+
+	in_len := len(in)
+	if in_len <= m2_MAX_LEN+5 {
+		t = in_len
+	} else {
+		out, t = compress(in)
+	}
+
+	if t > 0 {
+		ii := in_len - t
+		if len(out) == 0 && t <= 238 {
+			out = append(out, byte(17+t))
+		} else if t <= 3 {
+			out[len(out)-2] |= byte(t)
+		} else if t <= 18 {
+			out = append(out, byte(t-3))
+		} else {
+			out = append(out, 0)
+			out = appendMulti(out, t-18)
+		}
+		out = append(out, in[ii:ii+t]...)
+	}
+
+	out = append(out, m4_MARKER|1, 0, 0)
+	return
+}

vendor/github.com/rasky/go-lzo/compress999.go

@@ -0,0 +1,416 @@
+package lzo
+
+type compressor struct {
+	in []byte
+	ip int
+	bp int
+
+	// stats
+	matchBytes int
+	litBytes   int
+	lazy       int
+
+	r1lit int
+	r2lit int
+	m1am  uint
+	m2m   uint
+	m1bm  uint
+	m3m   uint
+	m4m   uint
+	lit1r uint
+	lit2r uint
+	lit3r uint
+
+	r1mlen int
+
+	lastmlen int
+	lastmoff int
+	textsize uint
+	mlen     int
+	moff     int
+	look     uint
+}
+
+func (ctx *compressor) codeMatch(out []byte, mlen int, moff int) []byte {
+	xlen := mlen
+	xoff := moff
+	ctx.matchBytes += mlen
+
+	switch {
+	case mlen == 2:
+		if moff > m1_MAX_OFFSET {
+			panic("codeMatch: mlen 2: moff error")
+		}
+		if ctx.r1lit < 1 || ctx.r1lit >= 4 {
+			panic("codeMatch: mlen 2: r1lit error")
+		}
+		moff -= 1
+		out = append(out,
+			m1_MARKER|byte((moff&3)<<2),
+			byte(moff>>2))
+		ctx.m1am++
+	case mlen <= m2_MAX_LEN && moff <= m2_MAX_OFFSET:
+		if mlen < 3 {
+			panic("codeMatch: m2: mlen error")
+		}
+		moff -= 1
+		out = append(out,
+			byte((mlen-1)<<5|(moff&7)<<2),
+			byte(moff>>3))
+		if out[len(out)-2] < m2_MARKER {
+			panic("codeMatch: m2: invalid marker")
+		}
+		ctx.m2m++
+	case mlen == m2_MIN_LEN && moff <= mX_MAX_OFFSET && ctx.r1lit >= 4:
+		if mlen != 3 {
+			panic("codeMatch: m2min: invalid mlen")
+		}
+		if moff <= m2_MAX_OFFSET {
+			panic("codeMatch: m2min: invalid moff")
+		}
+		moff -= 1 + m2_MAX_OFFSET
+		out = append(out,
+			byte(m1_MARKER|((moff&3)<<2)),
+			byte(moff>>2))
+		ctx.m1bm++
+	case moff <= m3_MAX_OFFSET:
+		if mlen < 3 {
+			panic("codeMatch: m3max: invalid mlen")
+		}
+		moff -= 1
+		if mlen <= m3_MAX_LEN {
+			out = append(out, byte(m3_MARKER|(mlen-2)))
+		} else {
+			mlen -= m3_MAX_LEN
+			out = append(out, byte(m3_MARKER|0))
+			out = appendMulti(out, mlen)
+		}
+		out = append(out, byte(moff<<2), byte(moff>>6))
+		ctx.m3m++
+	default:
+		if mlen < 3 {
+			panic("codeMatch: default: invalid mlen")
+		}
+		if moff <= 0x4000 || moff >= 0xc000 {
+			panic("codeMatch: default: invalid moff")
+		}
+		moff -= 0x4000
+		k := (moff & 0x4000) >> 11
+		if mlen <= m4_MAX_LEN {
+			out = append(out, byte(m4_MARKER|k|(mlen-2)))
+		} else {
+			mlen -= m4_MAX_LEN
+			out = append(out, byte(m4_MARKER|k|0))
+			out = appendMulti(out, mlen)
+		}
+		out = append(out, byte(moff<<2), byte(moff>>6))
+		ctx.m4m++
+	}
+
+	ctx.lastmlen = xlen
+	ctx.lastmoff = xoff
+	return out
+}
+
+func (ctx *compressor) storeRun(out []byte, ii int, t int) []byte {
+	ctx.litBytes += t
+
+	if len(out) == 0 && t <= 238 {
+		out = append(out, byte(17+t))
+	} else if t <= 3 {
+		out[len(out)-2] |= byte(t)
+		ctx.lit1r++
+	} else if t <= 18 {
+		out = append(out, byte(t-3))
+		ctx.lit2r++
+	} else {
+		out = append(out, 0)
+		out = appendMulti(out, t-18)
+		ctx.lit3r++
+	}
+
+	out = append(out, ctx.in[ii:ii+t]...)
+	return out
+}
+
+func (ctx *compressor) codeRun(out []byte, ii int, lit int, mlen int) []byte {
+	if lit > 0 {
+		if mlen < 2 {
+			panic("codeRun: invalid mlen")
+		}
+		out = ctx.storeRun(out, ii, lit)
+		ctx.r1mlen = mlen
+		ctx.r1lit = lit
+	} else {
+		if mlen < 3 {
+			panic("codeRun: invalid mlen")
+		}
+		ctx.r1mlen = 0
+		ctx.r1lit = 0
+	}
+	return out
+}
+
+func (ctx *compressor) lenOfCodedMatch(mlen int, moff int, lit int) int {
+	switch {
+	case mlen < 2:
+		return 0
+	case mlen == 2:
+		if moff <= m1_MAX_OFFSET && lit > 0 && lit < 4 {
+			return 2
+		}
+		return 0
+	case mlen <= m2_MAX_LEN && moff <= m2_MAX_OFFSET:
+		return 2
+	case mlen == m2_MIN_LEN && moff <= mX_MAX_OFFSET && lit >= 4:
+		return 2
+	case moff <= m3_MAX_OFFSET:
+		if mlen <= m3_MAX_LEN {
+			return 3
+		}
+		n := 4
+		mlen -= m3_MAX_LEN
+		for mlen > 255 {
+			mlen -= 255
+			n++
+		}
+		return n
+	case moff <= m4_MAX_OFFSET:
+		if mlen <= m4_MAX_LEN {
+			return 3
+		}
+		n := 4
+		mlen -= m4_MAX_LEN
+		for mlen > 255 {
+			mlen -= 255
+			n++
+		}
+		return n
+	default:
+		return 0
+	}
+}
+
+func (ctx *compressor) minGain(ahead int,
+	lit1, lit2 int, l1, l2, l3 int) int {
+
+	if ahead <= 0 {
+		panic("minGain: invalid ahead")
+	}
+	mingain := int(ahead)
+	if lit1 <= 3 {
+		if lit2 > 3 {
+			mingain += 2
+		}
+	} else if lit1 <= 18 {
+		if lit2 > 18 {
+			mingain += 1
+		}
+	}
+
+	mingain += int((l2 - l1) * 2)
+	if l3 != 0 {
+		mingain -= int((ahead - l3) * 2)
+	}
+	if mingain < 0 {
+		mingain = 0
+	}
+	return mingain
+}
+
+type parms struct {
+	TryLazy  int
+	GoodLen  uint
+	MaxLazy  uint
+	NiceLen  uint
+	MaxChain uint
+	Flags    uint32
+}
+
+func compress999(in []byte, p parms) []byte {
+	ctx := compressor{}
+	swd := swd{}
+
+	if p.TryLazy < 0 {
+		p.TryLazy = 1
+	}
+	if p.GoodLen == 0 {
+		p.GoodLen = 32
+	}
+	if p.MaxLazy == 0 {
+		p.MaxLazy = 32
+	}
+	if p.MaxChain == 0 {
+		p.MaxChain = cSWD_MAX_CHAIN
+	}
+
+	ctx.in = in
+
+	out := make([]byte, 0, len(in)/2)
+	ii := 0
+	lit := 0
+
+	ctx.initMatch(&swd, p.Flags)
+	if p.MaxChain > 0 {
+		swd.MaxChain = p.MaxChain
+	}
+	if p.NiceLen > 0 {
+		swd.NiceLength = p.NiceLen
+	}
+
+	ctx.findMatch(&swd, 0, 0)
+	for ctx.look > 0 {
+		mlen := ctx.mlen
+		moff := ctx.moff
+		if ctx.bp != ctx.ip-int(ctx.look) {
+			panic("assert: compress: invalid bp")
+		}
+		if ctx.bp < 0 {
+			panic("assert: compress: negative bp")
+		}
+		if lit == 0 {
+			ii = ctx.bp
+		}
+		if ii+lit != ctx.bp {
+			panic("assert: compress: invalid ii")
+		}
+		if swd.BChar != int(ctx.in[ctx.bp]) {
+			panic("assert: compress: invalid bchar")
+		}
+
+		if mlen < 2 ||
+			(mlen == 2 && (moff > m1_MAX_OFFSET || lit == 0 || lit >= 4)) ||
+			(mlen == 2 && len(out) == 0) ||
+			(len(out) == 0 && lit == 0) {
+			// literal
+			mlen = 0
+		} else if mlen == m2_MIN_LEN {
+			if moff > mX_MAX_OFFSET && lit >= 4 {
+				mlen = 0
+			}
+		}
+
+		if mlen == 0 {
+			// literal
+			lit++
+			swd.MaxChain = p.MaxChain
+			ctx.findMatch(&swd, 1, 0)
+			continue
+		}
+
+		// a match
+		if swd.UseBestOff {
+			mlen, moff = ctx.betterMatch(&swd, mlen, moff)
+		}
+
+		ctx.assertMatch(&swd, mlen, moff)
+
+		// check if we want to try a lazy match
+		ahead := 0
+		l1 := 0
+		maxahead := 0
+		if p.TryLazy != 0 && mlen < int(p.MaxLazy) {
+			l1 = ctx.lenOfCodedMatch(mlen, moff, lit)
+			if l1 == 0 {
+				panic("assert: compress: invalid len of coded match")
+			}
+			maxahead = p.TryLazy
+			if maxahead > l1-1 {
+				maxahead = l1 - 1
+			}
+		}
+
+		matchdone := false
+		for ahead < maxahead && int(ctx.look) > mlen {
+			if mlen >= int(p.GoodLen) {
+				swd.MaxChain = p.MaxChain >> 2
+			} else {
+				swd.MaxChain = p.MaxChain
+			}
+			ctx.findMatch(&swd, 1, 0)
+			ahead++
+			if ctx.look <= 0 {
+				panic("assert: compress: invalid look")
+			}
+			if ii+lit+ahead != ctx.bp {
+				panic("assert: compress: invalid bp")
+			}
+			if ctx.mlen < mlen {
+				continue
+			}
+			if ctx.mlen == mlen && ctx.moff >= moff {
+				continue
+			}
+			if swd.UseBestOff {
+				ctx.mlen, ctx.moff = ctx.betterMatch(&swd, ctx.mlen, ctx.moff)
+			}
+			l2 := ctx.lenOfCodedMatch(ctx.mlen, ctx.moff, lit+ahead)
+			if l2 == 0 {
+				continue
+			}
+			l3 := 0
+			if len(out) > 0 {
+				l3 = ctx.lenOfCodedMatch(ahead, moff, lit)
+			}
+			mingain := ctx.minGain(ahead, lit, lit+ahead, l1, l2, l3)
+			if ctx.mlen >= mlen+mingain {
+				ctx.lazy++
+				ctx.assertMatch(&swd, ctx.mlen, ctx.moff)
+
+				if l3 > 0 {
+					out = ctx.codeRun(out, ii, lit, ahead)
+					lit = 0
+					out = ctx.codeMatch(out, ahead, moff)
+				} else {
+					lit += ahead
+					if ii+lit != ctx.bp {
+						panic("assert: compress: invalid bp after l3")
+					}
+				}
+				matchdone = true
+				break
+			}
+		}
+
+		if !matchdone {
+			if ii+lit+ahead != ctx.bp {
+				panic("assert: compress: invalid bp out of for loop")
+			}
+
+			out = ctx.codeRun(out, ii, lit, mlen)
+			lit = 0
+			out = ctx.codeMatch(out, mlen, moff)
+			swd.MaxChain = p.MaxChain
+			ctx.findMatch(&swd, uint(mlen), uint(1+ahead))
+		}
+	}
+
+	if lit > 0 {
+		out = ctx.storeRun(out, ii, lit)
+	}
+	out = append(out, m4_MARKER|1, 0, 0)
+	if ctx.litBytes+ctx.matchBytes != len(ctx.in) {
+		panic("assert: compress999: not processed full input")
+	}
+	return out
+}
+
+var fixedLevels = [...]parms{
+	{0, 0, 0, 8, 4, 0},
+	{0, 0, 0, 16, 8, 0},
+	{0, 0, 0, 32, 16, 0},
+	{1, 4, 4, 16, 16, 0},
+	{1, 8, 16, 32, 32, 0},
+	{1, 8, 16, 128, 128, 0},
+	{2, 8, 32, 128, 256, 0},
+	{2, 32, 128, cSWD_F, 2048, 1},
+	{2, cSWD_F, cSWD_F, cSWD_F, 4096, 1},
+}
+
+func Compress1X999Level(in []byte, level int) []byte {
+	return compress999(in, fixedLevels[level-1])
+}
+
+func Compress1X999(in []byte) []byte {
+	return Compress1X999Level(in, 9)
+}

vendor/github.com/rasky/go-lzo/decompress.go

@@ -0,0 +1,289 @@
+package lzo
+
+import (
+	"errors"
+	"io"
+	"runtime"
+)
+
+var (
+	InputUnderrun      = errors.New("input underrun")
+	LookBehindUnderrun = errors.New("lookbehind underrun")
+)
+
+type reader struct {
+	r   io.Reader
+	len int
+	buf [4096]byte
+	cur []byte
+	Err error
+}
+
+func newReader(r io.Reader, inlen int) *reader {
+	if inlen == 0 {
+		inlen = -1
+	}
+	in := &reader{r: r, len: inlen}
+	in.Rebuffer()
+	return in
+}
+
+// Read more data from the underlying reader and put it into the buffer.
+// Also makes sure there is always at least 32 bytes in the buffer, so that
+// in the main loop we can avoid checking for the end of buffer.
+func (in *reader) Rebuffer() {
+	const RBUF_WND = 32
+	var rbuf [RBUF_WND]byte
+
+	if len(in.cur) > RBUF_WND || in.len == 0 {
+		return
+	}
+
+	rb := rbuf[:len(in.cur)]
+	copy(rb, in.cur)
+	in.cur = in.buf[:]
+	copy(in.cur, rb)
+
+	cur := in.cur[len(rb):]
+	if in.len >= 0 && len(cur) > in.len {
+		cur = cur[:in.len]
+	}
+	n, err := in.r.Read(cur)
+	if err != nil {
+		// If EOF is returned, treat it as error only if there are no further
+		// bytes in the window. Otherwise, let's postpone because those bytes
+		// could contain the terminator.
+		if err != io.EOF || len(rb) == 0 {
+			in.Err = err
+			in.cur = nil
+		}
+	}
+	in.cur = in.cur[:len(rb)+n]
+	if in.len >= 0 {
+		in.len -= n
+	}
+}
+
+func (in *reader) ReadAppend(out *[]byte, n int) {
+	for n > 0 {
+		m := len(in.cur)
+		if m > n {
+			m = n
+		}
+		*out = append(*out, in.cur[:m]...)
+		in.cur = in.cur[m:]
+		n -= m
+		if len(in.cur) == 0 {
+			in.Rebuffer()
+			if len(in.cur) == 0 {
+				in.Err = io.EOF
+				return
+			}
+		}
+	}
+	return
+}
+
+func (in *reader) ReadU8() (ch byte) {
+	ch = in.cur[0]
+	in.cur = in.cur[1:]
+	return
+}
+
+func (in *reader) ReadU16() int {
+	b0 := in.cur[0]
+	b1 := in.cur[1]
+	in.cur = in.cur[2:]
+	return int(b0) + int(b1)<<8
+}
+
+func (in *reader) ReadMulti(base int) (b int) {
+	for {
+		for i := 0; i < len(in.cur); i++ {
+			v := in.cur[i]
+			if v == 0 {
+				b += 255
+			} else {
+				b += int(v) + base
+				in.cur = in.cur[i+1:]
+				return
+			}
+		}
+		in.cur = in.cur[0:0]
+		in.Rebuffer()
+		if len(in.cur) == 0 {
+			in.Err = io.EOF
+			return
+		}
+	}
+}
+
+func copyMatch(out *[]byte, m_pos int, n int) {
+	if m_pos+n > len(*out) {
+		// fmt.Println("copy match WITH OVERLAP!")
+		for i := 0; i < n; i++ {
+			*out = append(*out, (*out)[m_pos])
+			m_pos++
+		}
+	} else {
+		// fmt.Println("copy match:", len(*out), m_pos, m_pos+n)
+		*out = append(*out, (*out)[m_pos:m_pos+n]...)
+	}
+}
+
+// Decompress an input compressed with LZO1X.
+//
+// LZO1X has a stream terminator marker, so the decompression will always stop
+// when this marker is found.
+//
+// If inLen is not zero, it is expected to match the length of the compressed
+// input stream, and it is used to limit reads from the underlying reader; if
+// inLen is smaller than the real stream, the decompression will abort with an
+// error; if inLen is larger than the real stream, or if it is zero, the
+// decompression will succeed but more bytes than necessary might be read
+// from the underlying reader. If the reader returns EOF before the termination
+// marker is found, the decompression aborts and EOF is returned.
+//
+// outLen is optional; if it's not zero, it is used as a hint to preallocate the
+// output buffer to increase performance of the decompression.
+func Decompress1X(r io.Reader, inLen int, outLen int) (out []byte, err error) {
+	var t, m_pos int
+	var last2 byte
+
+	defer func() {
+		// To gain performance, we don't do any bounds checking while reading
+		// the input, so if the decompressor reads past the end of the input
+		// stream, a runtime error is raised. This saves about 7% of performance
+		// as the reading functions are very hot in the decompressor.
+		if r := recover(); r != nil {
+			if re, ok := r.(runtime.Error); ok {
+				if re.Error() == "runtime error: index out of range" {
+					err = io.EOF
+					return
+				}
+			}
+			panic(r)
+		}
+	}()
+
+	out = make([]byte, 0, outLen)
+
+	in := newReader(r, inLen)
+	ip := in.ReadU8()
+	if ip > 17 {
+		t = int(ip) - 17
+		if t < 4 {
+			goto match_next
+		}
+		in.ReadAppend(&out, t)
+		// fmt.Println("begin:", string(out))
+		goto first_literal_run
+	}
+
+begin_loop:
+	t = int(ip)
+	if t >= 16 {
+		goto match
+	}
+	if t == 0 {
+		t = in.ReadMulti(15)
+	}
+	in.ReadAppend(&out, t+3)
+	// fmt.Println("readappend", t+3, string(out[len(out)-t-3:]))
+first_literal_run:
+	ip = in.ReadU8()
+	last2 = ip
+	t = int(ip)
+	if t >= 16 {
+		goto match
+	}
+	m_pos = len(out) - (1 + m2_MAX_OFFSET)
+	m_pos -= t >> 2
+	ip = in.ReadU8()
+	m_pos -= int(ip) << 2
+	// fmt.Println("m_pos flr", m_pos, len(out), "\n", string(out))
+	if m_pos < 0 {
+		err = LookBehindUnderrun
+		return
+	}
+	copyMatch(&out, m_pos, 3)
+	goto match_done
+
+match:
+	in.Rebuffer()
+	if in.Err != nil {
+		err = in.Err
+		return
+	}
+	t = int(ip)
+	last2 = ip
+	if t >= 64 {
+		m_pos = len(out) - 1
+		m_pos -= (t >> 2) & 7
+		ip = in.ReadU8()
+		m_pos -= int(ip) << 3
+		// fmt.Println("m_pos t64", m_pos, t, int(ip))
+		t = (t >> 5) - 1
+		goto copy_match
+	} else if t >= 32 {
+		t &= 31
+		if t == 0 {
+			t = in.ReadMulti(31)
+		}
+		m_pos = len(out) - 1
+		v16 := in.ReadU16()
+		m_pos -= v16 >> 2
+		last2 = byte(v16 & 0xFF)
+		// fmt.Println("m_pos t32", m_pos)
+	} else if t >= 16 {
+		m_pos = len(out)
+		m_pos -= (t & 8) << 11
+		t &= 7
+		if t == 0 {
+			t = in.ReadMulti(7)
+		}
+		v16 := in.ReadU16()
+		m_pos -= v16 >> 2
+		if m_pos == len(out) {
+			// fmt.Println("END", t, v16, m_pos)
+			return
+		}
+		m_pos -= 0x4000
+		last2 = byte(v16 & 0xFF)
+		// fmt.Println("m_pos t16", m_pos)
+	} else {
+		m_pos = len(out) - 1
+		m_pos -= t >> 2
+		ip = in.ReadU8()
+		m_pos -= int(ip) << 2
+		if m_pos < 0 {
+			err = LookBehindUnderrun
+			return
+		}
+		// fmt.Println("m_pos tX", m_pos)
+		copyMatch(&out, m_pos, 2)
+		goto match_done
+	}
+
+copy_match:
+	if m_pos < 0 {
+		err = LookBehindUnderrun
+		return
+	}
+	copyMatch(&out, m_pos, t+2)
+
+match_done:
+	t = int(last2 & 3)
+	if t == 0 {
+		goto match_end
+	}
+match_next:
+	// fmt.Println("read append finale:", t)
+	in.ReadAppend(&out, t)
+	ip = in.ReadU8()
+	goto match
+
+match_end:
+	ip = in.ReadU8()
+	goto begin_loop
+}

vendor/github.com/rasky/go-lzo/defs.go

@@ -0,0 +1,29 @@
+package lzo
+
+const (
+	m1_MAX_OFFSET = 0x0400
+	m2_MAX_OFFSET = 0x0800
+	m3_MAX_OFFSET = 0x4000
+	m4_MAX_OFFSET = 0xbfff
+	mX_MAX_OFFSET = m1_MAX_OFFSET + m2_MAX_OFFSET
+
+	m1_MIN_LEN = 2
+	m1_MAX_LEN = 2
+	m2_MIN_LEN = 3
+	m2_MAX_LEN = 8
+	m3_MIN_LEN = 3
+	m3_MAX_LEN = 33
+	m4_MIN_LEN = 3
+	m4_MAX_LEN = 9
+
+	m1_MARKER = 0
+	m2_MARKER = 64
+	m3_MARKER = 32
+	m4_MARKER = 16
+)
+
+const (
+	d_BITS = 14
+	d_MASK = (1 << d_BITS) - 1
+	d_HIGH = (d_MASK >> 1) + 1
+)

vendor/github.com/rasky/go-lzo/fuzz.go

@@ -0,0 +1,10 @@
+// +build gofuzz
+
+package lzo
+
+import "bytes"
+
+func Fuzz(data []byte) int {
+	Decompress1X(bytes.NewBuffer(data), 0, 0)
+	return 0
+}

vendor/github.com/rasky/go-lzo/match.go

@@ -0,0 +1,101 @@
+package lzo
+
+func (ctx *compressor) initMatch(s *swd, flags uint32) {
+	s.ctx = ctx
+	s.init()
+	if flags&1 != 0 {
+		s.UseBestOff = true
+	}
+}
+
+func (ctx *compressor) findMatch(s *swd, thislen uint, skip uint) {
+	if skip > 0 {
+		if thislen < skip {
+			panic("assert: findMatch: invalid thislen")
+		}
+		s.accept(thislen - skip)
+		ctx.textsize += thislen - skip + 1
+	} else {
+		if thislen > 1 {
+			panic("assert: findMatch: invalid thislen")
+		}
+		ctx.textsize += thislen - skip
+	}
+
+	s.MLen = cSWD_THRESHOLD
+	s.MOff = 0
+	for i := 0; i < len(s.bestPos); i++ {
+		s.bestPos[i] = 0
+	}
+
+	s.findbest()
+	ctx.mlen = int(s.MLen)
+	ctx.moff = int(s.MOff)
+
+	s.getbyte()
+	if s.BChar < 0 {
+		ctx.look = 0
+		ctx.mlen = 0
+	} else {
+		ctx.look = s.Look + 1
+	}
+
+	ctx.bp = ctx.ip - int(ctx.look)
+}
+
+func (ctx *compressor) betterMatch(s *swd, imlen, imoff int) (mlen int, moff int) {
+	mlen, moff = imlen, imoff
+	if mlen <= m2_MIN_LEN {
+		return
+	}
+	if moff <= m2_MAX_OFFSET {
+		return
+	}
+
+	if moff > m2_MAX_OFFSET && mlen >= m2_MIN_LEN+1 && mlen <= m2_MAX_LEN+1 &&
+		s.BestOff[mlen-1] > 0 && s.BestOff[mlen-1] <= m2_MAX_OFFSET {
+		mlen -= 1
+		moff = int(s.BestOff[mlen])
+		return
+	}
+
+	if moff > m3_MAX_OFFSET && mlen >= m4_MAX_LEN+1 && mlen <= m2_MAX_LEN+2 &&
+		s.BestOff[mlen-2] > 0 && s.BestOff[mlen-2] <= m2_MAX_OFFSET {
+		mlen -= 2
+		moff = int(s.BestOff[mlen])
+		return
+	}
+
+	if moff > m3_MAX_OFFSET && mlen >= m4_MAX_LEN+1 && mlen <= m3_MAX_LEN+1 &&
+		s.BestOff[mlen-1] > 0 && s.BestOff[mlen-1] <= m3_MAX_OFFSET {
+		mlen -= 1
+		moff = int(s.BestOff[mlen])
+		return
+	}
+
+	return
+}
+
+func assertMemcmp(b1, b2 []byte, l int) {
+	b1 = b1[:l]
+	b2 = b2[:l]
+	for i := 0; i < len(b1); i++ {
+		if b1[i] != b2[i] {
+			panic("assertMemcmp: dosn't match")
+		}
+	}
+}
+
+func (ctx *compressor) assertMatch(s *swd, mlen, moff int) {
+	if mlen < 2 {
+		panic("assertMatch: invalid mlen")
+	}
+	if moff <= ctx.bp {
+		if ctx.bp-moff+mlen >= ctx.ip {
+			panic("assertMatch: invalid bp")
+		}
+		assertMemcmp(ctx.in[ctx.bp:], ctx.in[ctx.bp-moff:], mlen)
+	} else {
+		panic("dict should not exit")
+	}
+}

vendor/github.com/rasky/go-lzo/swd.go

@@ -0,0 +1,349 @@
+package lzo
+
+const (
+	cSWD_N         = m4_MAX_OFFSET  // ring buffer size
+	cSWD_THRESHOLD = 1              // lower limit for match length
+	cSWD_F         = 2048           // upper limit for match length
+	cSWD_BEST_OFF  = m3_MAX_LEN + 1 // max(m2,m3,m4)+1
+	cSWD_HSIZE     = 16384
+	cSWD_MAX_CHAIN = 2048
+)
+
+type swd struct {
+	// Public builtin
+	SwdN         uint
+	SwdF         uint
+	SwdThreshold uint
+
+	// Public configuration
+	MaxChain   uint
+	NiceLength uint
+	UseBestOff bool
+	LazyInsert uint
+
+	// Output
+	MLen    uint
+	MOff    uint
+	Look    uint
+	BChar   int
+	BestOff [cSWD_BEST_OFF]uint
+
+	// Semi-public
+	ctx     *compressor
+	mpos    uint
+	bestPos [cSWD_BEST_OFF]uint
+
+	// Private
+	ip        uint // input pointer (lookahead)
+	bp        uint // buffer pointer
+	rp        uint // remove pointer
+	bsize     uint
+	bwrap     []byte
+	nodecount uint
+	firstrp   uint
+
+	b     [cSWD_N + cSWD_F + cSWD_F]byte
+	head3 [cSWD_HSIZE]uint16
+	succ3 [cSWD_N + cSWD_F]uint16
+	best3 [cSWD_N + cSWD_F]uint16
+	llen3 [cSWD_HSIZE]uint16
+	head2 [65536]uint16
+}
+
+func head2(data []byte) uint {
+	return uint(data[1])<<8 | uint(data[0])
+}
+
+func head3(data []byte) uint {
+	key := uint(data[0])
+	key = (key << 5) ^ uint(data[1])
+	key = (key << 5) ^ uint(data[2])
+	key = (key * 0x9f5f) >> 5
+	return key & (cSWD_HSIZE - 1)
+}
+
+func (s *swd) gethead3(key uint) uint16 {
+	if s.llen3[key] == 0 {
+		return 0xFFFF
+	}
+	return s.head3[key]
+}
+
+func (s *swd) removeNode(node uint) {
+	if s.nodecount == 0 {
+		key := head3(s.b[node:])
+		if s.llen3[key] == 0 {
+			panic("assert: swd.removeNode: invalid llen3")
+		}
+		s.llen3[key]--
+
+		key = head2(s.b[node:])
+		if s.head2[key] == 0xFFFF {
+			panic("assert: swd.removeNode: invalid head2")
+		}
+		if uint(s.head2[key]) == node {
+			s.head2[key] = 0xFFFF
+		}
+		return
+	}
+	s.nodecount--
+}
+
+func (s *swd) init() {
+	s.SwdN = cSWD_N
+	s.SwdF = cSWD_F
+	s.SwdThreshold = cSWD_THRESHOLD
+
+	s.MaxChain = cSWD_MAX_CHAIN
+	s.NiceLength = s.SwdF
+	s.bsize = s.SwdN + s.SwdF
+	s.bwrap = s.b[s.bsize:]
+	s.nodecount = s.SwdN
+
+	for i := 0; i < len(s.head2); i++ {
+		s.head2[i] = 0xFFFF
+	}
+
+	s.ip = 0
+	s.bp = s.ip
+	s.firstrp = s.ip
+	if s.ip+s.SwdF > s.bsize {
+		panic("assert: swd.init: invalid ip")
+	}
+
+	s.Look = uint(len(s.ctx.in)) - s.ip
+	if s.Look > 0 {
+		if s.Look > s.SwdF {
+			s.Look = s.SwdF
+		}
+		copy(s.b[s.ip:], s.ctx.in[:s.Look])
+		s.ctx.ip += int(s.Look)
+		s.ip += s.Look
+	}
+
+	if s.ip == s.bsize {
+		s.ip = 0
+	}
+
+	s.rp = s.firstrp
+	if s.rp >= s.nodecount {
+		s.rp -= s.nodecount
+	} else {
+		s.rp += s.bsize - s.nodecount
+	}
+
+	if s.Look < 3 {
+		s.b[s.bp+s.Look] = 0
+		s.b[s.bp+s.Look+1] = 0
+		s.b[s.bp+s.Look+2] = 0
+	}
+}
+
+func (s *swd) getbyte() {
+	c := -1
+	if s.ctx.ip < len(s.ctx.in) {
+		c = int(s.ctx.in[s.ctx.ip])
+		s.ctx.ip++
+		s.b[s.ip] = byte(c)
+		if s.ip < s.SwdF {
+			s.bwrap[s.ip] = byte(c)
+		}
+	} else {
+		if s.Look > 0 {
+			s.Look--
+		}
+		s.b[s.ip] = 0
+		if s.ip < s.SwdF {
+			s.bwrap[s.ip] = 0
+		}
+	}
+
+	s.ip++
+	if s.ip == s.bsize {
+		s.ip = 0
+	}
+	s.bp++
+	if s.bp == s.bsize {
+		s.bp = 0
+	}
+	s.rp++
+	if s.rp == s.bsize {
+		s.rp = 0
+	}
+}
+
+func (s *swd) accept(n uint) {
+	if n > s.Look {
+		panic("swd: accept: invalid n")
+	}
+
+	for i := uint(0); i < n; i++ {
+		s.removeNode(s.rp)
+
+		key := head3(s.b[s.bp:])
+		s.succ3[s.bp] = s.gethead3(key)
+		s.head3[key] = uint16(s.bp)
+		s.best3[s.bp] = uint16(s.SwdF + 1)
+		s.llen3[key]++
+		if uint(s.llen3[key]) > s.SwdN {
+			panic("swd: accept: invalid llen3")
+		}
+
+		key = head2(s.b[s.bp:])
+		s.head2[key] = uint16(s.bp)
+
+		s.getbyte()
+	}
+}
+
+func (s *swd) search(node uint, cnt uint) {
+	if s.MLen <= 0 {
+		panic("assert: search: invalid mlen")
+	}
+
+	mlen := s.MLen
+	bp := s.bp
+	bx := s.bp + s.Look
+
+	scanend1 := s.b[s.bp+mlen-1]
+	for ; cnt > 0; cnt-- {
+		p1 := bp
+		p2 := node
+		px := bx
+
+		if mlen >= s.Look {
+			panic("assert: search: invalid mlen in loop")
+		}
+		if s.b[p2+mlen-1] == scanend1 &&
+			s.b[p2+mlen] == s.b[p1+mlen] &&
+			s.b[p2] == s.b[p1] &&
+			s.b[p2+1] == s.b[p1+1] {
+
+			if s.b[bp] != s.b[node] || s.b[bp+1] != s.b[node+1] || s.b[bp+2] != s.b[node+2] {
+				panic("assert: seach: invalid initial match")
+			}
+			p1 = p1 + 2
+			p2 = p2 + 2
+			for p1 < px {
+				p1++
+				p2++
+				if s.b[p1] != s.b[p2] {
+					break
+				}
+			}
+			i := p1 - bp
+
+			for j := uint(0); j < i; j++ {
+				if s.b[s.bp+j] != s.b[node+j] {
+					panic("assert: search: invalid final match")
+				}
+			}
+
+			if i < cSWD_BEST_OFF {
+				if s.bestPos[i] == 0 {
+					s.bestPos[i] = node + 1
+				}
+			}
+			if i > mlen {
+				mlen = i
+				s.MLen = mlen
+				s.mpos = node
+				if mlen == s.Look {
+					return
+				}
+				if mlen >= s.NiceLength {
+					return
+				}
+				if mlen > uint(s.best3[node]) {
+					return
+				}
+				scanend1 = s.b[s.bp+mlen-1]
+			}
+		}
+
+		node = uint(s.succ3[node])
+	}
+}
+
+func (s *swd) search2() bool {
+	if s.Look < 2 {
+		panic("assert: search2: invalid look")
+	}
+	if s.MLen <= 0 {
+		panic("assert: search2: invalid mlen")
+	}
+
+	key := s.head2[head2(s.b[s.bp:])]
+	if key == 0xFFFF {
+		return false
+	}
+	if s.b[s.bp] != s.b[key] || s.b[s.bp+1] != s.b[key+1] {
+		panic("assert: search2: invalid key found")
+	}
+	if s.bestPos[2] == 0 {
+		s.bestPos[2] = uint(key + 1)
+	}
+	if s.MLen < 2 {
+		s.MLen = 2
+		s.mpos = uint(key)
+	}
+	return true
+}
+
+func (s *swd) findbest() {
+	if s.MLen == 0 {
+		panic("swd: findbest: invalid mlen")
+	}
+
+	key := head3(s.b[s.bp:])
+	node := s.gethead3(key)
+	s.succ3[s.bp] = node
+	cnt := uint(s.llen3[key])
+	s.llen3[key]++
+	if cnt > s.SwdN+s.SwdF {
+		panic("swd: findbest: invalid llen3")
+	}
+	if cnt > s.MaxChain && s.MaxChain > 0 {
+		cnt = s.MaxChain
+	}
+	s.head3[key] = uint16(s.bp)
+
+	s.BChar = int(s.b[s.bp])
+	len := s.MLen
+	if s.MLen >= s.Look {
+		if s.Look == 0 {
+			s.BChar = -1
+		}
+		s.MOff = 0
+		s.best3[s.bp] = uint16(s.SwdF + 1)
+	} else {
+		if s.search2() && s.Look >= 3 {
+			s.search(uint(node), cnt)
+		}
+
+		if s.MLen > len {
+			s.MOff = s.pos2off(s.mpos)
+		}
+
+		if s.UseBestOff {
+			for i := 2; i < cSWD_BEST_OFF; i++ {
+				if s.bestPos[i] > 0 {
+					s.BestOff[i] = s.pos2off(s.bestPos[i] - 1)
+				} else {
+					s.BestOff[i] = 0
+				}
+			}
+		}
+	}
+
+	s.removeNode(s.rp)
+	key = head2(s.b[s.bp:])
+	s.head2[key] = uint16(s.bp)
+}
+
+func (s *swd) pos2off(pos uint) uint {
+	if s.bp > pos {
+		return s.bp - pos
+	}
+	return s.bsize - (pos - s.bp)
+}