mirror of
https://github.com/pgsty/minio.git
synced 2026-08-06 15:51:14 +03:00
Add support for SSE-S3 server side encryption with vault (#6192)
Add support for sse-s3 encryption with vault as KMS. Also refactoring code to make use of headers and functions defined in crypto package and clean up duplicated code.
This commit is contained in:
poornas
+192
@@ -0,0 +1,192 @@
|
||||
package compressutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"compress/lzw"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/golang/snappy"
|
||||
"github.com/hashicorp/errwrap"
|
||||
)
|
||||
|
||||
const (
|
||||
// A byte value used as a canary prefix for the compressed information
|
||||
// which is used to distinguish if a JSON input is compressed or not.
|
||||
// The value of this constant should not be a first character of any
|
||||
// valid JSON string.
|
||||
|
||||
// Byte value used as canary when using Gzip format
|
||||
CompressionCanaryGzip byte = 'G'
|
||||
|
||||
// Byte value used as canary when using Lzw format
|
||||
CompressionCanaryLzw byte = 'L'
|
||||
|
||||
// Byte value used as canary when using Snappy format
|
||||
CompressionCanarySnappy byte = 'S'
|
||||
|
||||
CompressionTypeLzw = "lzw"
|
||||
|
||||
CompressionTypeGzip = "gzip"
|
||||
|
||||
CompressionTypeSnappy = "snappy"
|
||||
)
|
||||
|
||||
// SnappyReadCloser embeds the snappy reader which implements the io.Reader
|
||||
// interface. The decompress procedure in this utility expects an
|
||||
// io.ReadCloser. This type implements the io.Closer interface to retain the
|
||||
// generic way of decompression.
|
||||
type SnappyReadCloser struct {
|
||||
*snappy.Reader
|
||||
}
|
||||
|
||||
// Close is a noop method implemented only to satisfy the io.Closer interface
|
||||
func (s *SnappyReadCloser) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// CompressionConfig is used to select a compression type to be performed by
|
||||
// Compress and Decompress utilities.
|
||||
// Supported types are:
|
||||
// * CompressionTypeLzw
|
||||
// * CompressionTypeGzip
|
||||
// * CompressionTypeSnappy
|
||||
//
|
||||
// When using CompressionTypeGzip, the compression levels can also be chosen:
|
||||
// * gzip.DefaultCompression
|
||||
// * gzip.BestSpeed
|
||||
// * gzip.BestCompression
|
||||
type CompressionConfig struct {
|
||||
// Type of the compression algorithm to be used
|
||||
Type string
|
||||
|
||||
// When using Gzip format, the compression level to employ
|
||||
GzipCompressionLevel int
|
||||
}
|
||||
|
||||
// Compress places the canary byte in a buffer and uses the same buffer to fill
|
||||
// in the compressed information of the given input. The configuration supports
|
||||
// two type of compression: LZW and Gzip. When using Gzip compression format,
|
||||
// if GzipCompressionLevel is not specified, the 'gzip.DefaultCompression' will
|
||||
// be assumed.
|
||||
func Compress(data []byte, config *CompressionConfig) ([]byte, error) {
|
||||
var buf bytes.Buffer
|
||||
var writer io.WriteCloser
|
||||
var err error
|
||||
|
||||
if config == nil {
|
||||
return nil, fmt.Errorf("config is nil")
|
||||
}
|
||||
|
||||
// Write the canary into the buffer and create writer to compress the
|
||||
// input data based on the configured type
|
||||
switch config.Type {
|
||||
case CompressionTypeLzw:
|
||||
buf.Write([]byte{CompressionCanaryLzw})
|
||||
|
||||
writer = lzw.NewWriter(&buf, lzw.LSB, 8)
|
||||
case CompressionTypeGzip:
|
||||
buf.Write([]byte{CompressionCanaryGzip})
|
||||
|
||||
switch {
|
||||
case config.GzipCompressionLevel == gzip.BestCompression,
|
||||
config.GzipCompressionLevel == gzip.BestSpeed,
|
||||
config.GzipCompressionLevel == gzip.DefaultCompression:
|
||||
// These are valid compression levels
|
||||
default:
|
||||
// If compression level is set to NoCompression or to
|
||||
// any invalid value, fallback to Defaultcompression
|
||||
config.GzipCompressionLevel = gzip.DefaultCompression
|
||||
}
|
||||
writer, err = gzip.NewWriterLevel(&buf, config.GzipCompressionLevel)
|
||||
case CompressionTypeSnappy:
|
||||
buf.Write([]byte{CompressionCanarySnappy})
|
||||
writer = snappy.NewBufferedWriter(&buf)
|
||||
default:
|
||||
return nil, fmt.Errorf("unsupported compression type")
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, errwrap.Wrapf("failed to create a compression writer: {{err}}", err)
|
||||
}
|
||||
|
||||
if writer == nil {
|
||||
return nil, fmt.Errorf("failed to create a compression writer")
|
||||
}
|
||||
|
||||
// Compress the input and place it in the same buffer containing the
|
||||
// canary byte.
|
||||
if _, err = writer.Write(data); err != nil {
|
||||
return nil, errwrap.Wrapf("failed to compress input data: err: {{err}}", err)
|
||||
}
|
||||
|
||||
// Close the io.WriteCloser
|
||||
if err = writer.Close(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Return the compressed bytes with canary byte at the start
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// Decompress checks if the first byte in the input matches the canary byte.
|
||||
// If the first byte is a canary byte, then the input past the canary byte
|
||||
// will be decompressed using the method specified in the given configuration.
|
||||
// If the first byte isn't a canary byte, then the utility returns a boolean
|
||||
// value indicating that the input was not compressed.
|
||||
func Decompress(data []byte) ([]byte, bool, error) {
|
||||
var err error
|
||||
var reader io.ReadCloser
|
||||
if data == nil || len(data) == 0 {
|
||||
return nil, false, fmt.Errorf("'data' being decompressed is empty")
|
||||
}
|
||||
|
||||
switch {
|
||||
// If the first byte matches the canary byte, remove the canary
|
||||
// byte and try to decompress the data that is after the canary.
|
||||
case data[0] == CompressionCanaryGzip:
|
||||
if len(data) < 2 {
|
||||
return nil, false, fmt.Errorf("invalid 'data' after the canary")
|
||||
}
|
||||
data = data[1:]
|
||||
reader, err = gzip.NewReader(bytes.NewReader(data))
|
||||
case data[0] == CompressionCanaryLzw:
|
||||
if len(data) < 2 {
|
||||
return nil, false, fmt.Errorf("invalid 'data' after the canary")
|
||||
}
|
||||
data = data[1:]
|
||||
reader = lzw.NewReader(bytes.NewReader(data), lzw.LSB, 8)
|
||||
|
||||
case data[0] == CompressionCanarySnappy:
|
||||
if len(data) < 2 {
|
||||
return nil, false, fmt.Errorf("invalid 'data' after the canary")
|
||||
}
|
||||
data = data[1:]
|
||||
reader = &SnappyReadCloser{
|
||||
Reader: snappy.NewReader(bytes.NewReader(data)),
|
||||
}
|
||||
default:
|
||||
// If the first byte doesn't match the canary byte, it means
|
||||
// that the content was not compressed at all. Indicate the
|
||||
// caller that the input was not compressed.
|
||||
return nil, true, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, false, errwrap.Wrapf("failed to create a compression reader: {{err}}", err)
|
||||
}
|
||||
if reader == nil {
|
||||
return nil, false, fmt.Errorf("failed to create a compression reader")
|
||||
}
|
||||
|
||||
// Close the io.ReadCloser
|
||||
defer reader.Close()
|
||||
|
||||
// Read all the compressed data into a buffer
|
||||
var buf bytes.Buffer
|
||||
if _, err = io.Copy(&buf, reader); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
|
||||
return buf.Bytes(), false, nil
|
||||
}
|
||||
+36
@@ -0,0 +1,36 @@
|
||||
package hclutil
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
multierror "github.com/hashicorp/go-multierror"
|
||||
"github.com/hashicorp/hcl/hcl/ast"
|
||||
)
|
||||
|
||||
// CheckHCLKeys checks whether the keys in the AST list contains any of the valid keys provided.
|
||||
func CheckHCLKeys(node ast.Node, valid []string) error {
|
||||
var list *ast.ObjectList
|
||||
switch n := node.(type) {
|
||||
case *ast.ObjectList:
|
||||
list = n
|
||||
case *ast.ObjectType:
|
||||
list = n.List
|
||||
default:
|
||||
return fmt.Errorf("cannot check HCL keys of type %T", n)
|
||||
}
|
||||
|
||||
validMap := make(map[string]struct{}, len(valid))
|
||||
for _, v := range valid {
|
||||
validMap[v] = struct{}{}
|
||||
}
|
||||
|
||||
var result error
|
||||
for _, item := range list.Items {
|
||||
key := item.Keys[0].Token.Value().(string)
|
||||
if _, ok := validMap[key]; !ok {
|
||||
result = multierror.Append(result, fmt.Errorf("invalid key %q on line %d", key, item.Assign.Line))
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
package jsonutil
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
|
||||
"github.com/hashicorp/errwrap"
|
||||
"github.com/hashicorp/vault/helper/compressutil"
|
||||
)
|
||||
|
||||
// Encodes/Marshals the given object into JSON
|
||||
func EncodeJSON(in interface{}) ([]byte, error) {
|
||||
if in == nil {
|
||||
return nil, fmt.Errorf("input for encoding is nil")
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
enc := json.NewEncoder(&buf)
|
||||
if err := enc.Encode(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
// EncodeJSONAndCompress encodes the given input into JSON and compresses the
|
||||
// encoded value (using Gzip format BestCompression level, by default). A
|
||||
// canary byte is placed at the beginning of the returned bytes for the logic
|
||||
// in decompression method to identify compressed input.
|
||||
func EncodeJSONAndCompress(in interface{}, config *compressutil.CompressionConfig) ([]byte, error) {
|
||||
if in == nil {
|
||||
return nil, fmt.Errorf("input for encoding is nil")
|
||||
}
|
||||
|
||||
// First JSON encode the given input
|
||||
encodedBytes, err := EncodeJSON(in)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if config == nil {
|
||||
config = &compressutil.CompressionConfig{
|
||||
Type: compressutil.CompressionTypeGzip,
|
||||
GzipCompressionLevel: gzip.BestCompression,
|
||||
}
|
||||
}
|
||||
|
||||
return compressutil.Compress(encodedBytes, config)
|
||||
}
|
||||
|
||||
// DecodeJSON tries to decompress the given data. The call to decompress, fails
|
||||
// if the content was not compressed in the first place, which is identified by
|
||||
// a canary byte before the compressed data. If the data is not compressed, it
|
||||
// is JSON decoded directly. Otherwise the decompressed data will be JSON
|
||||
// decoded.
|
||||
func DecodeJSON(data []byte, out interface{}) error {
|
||||
if data == nil || len(data) == 0 {
|
||||
return fmt.Errorf("'data' being decoded is nil")
|
||||
}
|
||||
if out == nil {
|
||||
return fmt.Errorf("output parameter 'out' is nil")
|
||||
}
|
||||
|
||||
// Decompress the data if it was compressed in the first place
|
||||
decompressedBytes, uncompressed, err := compressutil.Decompress(data)
|
||||
if err != nil {
|
||||
return errwrap.Wrapf("failed to decompress JSON: {{err}}", err)
|
||||
}
|
||||
if !uncompressed && (decompressedBytes == nil || len(decompressedBytes) == 0) {
|
||||
return fmt.Errorf("decompressed data being decoded is invalid")
|
||||
}
|
||||
|
||||
// If the input supplied failed to contain the compression canary, it
|
||||
// will be notified by the compression utility. Decode the decompressed
|
||||
// input.
|
||||
if !uncompressed {
|
||||
data = decompressedBytes
|
||||
}
|
||||
|
||||
return DecodeJSONFromReader(bytes.NewReader(data), out)
|
||||
}
|
||||
|
||||
// Decodes/Unmarshals the given io.Reader pointing to a JSON, into a desired object
|
||||
func DecodeJSONFromReader(r io.Reader, out interface{}) error {
|
||||
if r == nil {
|
||||
return fmt.Errorf("'io.Reader' being decoded is nil")
|
||||
}
|
||||
if out == nil {
|
||||
return fmt.Errorf("output parameter 'out' is nil")
|
||||
}
|
||||
|
||||
dec := json.NewDecoder(r)
|
||||
|
||||
// While decoding JSON values, interpret the integer values as `json.Number`s instead of `float64`.
|
||||
dec.UseNumber()
|
||||
|
||||
// Since 'out' is an interface representing a pointer, pass it to the decoder without an '&'
|
||||
return dec.Decode(out)
|
||||
}
|
||||
+163
@@ -0,0 +1,163 @@
|
||||
package parseutil
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/hashicorp/errwrap"
|
||||
sockaddr "github.com/hashicorp/go-sockaddr"
|
||||
"github.com/hashicorp/vault/helper/strutil"
|
||||
"github.com/mitchellh/mapstructure"
|
||||
)
|
||||
|
||||
func ParseDurationSecond(in interface{}) (time.Duration, error) {
|
||||
var dur time.Duration
|
||||
jsonIn, ok := in.(json.Number)
|
||||
if ok {
|
||||
in = jsonIn.String()
|
||||
}
|
||||
switch in.(type) {
|
||||
case string:
|
||||
inp := in.(string)
|
||||
if inp == "" {
|
||||
return time.Duration(0), nil
|
||||
}
|
||||
var err error
|
||||
// Look for a suffix otherwise its a plain second value
|
||||
if strings.HasSuffix(inp, "s") || strings.HasSuffix(inp, "m") || strings.HasSuffix(inp, "h") || strings.HasSuffix(inp, "ms") {
|
||||
dur, err = time.ParseDuration(inp)
|
||||
if err != nil {
|
||||
return dur, err
|
||||
}
|
||||
} else {
|
||||
// Plain integer
|
||||
secs, err := strconv.ParseInt(inp, 10, 64)
|
||||
if err != nil {
|
||||
return dur, err
|
||||
}
|
||||
dur = time.Duration(secs) * time.Second
|
||||
}
|
||||
case int:
|
||||
dur = time.Duration(in.(int)) * time.Second
|
||||
case int32:
|
||||
dur = time.Duration(in.(int32)) * time.Second
|
||||
case int64:
|
||||
dur = time.Duration(in.(int64)) * time.Second
|
||||
case uint:
|
||||
dur = time.Duration(in.(uint)) * time.Second
|
||||
case uint32:
|
||||
dur = time.Duration(in.(uint32)) * time.Second
|
||||
case uint64:
|
||||
dur = time.Duration(in.(uint64)) * time.Second
|
||||
default:
|
||||
return 0, errors.New("could not parse duration from input")
|
||||
}
|
||||
|
||||
return dur, nil
|
||||
}
|
||||
|
||||
func ParseInt(in interface{}) (int64, error) {
|
||||
var ret int64
|
||||
jsonIn, ok := in.(json.Number)
|
||||
if ok {
|
||||
in = jsonIn.String()
|
||||
}
|
||||
switch in.(type) {
|
||||
case string:
|
||||
inp := in.(string)
|
||||
if inp == "" {
|
||||
return 0, nil
|
||||
}
|
||||
var err error
|
||||
left, err := strconv.ParseInt(inp, 10, 64)
|
||||
if err != nil {
|
||||
return ret, err
|
||||
}
|
||||
ret = left
|
||||
case int:
|
||||
ret = int64(in.(int))
|
||||
case int32:
|
||||
ret = int64(in.(int32))
|
||||
case int64:
|
||||
ret = in.(int64)
|
||||
case uint:
|
||||
ret = int64(in.(uint))
|
||||
case uint32:
|
||||
ret = int64(in.(uint32))
|
||||
case uint64:
|
||||
ret = int64(in.(uint64))
|
||||
default:
|
||||
return 0, errors.New("could not parse value from input")
|
||||
}
|
||||
|
||||
return ret, nil
|
||||
}
|
||||
|
||||
func ParseBool(in interface{}) (bool, error) {
|
||||
var result bool
|
||||
if err := mapstructure.WeakDecode(in, &result); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func ParseCommaStringSlice(in interface{}) ([]string, error) {
|
||||
var result []string
|
||||
config := &mapstructure.DecoderConfig{
|
||||
Result: &result,
|
||||
WeaklyTypedInput: true,
|
||||
DecodeHook: mapstructure.StringToSliceHookFunc(","),
|
||||
}
|
||||
decoder, err := mapstructure.NewDecoder(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := decoder.Decode(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return strutil.TrimStrings(result), nil
|
||||
}
|
||||
|
||||
func ParseAddrs(addrs interface{}) ([]*sockaddr.SockAddrMarshaler, error) {
|
||||
out := make([]*sockaddr.SockAddrMarshaler, 0)
|
||||
stringAddrs := make([]string, 0)
|
||||
|
||||
switch addrs.(type) {
|
||||
case string:
|
||||
stringAddrs = strutil.ParseArbitraryStringSlice(addrs.(string), ",")
|
||||
if len(stringAddrs) == 0 {
|
||||
return nil, fmt.Errorf("unable to parse addresses from %v", addrs)
|
||||
}
|
||||
|
||||
case []string:
|
||||
stringAddrs = addrs.([]string)
|
||||
|
||||
case []interface{}:
|
||||
for _, v := range addrs.([]interface{}) {
|
||||
stringAddr, ok := v.(string)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("error parsing %v as string", v)
|
||||
}
|
||||
stringAddrs = append(stringAddrs, stringAddr)
|
||||
}
|
||||
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown address input type %T", addrs)
|
||||
}
|
||||
|
||||
for _, addr := range stringAddrs {
|
||||
sa, err := sockaddr.NewSockAddr(addr)
|
||||
if err != nil {
|
||||
return nil, errwrap.Wrapf(fmt.Sprintf("error parsing address %q: {{err}}", addr), err)
|
||||
}
|
||||
out = append(out, &sockaddr.SockAddrMarshaler{
|
||||
SockAddr: sa,
|
||||
})
|
||||
}
|
||||
|
||||
return out, nil
|
||||
}
|
||||
+327
@@ -0,0 +1,327 @@
|
||||
package strutil
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/hashicorp/errwrap"
|
||||
glob "github.com/ryanuber/go-glob"
|
||||
)
|
||||
|
||||
// StrListContainsGlob looks for a string in a list of strings and allows
|
||||
// globs.
|
||||
func StrListContainsGlob(haystack []string, needle string) bool {
|
||||
for _, item := range haystack {
|
||||
if glob.Glob(item, needle) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// StrListContains looks for a string in a list of strings.
|
||||
func StrListContains(haystack []string, needle string) bool {
|
||||
for _, item := range haystack {
|
||||
if item == needle {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// StrListSubset checks if a given list is a subset
|
||||
// of another set
|
||||
func StrListSubset(super, sub []string) bool {
|
||||
for _, item := range sub {
|
||||
if !StrListContains(super, item) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// Parses a comma separated list of strings into a slice of strings.
|
||||
// The return slice will be sorted and will not contain duplicate or
|
||||
// empty items.
|
||||
func ParseDedupAndSortStrings(input string, sep string) []string {
|
||||
input = strings.TrimSpace(input)
|
||||
parsed := []string{}
|
||||
if input == "" {
|
||||
// Don't return nil
|
||||
return parsed
|
||||
}
|
||||
return RemoveDuplicates(strings.Split(input, sep), false)
|
||||
}
|
||||
|
||||
// Parses a comma separated list of strings into a slice of strings.
|
||||
// The return slice will be sorted and will not contain duplicate or
|
||||
// empty items. The values will be converted to lower case.
|
||||
func ParseDedupLowercaseAndSortStrings(input string, sep string) []string {
|
||||
input = strings.TrimSpace(input)
|
||||
parsed := []string{}
|
||||
if input == "" {
|
||||
// Don't return nil
|
||||
return parsed
|
||||
}
|
||||
return RemoveDuplicates(strings.Split(input, sep), true)
|
||||
}
|
||||
|
||||
// Parses a comma separated list of `<key>=<value>` tuples into a
|
||||
// map[string]string.
|
||||
func ParseKeyValues(input string, out map[string]string, sep string) error {
|
||||
if out == nil {
|
||||
return fmt.Errorf("'out is nil")
|
||||
}
|
||||
|
||||
keyValues := ParseDedupLowercaseAndSortStrings(input, sep)
|
||||
if len(keyValues) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
for _, keyValue := range keyValues {
|
||||
shards := strings.Split(keyValue, "=")
|
||||
if len(shards) != 2 {
|
||||
return fmt.Errorf("invalid <key,value> format")
|
||||
}
|
||||
|
||||
key := strings.TrimSpace(shards[0])
|
||||
value := strings.TrimSpace(shards[1])
|
||||
if key == "" || value == "" {
|
||||
return fmt.Errorf("invalid <key,value> pair: key: %q value: %q", key, value)
|
||||
}
|
||||
out[key] = value
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parses arbitrary <key,value> tuples. The input can be one of
|
||||
// the following:
|
||||
// * JSON string
|
||||
// * Base64 encoded JSON string
|
||||
// * Comma separated list of `<key>=<value>` pairs
|
||||
// * Base64 encoded string containing comma separated list of
|
||||
// `<key>=<value>` pairs
|
||||
//
|
||||
// Input will be parsed into the output parameter, which should
|
||||
// be a non-nil map[string]string.
|
||||
func ParseArbitraryKeyValues(input string, out map[string]string, sep string) error {
|
||||
input = strings.TrimSpace(input)
|
||||
if input == "" {
|
||||
return nil
|
||||
}
|
||||
if out == nil {
|
||||
return fmt.Errorf("'out' is nil")
|
||||
}
|
||||
|
||||
// Try to base64 decode the input. If successful, consider the decoded
|
||||
// value as input.
|
||||
inputBytes, err := base64.StdEncoding.DecodeString(input)
|
||||
if err == nil {
|
||||
input = string(inputBytes)
|
||||
}
|
||||
|
||||
// Try to JSON unmarshal the input. If successful, consider that the
|
||||
// metadata was supplied as JSON input.
|
||||
err = json.Unmarshal([]byte(input), &out)
|
||||
if err != nil {
|
||||
// If JSON unmarshalling fails, consider that the input was
|
||||
// supplied as a comma separated string of 'key=value' pairs.
|
||||
if err = ParseKeyValues(input, out, sep); err != nil {
|
||||
return errwrap.Wrapf("failed to parse the input: {{err}}", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the parsed input
|
||||
for key, value := range out {
|
||||
if key != "" && value == "" {
|
||||
return fmt.Errorf("invalid value for key %q", key)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Parses a `sep`-separated list of strings into a
|
||||
// []string.
|
||||
//
|
||||
// The output will always be a valid slice but may be of length zero.
|
||||
func ParseStringSlice(input string, sep string) []string {
|
||||
input = strings.TrimSpace(input)
|
||||
if input == "" {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
splitStr := strings.Split(input, sep)
|
||||
ret := make([]string, len(splitStr))
|
||||
for i, val := range splitStr {
|
||||
ret[i] = val
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// Parses arbitrary string slice. The input can be one of
|
||||
// the following:
|
||||
// * JSON string
|
||||
// * Base64 encoded JSON string
|
||||
// * `sep` separated list of values
|
||||
// * Base64-encoded string containing a `sep` separated list of values
|
||||
//
|
||||
// Note that the separator is ignored if the input is found to already be in a
|
||||
// structured format (e.g., JSON)
|
||||
//
|
||||
// The output will always be a valid slice but may be of length zero.
|
||||
func ParseArbitraryStringSlice(input string, sep string) []string {
|
||||
input = strings.TrimSpace(input)
|
||||
if input == "" {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
// Try to base64 decode the input. If successful, consider the decoded
|
||||
// value as input.
|
||||
inputBytes, err := base64.StdEncoding.DecodeString(input)
|
||||
if err == nil {
|
||||
input = string(inputBytes)
|
||||
}
|
||||
|
||||
ret := []string{}
|
||||
|
||||
// Try to JSON unmarshal the input. If successful, consider that the
|
||||
// metadata was supplied as JSON input.
|
||||
err = json.Unmarshal([]byte(input), &ret)
|
||||
if err != nil {
|
||||
// If JSON unmarshalling fails, consider that the input was
|
||||
// supplied as a separated string of values.
|
||||
return ParseStringSlice(input, sep)
|
||||
}
|
||||
|
||||
if ret == nil {
|
||||
return []string{}
|
||||
}
|
||||
|
||||
return ret
|
||||
}
|
||||
|
||||
// TrimStrings takes a slice of strings and returns a slice of strings
|
||||
// with trimmed spaces
|
||||
func TrimStrings(items []string) []string {
|
||||
ret := make([]string, len(items))
|
||||
for i, item := range items {
|
||||
ret[i] = strings.TrimSpace(item)
|
||||
}
|
||||
return ret
|
||||
}
|
||||
|
||||
// Removes duplicate and empty elements from a slice of strings. This also may
|
||||
// convert the items in the slice to lower case and returns a sorted slice.
|
||||
func RemoveDuplicates(items []string, lowercase bool) []string {
|
||||
itemsMap := map[string]bool{}
|
||||
for _, item := range items {
|
||||
item = strings.TrimSpace(item)
|
||||
if lowercase {
|
||||
item = strings.ToLower(item)
|
||||
}
|
||||
if item == "" {
|
||||
continue
|
||||
}
|
||||
itemsMap[item] = true
|
||||
}
|
||||
items = make([]string, 0, len(itemsMap))
|
||||
for item, _ := range itemsMap {
|
||||
items = append(items, item)
|
||||
}
|
||||
sort.Strings(items)
|
||||
return items
|
||||
}
|
||||
|
||||
// EquivalentSlices checks whether the given string sets are equivalent, as in,
|
||||
// they contain the same values.
|
||||
func EquivalentSlices(a, b []string) bool {
|
||||
if a == nil && b == nil {
|
||||
return true
|
||||
}
|
||||
|
||||
if a == nil || b == nil {
|
||||
return false
|
||||
}
|
||||
|
||||
// First we'll build maps to ensure unique values
|
||||
mapA := map[string]bool{}
|
||||
mapB := map[string]bool{}
|
||||
for _, keyA := range a {
|
||||
mapA[keyA] = true
|
||||
}
|
||||
for _, keyB := range b {
|
||||
mapB[keyB] = true
|
||||
}
|
||||
|
||||
// Now we'll build our checking slices
|
||||
var sortedA, sortedB []string
|
||||
for keyA, _ := range mapA {
|
||||
sortedA = append(sortedA, keyA)
|
||||
}
|
||||
for keyB, _ := range mapB {
|
||||
sortedB = append(sortedB, keyB)
|
||||
}
|
||||
sort.Strings(sortedA)
|
||||
sort.Strings(sortedB)
|
||||
|
||||
// Finally, compare
|
||||
if len(sortedA) != len(sortedB) {
|
||||
return false
|
||||
}
|
||||
|
||||
for i := range sortedA {
|
||||
if sortedA[i] != sortedB[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
// StrListDelete removes the first occurrence of the given item from the slice
|
||||
// of strings if the item exists.
|
||||
func StrListDelete(s []string, d string) []string {
|
||||
if s == nil {
|
||||
return s
|
||||
}
|
||||
|
||||
for index, element := range s {
|
||||
if element == d {
|
||||
return append(s[:index], s[index+1:]...)
|
||||
}
|
||||
}
|
||||
|
||||
return s
|
||||
}
|
||||
|
||||
func GlobbedStringsMatch(item, val string) bool {
|
||||
if len(item) < 2 {
|
||||
return val == item
|
||||
}
|
||||
|
||||
hasPrefix := strings.HasPrefix(item, "*")
|
||||
hasSuffix := strings.HasSuffix(item, "*")
|
||||
|
||||
if hasPrefix && hasSuffix {
|
||||
return strings.Contains(val, item[1:len(item)-1])
|
||||
} else if hasPrefix {
|
||||
return strings.HasSuffix(val, item[1:])
|
||||
} else if hasSuffix {
|
||||
return strings.HasPrefix(val, item[:len(item)-1])
|
||||
}
|
||||
|
||||
return val == item
|
||||
}
|
||||
|
||||
// AppendIfMissing adds a string to a slice if the given string is not present
|
||||
func AppendIfMissing(slice []string, i string) []string {
|
||||
if StrListContains(slice, i) {
|
||||
return slice
|
||||
}
|
||||
return append(slice, i)
|
||||
}
|
||||
Reference in New Issue
Block a user