Update GCS storage library to support GetObject API on gzipped objects (#6506)

Fixes #6280

vendor/google.golang.org/api/gensupport/go18.go

@@ -0,0 +1,17 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.8
+
+package gensupport
+
+import (
+	"io"
+	"net/http"
+)
+
+// SetGetBody sets the GetBody field of req to f.
+func SetGetBody(req *http.Request, f func() (io.ReadCloser, error)) {
+	req.GetBody = f
+}

vendor/google.golang.org/api/gensupport/json.go

@@ -22,23 +22,33 @@ func MarshalJSON(schema interface{}, forceSendFields, nullFields []string) ([]by
 		return json.Marshal(schema)
 	}
 
-	mustInclude := make(map[string]struct{})
+	mustInclude := make(map[string]bool)
 	for _, f := range forceSendFields {
-		mustInclude[f] = struct{}{}
+		mustInclude[f] = true
 	}
-	useNull := make(map[string]struct{})
-	for _, f := range nullFields {
-		useNull[f] = struct{}{}
+	useNull := make(map[string]bool)
+	useNullMaps := make(map[string]map[string]bool)
+	for _, nf := range nullFields {
+		parts := strings.SplitN(nf, ".", 2)
+		field := parts[0]
+		if len(parts) == 1 {
+			useNull[field] = true
+		} else {
+			if useNullMaps[field] == nil {
+				useNullMaps[field] = map[string]bool{}
+			}
+			useNullMaps[field][parts[1]] = true
+		}
 	}
 
-	dataMap, err := schemaToMap(schema, mustInclude, useNull)
+	dataMap, err := schemaToMap(schema, mustInclude, useNull, useNullMaps)
 	if err != nil {
 		return nil, err
 	}
 	return json.Marshal(dataMap)
 }
 
-func schemaToMap(schema interface{}, mustInclude, useNull map[string]struct{}) (map[string]interface{}, error) {
+func schemaToMap(schema interface{}, mustInclude, useNull map[string]bool, useNullMaps map[string]map[string]bool) (map[string]interface{}, error) {
 	m := make(map[string]interface{})
 	s := reflect.ValueOf(schema)
 	st := s.Type()
@@ -59,17 +69,35 @@ func schemaToMap(schema interface{}, mustInclude, useNull map[string]struct{}) (
 		v := s.Field(i)
 		f := st.Field(i)
 
-		if _, ok := useNull[f.Name]; ok {
+		if useNull[f.Name] {
 			if !isEmptyValue(v) {
 				return nil, fmt.Errorf("field %q in NullFields has non-empty value", f.Name)
 			}
 			m[tag.apiName] = nil
 			continue
 		}
+
 		if !includeField(v, f, mustInclude) {
 			continue
 		}
 
+		// If map fields are explicitly set to null, use a map[string]interface{}.
+		if f.Type.Kind() == reflect.Map && useNullMaps[f.Name] != nil {
+			ms, ok := v.Interface().(map[string]string)
+			if !ok {
+				return nil, fmt.Errorf("field %q has keys in NullFields but is not a map[string]string", f.Name)
+			}
+			mi := map[string]interface{}{}
+			for k, v := range ms {
+				mi[k] = v
+			}
+			for k := range useNullMaps[f.Name] {
+				mi[k] = nil
+			}
+			m[tag.apiName] = mi
+			continue
+		}
+
 		// nil maps are treated as empty maps.
 		if f.Type.Kind() == reflect.Map && v.IsNil() {
 			m[tag.apiName] = map[string]string{}
@@ -139,7 +167,7 @@ func parseJSONTag(val string) (jsonTag, error) {
 }
 
 // Reports whether the struct field "f" with value "v" should be included in JSON output.
-func includeField(v reflect.Value, f reflect.StructField, mustInclude map[string]struct{}) bool {
+func includeField(v reflect.Value, f reflect.StructField, mustInclude map[string]bool) bool {
 	// The regular JSON encoding of a nil pointer is "null", which means "delete this field".
 	// Therefore, we could enable field deletion by honoring pointer fields' presence in the mustInclude set.
 	// However, many fields are not pointers, so there would be no way to delete these fields.
@@ -156,8 +184,7 @@ func includeField(v reflect.Value, f reflect.StructField, mustInclude map[string
 		return false
 	}
 
-	_, ok := mustInclude[f.Name]
-	return ok || !isEmptyValue(v)
+	return mustInclude[f.Name] || !isEmptyValue(v)
 }
 
 // isEmptyValue reports whether v is the empty value for its type.  This

vendor/google.golang.org/api/gensupport/jsonfloat.go

@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
+// Copyright 2016 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

vendor/google.golang.org/api/gensupport/media.go

@@ -5,12 +5,15 @@
 package gensupport
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
+	"strings"
+	"sync"
 
 	"google.golang.org/api/googleapi"
 )
@@ -103,12 +106,13 @@ type typeReader struct {
 	typ string
 }
 
-// multipartReader combines the contents of multiple readers to creat a multipart/related HTTP body.
+// multipartReader combines the contents of multiple readers to create a multipart/related HTTP body.
 // Close must be called if reads from the multipartReader are abandoned before reaching EOF.
 type multipartReader struct {
 	pr       *io.PipeReader
-	pipeOpen bool
 	ctype    string
+	mu       sync.Mutex
+	pipeOpen bool
 }
 
 func newMultipartReader(parts []typeReader) *multipartReader {
@@ -144,10 +148,13 @@ func (mp *multipartReader) Read(data []byte) (n int, err error) {
 }
 
 func (mp *multipartReader) Close() error {
+	mp.mu.Lock()
 	if !mp.pipeOpen {
+		mp.mu.Unlock()
 		return nil
 	}
 	mp.pipeOpen = false
+	mp.mu.Unlock()
 	return mp.pr.Close()
 }
 
@@ -174,26 +181,156 @@ func typeHeader(contentType string) textproto.MIMEHeader {
 // PrepareUpload determines whether the data in the supplied reader should be
 // uploaded in a single request, or in sequential chunks.
 // chunkSize is the size of the chunk that media should be split into.
-// If chunkSize is non-zero and the contents of media do not fit in a single
-// chunk (or there is an error reading media), then media will be returned as a
-// MediaBuffer.  Otherwise, media will be returned as a Reader.
+//
+// If chunkSize is zero, media is returned as the first value, and the other
+// two return values are nil, true.
+//
+// Otherwise, a MediaBuffer is returned, along with a bool indicating whether the
+// contents of media fit in a single chunk.
 //
 // After PrepareUpload has been called, media should no longer be used: the
 // media content should be accessed via one of the return values.
-func PrepareUpload(media io.Reader, chunkSize int) (io.Reader, *MediaBuffer) {
+func PrepareUpload(media io.Reader, chunkSize int) (r io.Reader, mb *MediaBuffer, singleChunk bool) {
 	if chunkSize == 0 { // do not chunk
-		return media, nil
+		return media, nil, true
+	}
+	mb = NewMediaBuffer(media, chunkSize)
+	_, _, _, err := mb.Chunk()
+	// If err is io.EOF, we can upload this in a single request. Otherwise, err is
+	// either nil or a non-EOF error. If it is the latter, then the next call to
+	// mb.Chunk will return the same error. Returning a MediaBuffer ensures that this
+	// error will be handled at some point.
+	return nil, mb, err == io.EOF
+}
+
+// MediaInfo holds information for media uploads. It is intended for use by generated
+// code only.
+type MediaInfo struct {
+	// At most one of Media and MediaBuffer will be set.
+	media           io.Reader
+	buffer          *MediaBuffer
+	singleChunk     bool
+	mType           string
+	size            int64 // mediaSize, if known.  Used only for calls to progressUpdater_.
+	progressUpdater googleapi.ProgressUpdater
+}
+
+// NewInfoFromMedia should be invoked from the Media method of a call. It returns a
+// MediaInfo populated with chunk size and content type, and a reader or MediaBuffer
+// if needed.
+func NewInfoFromMedia(r io.Reader, options []googleapi.MediaOption) *MediaInfo {
+	mi := &MediaInfo{}
+	opts := googleapi.ProcessMediaOptions(options)
+	if !opts.ForceEmptyContentType {
+		r, mi.mType = DetermineContentType(r, opts.ContentType)
+	}
+	mi.media, mi.buffer, mi.singleChunk = PrepareUpload(r, opts.ChunkSize)
+	return mi
+}
+
+// NewInfoFromResumableMedia should be invoked from the ResumableMedia method of a
+// call. It returns a MediaInfo using the given reader, size and media type.
+func NewInfoFromResumableMedia(r io.ReaderAt, size int64, mediaType string) *MediaInfo {
+	rdr := ReaderAtToReader(r, size)
+	rdr, mType := DetermineContentType(rdr, mediaType)
+	return &MediaInfo{
+		size:        size,
+		mType:       mType,
+		buffer:      NewMediaBuffer(rdr, googleapi.DefaultUploadChunkSize),
+		media:       nil,
+		singleChunk: false,
+	}
+}
+
+func (mi *MediaInfo) SetProgressUpdater(pu googleapi.ProgressUpdater) {
+	if mi != nil {
+		mi.progressUpdater = pu
+	}
+}
+
+// UploadType determines the type of upload: a single request, or a resumable
+// series of requests.
+func (mi *MediaInfo) UploadType() string {
+	if mi.singleChunk {
+		return "multipart"
+	}
+	return "resumable"
+}
+
+// UploadRequest sets up an HTTP request for media upload. It adds headers
+// as necessary, and returns a replacement for the body and a function for http.Request.GetBody.
+func (mi *MediaInfo) UploadRequest(reqHeaders http.Header, body io.Reader) (newBody io.Reader, getBody func() (io.ReadCloser, error), cleanup func()) {
+	cleanup = func() {}
+	if mi == nil {
+		return body, nil, cleanup
+	}
+	var media io.Reader
+	if mi.media != nil {
+		// This only happens when the caller has turned off chunking. In that
+		// case, we write all of media in a single non-retryable request.
+		media = mi.media
+	} else if mi.singleChunk {
+		// The data fits in a single chunk, which has now been read into the MediaBuffer.
+		// We obtain that chunk so we can write it in a single request. The request can
+		// be retried because the data is stored in the MediaBuffer.
+		media, _, _, _ = mi.buffer.Chunk()
+	}
+	if media != nil {
+		fb := readerFunc(body)
+		fm := readerFunc(media)
+		combined, ctype := CombineBodyMedia(body, "application/json", media, mi.mType)
+		if fb != nil && fm != nil {
+			getBody = func() (io.ReadCloser, error) {
+				rb := ioutil.NopCloser(fb())
+				rm := ioutil.NopCloser(fm())
+				r, _ := CombineBodyMedia(rb, "application/json", rm, mi.mType)
+				return r, nil
+			}
+		}
+		cleanup = func() { combined.Close() }
+		reqHeaders.Set("Content-Type", ctype)
+		body = combined
+	}
+	if mi.buffer != nil && mi.mType != "" && !mi.singleChunk {
+		reqHeaders.Set("X-Upload-Content-Type", mi.mType)
+	}
+	return body, getBody, cleanup
+}
+
+// readerFunc returns a function that always returns an io.Reader that has the same
+// contents as r, provided that can be done without consuming r. Otherwise, it
+// returns nil.
+// See http.NewRequest (in net/http/request.go).
+func readerFunc(r io.Reader) func() io.Reader {
+	switch r := r.(type) {
+	case *bytes.Buffer:
+		buf := r.Bytes()
+		return func() io.Reader { return bytes.NewReader(buf) }
+	case *bytes.Reader:
+		snapshot := *r
+		return func() io.Reader { r := snapshot; return &r }
+	case *strings.Reader:
+		snapshot := *r
+		return func() io.Reader { r := snapshot; return &r }
+	default:
+		return nil
+	}
+}
+
+// ResumableUpload returns an appropriately configured ResumableUpload value if the
+// upload is resumable, or nil otherwise.
+func (mi *MediaInfo) ResumableUpload(locURI string) *ResumableUpload {
+	if mi == nil || mi.singleChunk {
+		return nil
+	}
+	return &ResumableUpload{
+		URI:       locURI,
+		Media:     mi.buffer,
+		MediaType: mi.mType,
+		Callback: func(curr int64) {
+			if mi.progressUpdater != nil {
+				mi.progressUpdater(curr, mi.size)
+			}
+		},
 	}
-
-	mb := NewMediaBuffer(media, chunkSize)
-	rdr, _, _, err := mb.Chunk()
-
-	if err == io.EOF { // we can upload this in a single request
-		return rdr, nil
-	}
-	// err might be a non-EOF error. If it is, the next call to mb.Chunk will
-	// return the same error. Returning a MediaBuffer ensures that this error
-	// will be handled at some point.
-
-	return nil, mb
 }

vendor/google.golang.org/api/gensupport/not_go18.go

@@ -0,0 +1,14 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.8
+
+package gensupport
+
+import (
+	"io"
+	"net/http"
+)
+
+func SetGetBody(req *http.Request, f func() (io.ReadCloser, error)) {}

vendor/google.golang.org/api/gensupport/retry.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

vendor/google.golang.org/api/gensupport/send.go

@@ -5,6 +5,8 @@
 package gensupport
 
 import (
+	"encoding/json"
+	"errors"
 	"net/http"
 
 	"golang.org/x/net/context"
@@ -32,6 +34,11 @@ func RegisterHook(h Hook) {
 // If ctx is non-nil, it calls all hooks, then sends the request with
 // ctxhttp.Do, then calls any functions returned by the hooks in reverse order.
 func SendRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
+	// Disallow Accept-Encoding because it interferes with the automatic gzip handling
+	// done by the default http.Transport. See https://github.com/google/google-api-go-client/issues/219.
+	if _, ok := req.Header["Accept-Encoding"]; ok {
+		return nil, errors.New("google api: custom Accept-Encoding headers not allowed")
+	}
 	if ctx == nil {
 		return client.Do(req)
 	}
@@ -53,3 +60,12 @@ func SendRequest(ctx context.Context, client *http.Client, req *http.Request) (*
 	}
 	return resp, err
 }
+
+// DecodeResponse decodes the body of res into target. If there is no body,
+// target is unchanged.
+func DecodeResponse(target interface{}, res *http.Response) error {
+	if res.StatusCode == http.StatusNoContent {
+		return nil
+	}
+	return json.NewDecoder(res.Body).Decode(target)
+}