Update GCS storage library to support GetObject API on gzipped objects (#6506)

Fixes #6280

vendor/google.golang.org/api/gensupport/go18.go

@@ -0,0 +1,17 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build go1.8
+
+package gensupport
+
+import (
+	"io"
+	"net/http"
+)
+
+// SetGetBody sets the GetBody field of req to f.
+func SetGetBody(req *http.Request, f func() (io.ReadCloser, error)) {
+	req.GetBody = f
+}

vendor/google.golang.org/api/gensupport/json.go

@@ -22,23 +22,33 @@ func MarshalJSON(schema interface{}, forceSendFields, nullFields []string) ([]by
 		return json.Marshal(schema)
 	}
 
-	mustInclude := make(map[string]struct{})
+	mustInclude := make(map[string]bool)
 	for _, f := range forceSendFields {
-		mustInclude[f] = struct{}{}
+		mustInclude[f] = true
 	}
-	useNull := make(map[string]struct{})
-	for _, f := range nullFields {
-		useNull[f] = struct{}{}
+	useNull := make(map[string]bool)
+	useNullMaps := make(map[string]map[string]bool)
+	for _, nf := range nullFields {
+		parts := strings.SplitN(nf, ".", 2)
+		field := parts[0]
+		if len(parts) == 1 {
+			useNull[field] = true
+		} else {
+			if useNullMaps[field] == nil {
+				useNullMaps[field] = map[string]bool{}
+			}
+			useNullMaps[field][parts[1]] = true
+		}
 	}
 
-	dataMap, err := schemaToMap(schema, mustInclude, useNull)
+	dataMap, err := schemaToMap(schema, mustInclude, useNull, useNullMaps)
 	if err != nil {
 		return nil, err
 	}
 	return json.Marshal(dataMap)
 }
 
-func schemaToMap(schema interface{}, mustInclude, useNull map[string]struct{}) (map[string]interface{}, error) {
+func schemaToMap(schema interface{}, mustInclude, useNull map[string]bool, useNullMaps map[string]map[string]bool) (map[string]interface{}, error) {
 	m := make(map[string]interface{})
 	s := reflect.ValueOf(schema)
 	st := s.Type()
@@ -59,17 +69,35 @@ func schemaToMap(schema interface{}, mustInclude, useNull map[string]struct{}) (
 		v := s.Field(i)
 		f := st.Field(i)
 
-		if _, ok := useNull[f.Name]; ok {
+		if useNull[f.Name] {
 			if !isEmptyValue(v) {
 				return nil, fmt.Errorf("field %q in NullFields has non-empty value", f.Name)
 			}
 			m[tag.apiName] = nil
 			continue
 		}
+
 		if !includeField(v, f, mustInclude) {
 			continue
 		}
 
+		// If map fields are explicitly set to null, use a map[string]interface{}.
+		if f.Type.Kind() == reflect.Map && useNullMaps[f.Name] != nil {
+			ms, ok := v.Interface().(map[string]string)
+			if !ok {
+				return nil, fmt.Errorf("field %q has keys in NullFields but is not a map[string]string", f.Name)
+			}
+			mi := map[string]interface{}{}
+			for k, v := range ms {
+				mi[k] = v
+			}
+			for k := range useNullMaps[f.Name] {
+				mi[k] = nil
+			}
+			m[tag.apiName] = mi
+			continue
+		}
+
 		// nil maps are treated as empty maps.
 		if f.Type.Kind() == reflect.Map && v.IsNil() {
 			m[tag.apiName] = map[string]string{}
@@ -139,7 +167,7 @@ func parseJSONTag(val string) (jsonTag, error) {
 }
 
 // Reports whether the struct field "f" with value "v" should be included in JSON output.
-func includeField(v reflect.Value, f reflect.StructField, mustInclude map[string]struct{}) bool {
+func includeField(v reflect.Value, f reflect.StructField, mustInclude map[string]bool) bool {
 	// The regular JSON encoding of a nil pointer is "null", which means "delete this field".
 	// Therefore, we could enable field deletion by honoring pointer fields' presence in the mustInclude set.
 	// However, many fields are not pointers, so there would be no way to delete these fields.
@@ -156,8 +184,7 @@ func includeField(v reflect.Value, f reflect.StructField, mustInclude map[string
 		return false
 	}
 
-	_, ok := mustInclude[f.Name]
-	return ok || !isEmptyValue(v)
+	return mustInclude[f.Name] || !isEmptyValue(v)
 }
 
 // isEmptyValue reports whether v is the empty value for its type.  This

vendor/google.golang.org/api/gensupport/jsonfloat.go

@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
+// Copyright 2016 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

vendor/google.golang.org/api/gensupport/media.go

@@ -5,12 +5,15 @@
 package gensupport
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
 	"mime/multipart"
 	"net/http"
 	"net/textproto"
+	"strings"
+	"sync"
 
 	"google.golang.org/api/googleapi"
 )
@@ -103,12 +106,13 @@ type typeReader struct {
 	typ string
 }
 
-// multipartReader combines the contents of multiple readers to creat a multipart/related HTTP body.
+// multipartReader combines the contents of multiple readers to create a multipart/related HTTP body.
 // Close must be called if reads from the multipartReader are abandoned before reaching EOF.
 type multipartReader struct {
 	pr       *io.PipeReader
-	pipeOpen bool
 	ctype    string
+	mu       sync.Mutex
+	pipeOpen bool
 }
 
 func newMultipartReader(parts []typeReader) *multipartReader {
@@ -144,10 +148,13 @@ func (mp *multipartReader) Read(data []byte) (n int, err error) {
 }
 
 func (mp *multipartReader) Close() error {
+	mp.mu.Lock()
 	if !mp.pipeOpen {
+		mp.mu.Unlock()
 		return nil
 	}
 	mp.pipeOpen = false
+	mp.mu.Unlock()
 	return mp.pr.Close()
 }
 
@@ -174,26 +181,156 @@ func typeHeader(contentType string) textproto.MIMEHeader {
 // PrepareUpload determines whether the data in the supplied reader should be
 // uploaded in a single request, or in sequential chunks.
 // chunkSize is the size of the chunk that media should be split into.
-// If chunkSize is non-zero and the contents of media do not fit in a single
-// chunk (or there is an error reading media), then media will be returned as a
-// MediaBuffer.  Otherwise, media will be returned as a Reader.
+//
+// If chunkSize is zero, media is returned as the first value, and the other
+// two return values are nil, true.
+//
+// Otherwise, a MediaBuffer is returned, along with a bool indicating whether the
+// contents of media fit in a single chunk.
 //
 // After PrepareUpload has been called, media should no longer be used: the
 // media content should be accessed via one of the return values.
-func PrepareUpload(media io.Reader, chunkSize int) (io.Reader, *MediaBuffer) {
+func PrepareUpload(media io.Reader, chunkSize int) (r io.Reader, mb *MediaBuffer, singleChunk bool) {
 	if chunkSize == 0 { // do not chunk
-		return media, nil
+		return media, nil, true
+	}
+	mb = NewMediaBuffer(media, chunkSize)
+	_, _, _, err := mb.Chunk()
+	// If err is io.EOF, we can upload this in a single request. Otherwise, err is
+	// either nil or a non-EOF error. If it is the latter, then the next call to
+	// mb.Chunk will return the same error. Returning a MediaBuffer ensures that this
+	// error will be handled at some point.
+	return nil, mb, err == io.EOF
+}
+
+// MediaInfo holds information for media uploads. It is intended for use by generated
+// code only.
+type MediaInfo struct {
+	// At most one of Media and MediaBuffer will be set.
+	media           io.Reader
+	buffer          *MediaBuffer
+	singleChunk     bool
+	mType           string
+	size            int64 // mediaSize, if known.  Used only for calls to progressUpdater_.
+	progressUpdater googleapi.ProgressUpdater
+}
+
+// NewInfoFromMedia should be invoked from the Media method of a call. It returns a
+// MediaInfo populated with chunk size and content type, and a reader or MediaBuffer
+// if needed.
+func NewInfoFromMedia(r io.Reader, options []googleapi.MediaOption) *MediaInfo {
+	mi := &MediaInfo{}
+	opts := googleapi.ProcessMediaOptions(options)
+	if !opts.ForceEmptyContentType {
+		r, mi.mType = DetermineContentType(r, opts.ContentType)
+	}
+	mi.media, mi.buffer, mi.singleChunk = PrepareUpload(r, opts.ChunkSize)
+	return mi
+}
+
+// NewInfoFromResumableMedia should be invoked from the ResumableMedia method of a
+// call. It returns a MediaInfo using the given reader, size and media type.
+func NewInfoFromResumableMedia(r io.ReaderAt, size int64, mediaType string) *MediaInfo {
+	rdr := ReaderAtToReader(r, size)
+	rdr, mType := DetermineContentType(rdr, mediaType)
+	return &MediaInfo{
+		size:        size,
+		mType:       mType,
+		buffer:      NewMediaBuffer(rdr, googleapi.DefaultUploadChunkSize),
+		media:       nil,
+		singleChunk: false,
+	}
+}
+
+func (mi *MediaInfo) SetProgressUpdater(pu googleapi.ProgressUpdater) {
+	if mi != nil {
+		mi.progressUpdater = pu
+	}
+}
+
+// UploadType determines the type of upload: a single request, or a resumable
+// series of requests.
+func (mi *MediaInfo) UploadType() string {
+	if mi.singleChunk {
+		return "multipart"
+	}
+	return "resumable"
+}
+
+// UploadRequest sets up an HTTP request for media upload. It adds headers
+// as necessary, and returns a replacement for the body and a function for http.Request.GetBody.
+func (mi *MediaInfo) UploadRequest(reqHeaders http.Header, body io.Reader) (newBody io.Reader, getBody func() (io.ReadCloser, error), cleanup func()) {
+	cleanup = func() {}
+	if mi == nil {
+		return body, nil, cleanup
+	}
+	var media io.Reader
+	if mi.media != nil {
+		// This only happens when the caller has turned off chunking. In that
+		// case, we write all of media in a single non-retryable request.
+		media = mi.media
+	} else if mi.singleChunk {
+		// The data fits in a single chunk, which has now been read into the MediaBuffer.
+		// We obtain that chunk so we can write it in a single request. The request can
+		// be retried because the data is stored in the MediaBuffer.
+		media, _, _, _ = mi.buffer.Chunk()
+	}
+	if media != nil {
+		fb := readerFunc(body)
+		fm := readerFunc(media)
+		combined, ctype := CombineBodyMedia(body, "application/json", media, mi.mType)
+		if fb != nil && fm != nil {
+			getBody = func() (io.ReadCloser, error) {
+				rb := ioutil.NopCloser(fb())
+				rm := ioutil.NopCloser(fm())
+				r, _ := CombineBodyMedia(rb, "application/json", rm, mi.mType)
+				return r, nil
+			}
+		}
+		cleanup = func() { combined.Close() }
+		reqHeaders.Set("Content-Type", ctype)
+		body = combined
+	}
+	if mi.buffer != nil && mi.mType != "" && !mi.singleChunk {
+		reqHeaders.Set("X-Upload-Content-Type", mi.mType)
+	}
+	return body, getBody, cleanup
+}
+
+// readerFunc returns a function that always returns an io.Reader that has the same
+// contents as r, provided that can be done without consuming r. Otherwise, it
+// returns nil.
+// See http.NewRequest (in net/http/request.go).
+func readerFunc(r io.Reader) func() io.Reader {
+	switch r := r.(type) {
+	case *bytes.Buffer:
+		buf := r.Bytes()
+		return func() io.Reader { return bytes.NewReader(buf) }
+	case *bytes.Reader:
+		snapshot := *r
+		return func() io.Reader { r := snapshot; return &r }
+	case *strings.Reader:
+		snapshot := *r
+		return func() io.Reader { r := snapshot; return &r }
+	default:
+		return nil
+	}
+}
+
+// ResumableUpload returns an appropriately configured ResumableUpload value if the
+// upload is resumable, or nil otherwise.
+func (mi *MediaInfo) ResumableUpload(locURI string) *ResumableUpload {
+	if mi == nil || mi.singleChunk {
+		return nil
+	}
+	return &ResumableUpload{
+		URI:       locURI,
+		Media:     mi.buffer,
+		MediaType: mi.mType,
+		Callback: func(curr int64) {
+			if mi.progressUpdater != nil {
+				mi.progressUpdater(curr, mi.size)
+			}
+		},
 	}
-
-	mb := NewMediaBuffer(media, chunkSize)
-	rdr, _, _, err := mb.Chunk()
-
-	if err == io.EOF { // we can upload this in a single request
-		return rdr, nil
-	}
-	// err might be a non-EOF error. If it is, the next call to mb.Chunk will
-	// return the same error. Returning a MediaBuffer ensures that this error
-	// will be handled at some point.
-
-	return nil, mb
 }

vendor/google.golang.org/api/gensupport/not_go18.go

@@ -0,0 +1,14 @@
+// Copyright 2018 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// +build !go1.8
+
+package gensupport
+
+import (
+	"io"
+	"net/http"
+)
+
+func SetGetBody(req *http.Request, f func() (io.ReadCloser, error)) {}

vendor/google.golang.org/api/gensupport/retry.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

vendor/google.golang.org/api/gensupport/send.go

@@ -5,6 +5,8 @@
 package gensupport
 
 import (
+	"encoding/json"
+	"errors"
 	"net/http"
 
 	"golang.org/x/net/context"
@@ -32,6 +34,11 @@ func RegisterHook(h Hook) {
 // If ctx is non-nil, it calls all hooks, then sends the request with
 // ctxhttp.Do, then calls any functions returned by the hooks in reverse order.
 func SendRequest(ctx context.Context, client *http.Client, req *http.Request) (*http.Response, error) {
+	// Disallow Accept-Encoding because it interferes with the automatic gzip handling
+	// done by the default http.Transport. See https://github.com/google/google-api-go-client/issues/219.
+	if _, ok := req.Header["Accept-Encoding"]; ok {
+		return nil, errors.New("google api: custom Accept-Encoding headers not allowed")
+	}
 	if ctx == nil {
 		return client.Do(req)
 	}
@@ -53,3 +60,12 @@ func SendRequest(ctx context.Context, client *http.Client, req *http.Request) (*
 	}
 	return resp, err
 }
+
+// DecodeResponse decodes the body of res into target. If there is no body,
+// target is unchanged.
+func DecodeResponse(target interface{}, res *http.Response) error {
+	if res.StatusCode == http.StatusNoContent {
+		return nil
+	}
+	return json.NewDecoder(res.Body).Decode(target)
+}

vendor/google.golang.org/api/internal/creds.go

@@ -0,0 +1,42 @@
+// Copyright 2017 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package internal
+
+import (
+	"fmt"
+	"io/ioutil"
+
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2/google"
+)
+
+// Creds returns credential information obtained from DialSettings, or if none, then
+// it returns default credential information.
+func Creds(ctx context.Context, ds *DialSettings) (*google.DefaultCredentials, error) {
+	if ds.Credentials != nil {
+		return ds.Credentials, nil
+	}
+	if ds.CredentialsFile != "" {
+		data, err := ioutil.ReadFile(ds.CredentialsFile)
+		if err != nil {
+			return nil, fmt.Errorf("cannot read credentials file: %v", err)
+		}
+		return google.CredentialsFromJSON(ctx, data, ds.Scopes...)
+	}
+	if ds.TokenSource != nil {
+		return &google.DefaultCredentials{TokenSource: ds.TokenSource}, nil
+	}
+	return google.FindDefaultCredentials(ctx, ds.Scopes...)
+}

vendor/google.golang.org/api/internal/pool.go

@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
+// Copyright 2016 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.

vendor/google.golang.org/api/internal/settings.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -16,22 +16,47 @@
 package internal
 
 import (
+	"errors"
 	"net/http"
 
 	"golang.org/x/oauth2"
+	"golang.org/x/oauth2/google"
 	"google.golang.org/grpc"
 )
 
 // DialSettings holds information needed to establish a connection with a
 // Google API service.
 type DialSettings struct {
-	Endpoint                   string
-	Scopes                     []string
-	ServiceAccountJSONFilename string // if set, TokenSource is ignored.
-	TokenSource                oauth2.TokenSource
-	UserAgent                  string
-	APIKey                     string
-	HTTPClient                 *http.Client
-	GRPCDialOpts               []grpc.DialOption
-	GRPCConn                   *grpc.ClientConn
+	Endpoint        string
+	Scopes          []string
+	TokenSource     oauth2.TokenSource
+	Credentials     *google.DefaultCredentials
+	CredentialsFile string // if set, Token Source is ignored.
+	UserAgent       string
+	APIKey          string
+	HTTPClient      *http.Client
+	GRPCDialOpts    []grpc.DialOption
+	GRPCConn        *grpc.ClientConn
+	NoAuth          bool
+}
+
+// Validate reports an error if ds is invalid.
+func (ds *DialSettings) Validate() error {
+	hasCreds := ds.APIKey != "" || ds.TokenSource != nil || ds.CredentialsFile != "" || ds.Credentials != nil
+	if ds.NoAuth && hasCreds {
+		return errors.New("options.WithoutAuthentication is incompatible with any option that provides credentials")
+	}
+	// Credentials should not appear with other options.
+	// We currently allow TokenSource and CredentialsFile to coexist.
+	// TODO(jba): make TokenSource & CredentialsFile an error (breaking change).
+	if ds.Credentials != nil && (ds.APIKey != "" || ds.TokenSource != nil || ds.CredentialsFile != "") {
+		return errors.New("multiple credential options provided")
+	}
+	if ds.HTTPClient != nil && ds.GRPCConn != nil {
+		return errors.New("WithHTTPClient is incompatible with WithGRPCConn")
+	}
+	if ds.HTTPClient != nil && ds.GRPCDialOpts != nil {
+		return errors.New("WithHTTPClient is incompatible with gRPC dial options")
+	}
+	return nil
 }

vendor/google.golang.org/api/option/credentials_go19.go

@@ -0,0 +1,32 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.9
+
+package option
+
+import (
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/internal"
+)
+
+type withCreds google.Credentials
+
+func (w *withCreds) Apply(o *internal.DialSettings) {
+	o.Credentials = (*google.Credentials)(w)
+}
+
+func WithCredentials(creds *google.Credentials) ClientOption {
+	return (*withCreds)(creds)
+}

vendor/google.golang.org/api/option/credentials_notgo19.go

@@ -0,0 +1,32 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !go1.9
+
+package option
+
+import (
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/internal"
+)
+
+type withCreds google.DefaultCredentials
+
+func (w *withCreds) Apply(o *internal.DialSettings) {
+	o.Credentials = (*google.DefaultCredentials)(w)
+}
+
+func WithCredentials(creds *google.DefaultCredentials) ClientOption {
+	return (*withCreds)(creds)
+}

vendor/google.golang.org/api/option/option.go

@@ -1,4 +1,4 @@
-// Copyright 2017 Google Inc. All Rights Reserved.
+// Copyright 2017 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -40,19 +40,25 @@ func (w withTokenSource) Apply(o *internal.DialSettings) {
 	o.TokenSource = w.ts
 }
 
-// WithServiceAccountFile returns a ClientOption that uses a Google service
-// account credentials file to authenticate.
-// Use WithTokenSource with a token source created from
-// golang.org/x/oauth2/google.JWTConfigFromJSON
-// if reading the file from disk is not an option.
-func WithServiceAccountFile(filename string) ClientOption {
-	return withServiceAccountFile(filename)
+type withCredFile string
+
+func (w withCredFile) Apply(o *internal.DialSettings) {
+	o.CredentialsFile = string(w)
 }
 
-type withServiceAccountFile string
+// WithCredentialsFile returns a ClientOption that authenticates
+// API calls with the given service account or refresh token JSON
+// credentials file.
+func WithCredentialsFile(filename string) ClientOption {
+	return withCredFile(filename)
+}
 
-func (w withServiceAccountFile) Apply(o *internal.DialSettings) {
-	o.ServiceAccountJSONFilename = string(w)
+// WithServiceAccountFile returns a ClientOption that uses a Google service
+// account credentials file to authenticate.
+//
+// Deprecated: Use WithCredentialsFile instead.
+func WithServiceAccountFile(filename string) ClientOption {
+	return WithCredentialsFile(filename)
 }
 
 // WithEndpoint returns a ClientOption that overrides the default endpoint
@@ -147,6 +153,9 @@ func (w withGRPCConnectionPool) Apply(o *internal.DialSettings) {
 
 // WithAPIKey returns a ClientOption that specifies an API key to be used
 // as the basis for authentication.
+//
+// API Keys can only be used for JSON-over-HTTP APIs, including those under
+// the import path google.golang.org/api/....
 func WithAPIKey(apiKey string) ClientOption {
 	return withAPIKey(apiKey)
 }
@@ -154,3 +163,16 @@ func WithAPIKey(apiKey string) ClientOption {
 type withAPIKey string
 
 func (w withAPIKey) Apply(o *internal.DialSettings) { o.APIKey = string(w) }
+
+// WithoutAuthentication returns a ClientOption that specifies that no
+// authentication should be used. It is suitable only for testing and for
+// accessing public resources, like public Google Cloud Storage buckets.
+// It is an error to provide both WithoutAuthentication and any of WithAPIKey,
+// WithTokenSource, WithCredentialsFile or WithServiceAccountFile.
+func WithoutAuthentication() ClientOption {
+	return withoutAuthentication{}
+}
+
+type withoutAuthentication struct{}
+
+func (w withoutAuthentication) Apply(o *internal.DialSettings) { o.NoAuth = true }

vendor/google.golang.org/api/transport/dial.go

@@ -1,4 +1,4 @@
-// Copyright 2015 Google Inc. All Rights Reserved.
+// Copyright 2015 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -18,184 +18,32 @@
 package transport
 
 import (
-	"errors"
-	"fmt"
-	"io/ioutil"
 	"net/http"
 
 	"golang.org/x/net/context"
-	"golang.org/x/oauth2"
-	"golang.org/x/oauth2/google"
 	"google.golang.org/grpc"
-	"google.golang.org/grpc/credentials"
-	"google.golang.org/grpc/credentials/oauth"
 
-	gtransport "google.golang.org/api/googleapi/transport"
-	"google.golang.org/api/internal"
 	"google.golang.org/api/option"
+	gtransport "google.golang.org/api/transport/grpc"
+	htransport "google.golang.org/api/transport/http"
 )
 
 // NewHTTPClient returns an HTTP client for use communicating with a Google cloud
 // service, configured with the given ClientOptions. It also returns the endpoint
 // for the service as specified in the options.
 func NewHTTPClient(ctx context.Context, opts ...option.ClientOption) (*http.Client, string, error) {
-	var o internal.DialSettings
-	for _, opt := range opts {
-		opt.Apply(&o)
-	}
-	if o.GRPCConn != nil {
-		return nil, "", errors.New("unsupported gRPC connection specified")
-	}
-	// TODO(cbro): consider injecting the User-Agent even if an explicit HTTP client is provided?
-	if o.HTTPClient != nil {
-		return o.HTTPClient, o.Endpoint, nil
-	}
-	if o.APIKey != "" {
-		hc := &http.Client{
-			Transport: &gtransport.APIKey{
-				Key: o.APIKey,
-				Transport: userAgentTransport{
-					base:      baseTransport(ctx),
-					userAgent: o.UserAgent,
-				},
-			},
-		}
-		return hc, o.Endpoint, nil
-	}
-	if o.ServiceAccountJSONFilename != "" {
-		ts, err := serviceAcctTokenSource(ctx, o.ServiceAccountJSONFilename, o.Scopes...)
-		if err != nil {
-			return nil, "", err
-		}
-		o.TokenSource = ts
-	}
-	if o.TokenSource == nil {
-		var err error
-		o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...)
-		if err != nil {
-			return nil, "", fmt.Errorf("google.DefaultTokenSource: %v", err)
-		}
-	}
-	hc := &http.Client{
-		Transport: &oauth2.Transport{
-			Source: o.TokenSource,
-			Base: userAgentTransport{
-				base:      baseTransport(ctx),
-				userAgent: o.UserAgent,
-			},
-		},
-	}
-	return hc, o.Endpoint, nil
-}
-
-type userAgentTransport struct {
-	userAgent string
-	base      http.RoundTripper
-}
-
-func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	rt := t.base
-	if rt == nil {
-		return nil, errors.New("transport: no Transport specified")
-	}
-	if t.userAgent == "" {
-		return rt.RoundTrip(req)
-	}
-	newReq := *req
-	newReq.Header = make(http.Header)
-	for k, vv := range req.Header {
-		newReq.Header[k] = vv
-	}
-	// TODO(cbro): append to existing User-Agent header?
-	newReq.Header["User-Agent"] = []string{t.userAgent}
-	return rt.RoundTrip(&newReq)
-}
-
-// Set at init time by dial_appengine.go. If nil, we're not on App Engine.
-var appengineDialerHook func(context.Context) grpc.DialOption
-var appengineUrlfetchHook func(context.Context) http.RoundTripper
-
-// baseTransport returns the base HTTP transport.
-// On App Engine, this is urlfetch.Transport, otherwise it's http.DefaultTransport.
-func baseTransport(ctx context.Context) http.RoundTripper {
-	if appengineUrlfetchHook != nil {
-		return appengineUrlfetchHook(ctx)
-	}
-	return http.DefaultTransport
+	return htransport.NewClient(ctx, opts...)
 }
 
 // DialGRPC returns a GRPC connection for use communicating with a Google cloud
 // service, configured with the given ClientOptions.
 func DialGRPC(ctx context.Context, opts ...option.ClientOption) (*grpc.ClientConn, error) {
-	var o internal.DialSettings
-	for _, opt := range opts {
-		opt.Apply(&o)
-	}
-	if o.HTTPClient != nil {
-		return nil, errors.New("unsupported HTTP client specified")
-	}
-	if o.GRPCConn != nil {
-		return o.GRPCConn, nil
-	}
-	if o.ServiceAccountJSONFilename != "" {
-		ts, err := serviceAcctTokenSource(ctx, o.ServiceAccountJSONFilename, o.Scopes...)
-		if err != nil {
-			return nil, err
-		}
-		o.TokenSource = ts
-	}
-	if o.TokenSource == nil {
-		var err error
-		o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...)
-		if err != nil {
-			return nil, fmt.Errorf("google.DefaultTokenSource: %v", err)
-		}
-	}
-	grpcOpts := []grpc.DialOption{
-		grpc.WithPerRPCCredentials(oauth.TokenSource{o.TokenSource}),
-		grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")),
-	}
-	if appengineDialerHook != nil {
-		// Use the Socket API on App Engine.
-		grpcOpts = append(grpcOpts, appengineDialerHook(ctx))
-	}
-	grpcOpts = append(grpcOpts, o.GRPCDialOpts...)
-	if o.UserAgent != "" {
-		grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent))
-	}
-	return grpc.DialContext(ctx, o.Endpoint, grpcOpts...)
-}
-
-func serviceAcctTokenSource(ctx context.Context, filename string, scope ...string) (oauth2.TokenSource, error) {
-	data, err := ioutil.ReadFile(filename)
-	if err != nil {
-		return nil, fmt.Errorf("cannot read service account file: %v", err)
-	}
-	cfg, err := google.JWTConfigFromJSON(data, scope...)
-	if err != nil {
-		return nil, fmt.Errorf("google.JWTConfigFromJSON: %v", err)
-	}
-	return cfg.TokenSource(ctx), nil
+	return gtransport.Dial(ctx, opts...)
 }
 
 // DialGRPCInsecure returns an insecure GRPC connection for use communicating
 // with fake or mock Google cloud service implementations, such as emulators.
 // The connection is configured with the given ClientOptions.
 func DialGRPCInsecure(ctx context.Context, opts ...option.ClientOption) (*grpc.ClientConn, error) {
-	var o internal.DialSettings
-	for _, opt := range opts {
-		opt.Apply(&o)
-	}
-	if o.HTTPClient != nil {
-		return nil, errors.New("unsupported HTTP client specified")
-	}
-	if o.GRPCConn != nil {
-		return o.GRPCConn, nil
-	}
-	grpcOpts := []grpc.DialOption{grpc.WithInsecure()}
-	grpcOpts = append(grpcOpts, o.GRPCDialOpts...)
-	if o.UserAgent != "" {
-		grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent))
-	}
-	return grpc.DialContext(ctx, o.Endpoint, grpcOpts...)
+	return gtransport.DialInsecure(ctx, opts...)
 }

vendor/google.golang.org/api/transport/go19.go

@@ -0,0 +1,34 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.9
+
+package transport
+
+import (
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/internal"
+	"google.golang.org/api/option"
+)
+
+// Creds constructs a google.Credentials from the information in the options,
+// or obtains the default credentials in the same way as google.FindDefaultCredentials.
+func Creds(ctx context.Context, opts ...option.ClientOption) (*google.Credentials, error) {
+	var ds internal.DialSettings
+	for _, opt := range opts {
+		opt.Apply(&ds)
+	}
+	return internal.Creds(ctx, &ds)
+}

vendor/google.golang.org/api/transport/grpc/dial.go

@@ -0,0 +1,91 @@
+// Copyright 2015 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package transport/grpc supports network connections to GRPC servers.
+// This package is not intended for use by end developers. Use the
+// google.golang.org/api/option package to configure API clients.
+package grpc
+
+import (
+	"errors"
+	"log"
+
+	"golang.org/x/net/context"
+	"google.golang.org/api/internal"
+	"google.golang.org/api/option"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials"
+	"google.golang.org/grpc/credentials/oauth"
+)
+
+// Set at init time by dial_appengine.go. If nil, we're not on App Engine.
+var appengineDialerHook func(context.Context) grpc.DialOption
+
+// Dial returns a GRPC connection for use communicating with a Google cloud
+// service, configured with the given ClientOptions.
+func Dial(ctx context.Context, opts ...option.ClientOption) (*grpc.ClientConn, error) {
+	return dial(ctx, false, opts)
+}
+
+// DialInsecure returns an insecure GRPC connection for use communicating
+// with fake or mock Google cloud service implementations, such as emulators.
+// The connection is configured with the given ClientOptions.
+func DialInsecure(ctx context.Context, opts ...option.ClientOption) (*grpc.ClientConn, error) {
+	return dial(ctx, true, opts)
+}
+
+func dial(ctx context.Context, insecure bool, opts []option.ClientOption) (*grpc.ClientConn, error) {
+	var o internal.DialSettings
+	for _, opt := range opts {
+		opt.Apply(&o)
+	}
+	if err := o.Validate(); err != nil {
+		return nil, err
+	}
+	if o.HTTPClient != nil {
+		return nil, errors.New("unsupported HTTP client specified")
+	}
+	if o.GRPCConn != nil {
+		return o.GRPCConn, nil
+	}
+	var grpcOpts []grpc.DialOption
+	if insecure {
+		grpcOpts = []grpc.DialOption{grpc.WithInsecure()}
+	} else if !o.NoAuth {
+		if o.APIKey != "" {
+			log.Print("API keys are not supported for gRPC APIs. Remove the WithAPIKey option from your client-creating call.")
+		}
+		creds, err := internal.Creds(ctx, &o)
+		if err != nil {
+			return nil, err
+		}
+		grpcOpts = []grpc.DialOption{
+			grpc.WithPerRPCCredentials(oauth.TokenSource{creds.TokenSource}),
+			grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")),
+		}
+	}
+	if appengineDialerHook != nil {
+		// Use the Socket API on App Engine.
+		grpcOpts = append(grpcOpts, appengineDialerHook(ctx))
+	}
+	// Add tracing, but before the other options, so that clients can override the
+	// gRPC stats handler.
+	// This assumes that gRPC options are processed in order, left to right.
+	grpcOpts = addOCStatsHandler(grpcOpts)
+	grpcOpts = append(grpcOpts, o.GRPCDialOpts...)
+	if o.UserAgent != "" {
+		grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent))
+	}
+	return grpc.DialContext(ctx, o.Endpoint, grpcOpts...)
+}

vendor/google.golang.org/api/transport/grpc/dial_appengine.go

@@ -1,4 +1,4 @@
-// Copyright 2016 Google Inc. All Rights Reserved.
+// Copyright 2016 Google LLC
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -14,27 +14,28 @@
 
 // +build appengine
 
-package transport
+package grpc
 
 import (
 	"net"
-	"net/http"
 	"time"
 
 	"golang.org/x/net/context"
+	"google.golang.org/appengine"
 	"google.golang.org/appengine/socket"
-	"google.golang.org/appengine/urlfetch"
 	"google.golang.org/grpc"
 )
 
 func init() {
+	// NOTE: dev_appserver doesn't currently support SSL.
+	// When it does, this code can be removed.
+	if appengine.IsDevAppServer() {
+		return
+	}
+
 	appengineDialerHook = func(ctx context.Context) grpc.DialOption {
 		return grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
 			return socket.DialTimeout(ctx, "tcp", addr, timeout)
 		})
 	}
-
-	appengineUrlfetchHook = func(ctx context.Context) http.RoundTripper {
-		return &urlfetch.Transport{Context: ctx}
-	}
 }

vendor/google.golang.org/api/transport/grpc/go18.go

@@ -0,0 +1,26 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package grpc
+
+import (
+	"go.opencensus.io/plugin/ocgrpc"
+	"google.golang.org/grpc"
+)
+
+func addOCStatsHandler(opts []grpc.DialOption) []grpc.DialOption {
+	return append(opts, grpc.WithStatsHandler(&ocgrpc.ClientHandler{}))
+}

vendor/google.golang.org/api/transport/grpc/not_go18.go

@@ -0,0 +1,21 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !go1.8
+
+package grpc
+
+import "google.golang.org/grpc"
+
+func addOCStatsHandler(opts []grpc.DialOption) []grpc.DialOption { return opts }

vendor/google.golang.org/api/transport/http/dial.go

@@ -0,0 +1,138 @@
+// Copyright 2015 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package transport/http supports network connections to HTTP servers.
+// This package is not intended for use by end developers. Use the
+// google.golang.org/api/option package to configure API clients.
+package http
+
+import (
+	"errors"
+	"net/http"
+
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2"
+	"google.golang.org/api/googleapi/transport"
+	"google.golang.org/api/internal"
+	"google.golang.org/api/option"
+)
+
+// NewClient returns an HTTP client for use communicating with a Google cloud
+// service, configured with the given ClientOptions. It also returns the endpoint
+// for the service as specified in the options.
+func NewClient(ctx context.Context, opts ...option.ClientOption) (*http.Client, string, error) {
+	settings, err := newSettings(opts)
+	if err != nil {
+		return nil, "", err
+	}
+	// TODO(cbro): consider injecting the User-Agent even if an explicit HTTP client is provided?
+	if settings.HTTPClient != nil {
+		return settings.HTTPClient, settings.Endpoint, nil
+	}
+	trans, err := newTransport(ctx, defaultBaseTransport(ctx), settings)
+	if err != nil {
+		return nil, "", err
+	}
+	return &http.Client{Transport: trans}, settings.Endpoint, nil
+}
+
+// NewTransport creates an http.RoundTripper for use communicating with a Google
+// cloud service, configured with the given ClientOptions. Its RoundTrip method delegates to base.
+func NewTransport(ctx context.Context, base http.RoundTripper, opts ...option.ClientOption) (http.RoundTripper, error) {
+	settings, err := newSettings(opts)
+	if err != nil {
+		return nil, err
+	}
+	if settings.HTTPClient != nil {
+		return nil, errors.New("transport/http: WithHTTPClient passed to NewTransport")
+	}
+	return newTransport(ctx, base, settings)
+}
+
+func newTransport(ctx context.Context, base http.RoundTripper, settings *internal.DialSettings) (http.RoundTripper, error) {
+	trans := base
+	trans = userAgentTransport{
+		base:      trans,
+		userAgent: settings.UserAgent,
+	}
+	trans = addOCTransport(trans)
+	switch {
+	case settings.NoAuth:
+		// Do nothing.
+	case settings.APIKey != "":
+		trans = &transport.APIKey{
+			Transport: trans,
+			Key:       settings.APIKey,
+		}
+	default:
+		creds, err := internal.Creds(ctx, settings)
+		if err != nil {
+			return nil, err
+		}
+		trans = &oauth2.Transport{
+			Base:   trans,
+			Source: creds.TokenSource,
+		}
+	}
+	return trans, nil
+}
+
+func newSettings(opts []option.ClientOption) (*internal.DialSettings, error) {
+	var o internal.DialSettings
+	for _, opt := range opts {
+		opt.Apply(&o)
+	}
+	if err := o.Validate(); err != nil {
+		return nil, err
+	}
+	if o.GRPCConn != nil {
+		return nil, errors.New("unsupported gRPC connection specified")
+	}
+	return &o, nil
+}
+
+type userAgentTransport struct {
+	userAgent string
+	base      http.RoundTripper
+}
+
+func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	rt := t.base
+	if rt == nil {
+		return nil, errors.New("transport: no Transport specified")
+	}
+	if t.userAgent == "" {
+		return rt.RoundTrip(req)
+	}
+	newReq := *req
+	newReq.Header = make(http.Header)
+	for k, vv := range req.Header {
+		newReq.Header[k] = vv
+	}
+	// TODO(cbro): append to existing User-Agent header?
+	newReq.Header["User-Agent"] = []string{t.userAgent}
+	return rt.RoundTrip(&newReq)
+}
+
+// Set at init time by dial_appengine.go. If nil, we're not on App Engine.
+var appengineUrlfetchHook func(context.Context) http.RoundTripper
+
+// defaultBaseTransport returns the base HTTP transport.
+// On App Engine, this is urlfetch.Transport, otherwise it's http.DefaultTransport.
+func defaultBaseTransport(ctx context.Context) http.RoundTripper {
+	if appengineUrlfetchHook != nil {
+		return appengineUrlfetchHook(ctx)
+	}
+	return http.DefaultTransport
+}

vendor/google.golang.org/api/transport/http/dial_appengine.go

@@ -0,0 +1,30 @@
+// Copyright 2016 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build appengine
+
+package http
+
+import (
+	"net/http"
+
+	"golang.org/x/net/context"
+	"google.golang.org/appengine/urlfetch"
+)
+
+func init() {
+	appengineUrlfetchHook = func(ctx context.Context) http.RoundTripper {
+		return &urlfetch.Transport{Context: ctx}
+	}
+}

vendor/google.golang.org/api/transport/http/go18.go

@@ -0,0 +1,31 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build go1.8
+
+package http
+
+import (
+	"net/http"
+
+	"contrib.go.opencensus.io/exporter/stackdriver/propagation"
+	"go.opencensus.io/plugin/ochttp"
+)
+
+func addOCTransport(trans http.RoundTripper) http.RoundTripper {
+	return &ochttp.Transport{
+		Base:        trans,
+		Propagation: &propagation.HTTPFormat{},
+	}
+}

vendor/google.golang.org/api/transport/http/not_go18.go

@@ -0,0 +1,21 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !go1.8
+
+package http
+
+import "net/http"
+
+func addOCTransport(trans http.RoundTripper) http.RoundTripper { return trans }

vendor/google.golang.org/api/transport/not_go19.go

@@ -0,0 +1,34 @@
+// Copyright 2018 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// +build !go1.9
+
+package transport
+
+import (
+	"golang.org/x/net/context"
+	"golang.org/x/oauth2/google"
+	"google.golang.org/api/internal"
+	"google.golang.org/api/option"
+)
+
+// Creds constructs a google.DefaultCredentials from the information in the options,
+// or obtains the default credentials in the same way as google.FindDefaultCredentials.
+func Creds(ctx context.Context, opts ...option.ClientOption) (*google.DefaultCredentials, error) {
+	var ds internal.DialSettings
+	for _, opt := range opts {
+		opt.Apply(&ds)
+	}
+	return internal.Creds(ctx, &ds)
+}