astelm/minio
1
0
Fork 0
mirror of https://github.com/pgsty/minio.git synced 2026-08-10 16:23:28 +03:00
Code Issues Packages Projects Releases Wiki Activity

Update GCS storage library to support GetObject API on gzipped objects (#6506)

Browse Source 
Fixes #6280
This commit is contained in:
kannappanr
2018-09-21 11:43:10 -07:00
committed by GitHub
parent 3c8fabd116
commit c1798cc89a
132 changed files with 66518 additions and 1280 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/google.golang.org/api/transport/dial.go
Generated Vendored
+6 -158
View File
@@ -1,4 +1,4 @@
// Copyright 2015 Google Inc. All Rights Reserved.
// Copyright 2015 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -18,184 +18,32 @@
package transport
import (
"errors"
"fmt"
"io/ioutil"
"net/http"
"golang.org/x/net/context"
"golang.org/x/oauth2"
"golang.org/x/oauth2/google"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/oauth"
gtransport "google.golang.org/api/googleapi/transport"
"google.golang.org/api/internal"
"google.golang.org/api/option"
gtransport "google.golang.org/api/transport/grpc"
htransport "google.golang.org/api/transport/http"
)
// NewHTTPClient returns an HTTP client for use communicating with a Google cloud
// service, configured with the given ClientOptions. It also returns the endpoint
// for the service as specified in the options.
func NewHTTPClient(ctx context.Context, opts ...option.ClientOption) (*http.Client, string, error) {
var o internal.DialSettings
for _, opt := range opts {
opt.Apply(&o)
}
if o.GRPCConn != nil {
return nil, "", errors.New("unsupported gRPC connection specified")
}
// TODO(cbro): consider injecting the User-Agent even if an explicit HTTP client is provided?
if o.HTTPClient != nil {
return o.HTTPClient, o.Endpoint, nil
}
if o.APIKey != "" {
hc := &http.Client{
Transport: &gtransport.APIKey{
Key: o.APIKey,
Transport: userAgentTransport{
base: baseTransport(ctx),
userAgent: o.UserAgent,
},
},
}
return hc, o.Endpoint, nil
}
if o.ServiceAccountJSONFilename != "" {
ts, err := serviceAcctTokenSource(ctx, o.ServiceAccountJSONFilename, o.Scopes...)
if err != nil {
return nil, "", err
}
o.TokenSource = ts
}
if o.TokenSource == nil {
var err error
o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...)
if err != nil {
return nil, "", fmt.Errorf("google.DefaultTokenSource: %v", err)
}
}
hc := &http.Client{
Transport: &oauth2.Transport{
Source: o.TokenSource,
Base: userAgentTransport{
base: baseTransport(ctx),
userAgent: o.UserAgent,
},
},
}
return hc, o.Endpoint, nil
}
type userAgentTransport struct {
userAgent string
base http.RoundTripper
}
func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
rt := t.base
if rt == nil {
return nil, errors.New("transport: no Transport specified")
}
if t.userAgent == "" {
return rt.RoundTrip(req)
}
newReq := *req
newReq.Header = make(http.Header)
for k, vv := range req.Header {
newReq.Header[k] = vv
}
// TODO(cbro): append to existing User-Agent header?
newReq.Header["User-Agent"] = []string{t.userAgent}
return rt.RoundTrip(&newReq)
}
// Set at init time by dial_appengine.go. If nil, we're not on App Engine.
var appengineDialerHook func(context.Context) grpc.DialOption
var appengineUrlfetchHook func(context.Context) http.RoundTripper
// baseTransport returns the base HTTP transport.
// On App Engine, this is urlfetch.Transport, otherwise it's http.DefaultTransport.
func baseTransport(ctx context.Context) http.RoundTripper {
if appengineUrlfetchHook != nil {
return appengineUrlfetchHook(ctx)
}
return http.DefaultTransport
return htransport.NewClient(ctx, opts...)
}
// DialGRPC returns a GRPC connection for use communicating with a Google cloud
// service, configured with the given ClientOptions.
func DialGRPC(ctx context.Context, opts ...option.ClientOption) (*grpc.ClientConn, error) {
var o internal.DialSettings
for _, opt := range opts {
opt.Apply(&o)
}
if o.HTTPClient != nil {
return nil, errors.New("unsupported HTTP client specified")
}
if o.GRPCConn != nil {
return o.GRPCConn, nil
}
if o.ServiceAccountJSONFilename != "" {
ts, err := serviceAcctTokenSource(ctx, o.ServiceAccountJSONFilename, o.Scopes...)
if err != nil {
return nil, err
}
o.TokenSource = ts
}
if o.TokenSource == nil {
var err error
o.TokenSource, err = google.DefaultTokenSource(ctx, o.Scopes...)
if err != nil {
return nil, fmt.Errorf("google.DefaultTokenSource: %v", err)
}
}
grpcOpts := []grpc.DialOption{
grpc.WithPerRPCCredentials(oauth.TokenSource{o.TokenSource}),
grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")),
}
if appengineDialerHook != nil {
// Use the Socket API on App Engine.
grpcOpts = append(grpcOpts, appengineDialerHook(ctx))
}
grpcOpts = append(grpcOpts, o.GRPCDialOpts...)
if o.UserAgent != "" {
grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent))
}
return grpc.DialContext(ctx, o.Endpoint, grpcOpts...)
}
func serviceAcctTokenSource(ctx context.Context, filename string, scope ...string) (oauth2.TokenSource, error) {
data, err := ioutil.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("cannot read service account file: %v", err)
}
cfg, err := google.JWTConfigFromJSON(data, scope...)
if err != nil {
return nil, fmt.Errorf("google.JWTConfigFromJSON: %v", err)
}
return cfg.TokenSource(ctx), nil
return gtransport.Dial(ctx, opts...)
}
// DialGRPCInsecure returns an insecure GRPC connection for use communicating
// with fake or mock Google cloud service implementations, such as emulators.
// The connection is configured with the given ClientOptions.
func DialGRPCInsecure(ctx context.Context, opts ...option.ClientOption) (*grpc.ClientConn, error) {
var o internal.DialSettings
for _, opt := range opts {
opt.Apply(&o)
}
if o.HTTPClient != nil {
return nil, errors.New("unsupported HTTP client specified")
}
if o.GRPCConn != nil {
return o.GRPCConn, nil
}
grpcOpts := []grpc.DialOption{grpc.WithInsecure()}
grpcOpts = append(grpcOpts, o.GRPCDialOpts...)
if o.UserAgent != "" {
grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent))
}
return grpc.DialContext(ctx, o.Endpoint, grpcOpts...)
return gtransport.DialInsecure(ctx, opts...)
}
vendor/google.golang.org/api/transport/go19.go
Generated Vendored
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.9
package transport
import (
"golang.org/x/net/context"
"golang.org/x/oauth2/google"
"google.golang.org/api/internal"
"google.golang.org/api/option"
)
// Creds constructs a google.Credentials from the information in the options,
// or obtains the default credentials in the same way as google.FindDefaultCredentials.
func Creds(ctx context.Context, opts ...option.ClientOption) (*google.Credentials, error) {
var ds internal.DialSettings
for _, opt := range opts {
opt.Apply(&ds)
}
return internal.Creds(ctx, &ds)
}
vendor/google.golang.org/api/transport/grpc/dial.go
Generated Vendored
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2015 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package transport/grpc supports network connections to GRPC servers.
// This package is not intended for use by end developers. Use the
// google.golang.org/api/option package to configure API clients.
package grpc
import (
"errors"
"log"
"golang.org/x/net/context"
"google.golang.org/api/internal"
"google.golang.org/api/option"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials"
"google.golang.org/grpc/credentials/oauth"
)
// Set at init time by dial_appengine.go. If nil, we're not on App Engine.
var appengineDialerHook func(context.Context) grpc.DialOption
// Dial returns a GRPC connection for use communicating with a Google cloud
// service, configured with the given ClientOptions.
func Dial(ctx context.Context, opts ...option.ClientOption) (*grpc.ClientConn, error) {
return dial(ctx, false, opts)
}
// DialInsecure returns an insecure GRPC connection for use communicating
// with fake or mock Google cloud service implementations, such as emulators.
// The connection is configured with the given ClientOptions.
func DialInsecure(ctx context.Context, opts ...option.ClientOption) (*grpc.ClientConn, error) {
return dial(ctx, true, opts)
}
func dial(ctx context.Context, insecure bool, opts []option.ClientOption) (*grpc.ClientConn, error) {
var o internal.DialSettings
for _, opt := range opts {
opt.Apply(&o)
}
if err := o.Validate(); err != nil {
return nil, err
}
if o.HTTPClient != nil {
return nil, errors.New("unsupported HTTP client specified")
}
if o.GRPCConn != nil {
return o.GRPCConn, nil
}
var grpcOpts []grpc.DialOption
if insecure {
grpcOpts = []grpc.DialOption{grpc.WithInsecure()}
} else if !o.NoAuth {
if o.APIKey != "" {
log.Print("API keys are not supported for gRPC APIs. Remove the WithAPIKey option from your client-creating call.")
}
creds, err := internal.Creds(ctx, &o)
if err != nil {
return nil, err
}
grpcOpts = []grpc.DialOption{
grpc.WithPerRPCCredentials(oauth.TokenSource{creds.TokenSource}),
grpc.WithTransportCredentials(credentials.NewClientTLSFromCert(nil, "")),
}
}
if appengineDialerHook != nil {
// Use the Socket API on App Engine.
grpcOpts = append(grpcOpts, appengineDialerHook(ctx))
}
// Add tracing, but before the other options, so that clients can override the
// gRPC stats handler.
// This assumes that gRPC options are processed in order, left to right.
grpcOpts = addOCStatsHandler(grpcOpts)
grpcOpts = append(grpcOpts, o.GRPCDialOpts...)
if o.UserAgent != "" {
grpcOpts = append(grpcOpts, grpc.WithUserAgent(o.UserAgent))
}
return grpc.DialContext(ctx, o.Endpoint, grpcOpts...)
}
vendor/google.golang.org/api/transport/dial_appengine.go → vendor/google.golang.org/api/transport/grpc/dial_appengine.go
Generated Vendored
+9 -8
View File
@@ -1,4 +1,4 @@
// Copyright 2016 Google Inc. All Rights Reserved.
// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@@ -14,27 +14,28 @@
// +build appengine
package transport
package grpc
import (
"net"
"net/http"
"time"
"golang.org/x/net/context"
"google.golang.org/appengine"
"google.golang.org/appengine/socket"
"google.golang.org/appengine/urlfetch"
"google.golang.org/grpc"
)
func init() {
// NOTE: dev_appserver doesn't currently support SSL.
// When it does, this code can be removed.
if appengine.IsDevAppServer() {
return
}
appengineDialerHook = func(ctx context.Context) grpc.DialOption {
return grpc.WithDialer(func(addr string, timeout time.Duration) (net.Conn, error) {
return socket.DialTimeout(ctx, "tcp", addr, timeout)
})
}
appengineUrlfetchHook = func(ctx context.Context) http.RoundTripper {
return &urlfetch.Transport{Context: ctx}
}
}
vendor/google.golang.org/api/transport/grpc/go18.go
Generated Vendored
+26
View File
@@ -0,0 +1,26 @@
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.8
package grpc
import (
"go.opencensus.io/plugin/ocgrpc"
"google.golang.org/grpc"
)
func addOCStatsHandler(opts []grpc.DialOption) []grpc.DialOption {
return append(opts, grpc.WithStatsHandler(&ocgrpc.ClientHandler{}))
}
vendor/google.golang.org/api/transport/grpc/not_go18.go
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.8
package grpc
import "google.golang.org/grpc"
func addOCStatsHandler(opts []grpc.DialOption) []grpc.DialOption { return opts }
vendor/google.golang.org/api/transport/http/dial.go
Generated Vendored
+138
View File
@@ -0,0 +1,138 @@
// Copyright 2015 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package transport/http supports network connections to HTTP servers.
// This package is not intended for use by end developers. Use the
// google.golang.org/api/option package to configure API clients.
package http
import (
"errors"
"net/http"
"golang.org/x/net/context"
"golang.org/x/oauth2"
"google.golang.org/api/googleapi/transport"
"google.golang.org/api/internal"
"google.golang.org/api/option"
)
// NewClient returns an HTTP client for use communicating with a Google cloud
// service, configured with the given ClientOptions. It also returns the endpoint
// for the service as specified in the options.
func NewClient(ctx context.Context, opts ...option.ClientOption) (*http.Client, string, error) {
settings, err := newSettings(opts)
if err != nil {
return nil, "", err
}
// TODO(cbro): consider injecting the User-Agent even if an explicit HTTP client is provided?
if settings.HTTPClient != nil {
return settings.HTTPClient, settings.Endpoint, nil
}
trans, err := newTransport(ctx, defaultBaseTransport(ctx), settings)
if err != nil {
return nil, "", err
}
return &http.Client{Transport: trans}, settings.Endpoint, nil
}
// NewTransport creates an http.RoundTripper for use communicating with a Google
// cloud service, configured with the given ClientOptions. Its RoundTrip method delegates to base.
func NewTransport(ctx context.Context, base http.RoundTripper, opts ...option.ClientOption) (http.RoundTripper, error) {
settings, err := newSettings(opts)
if err != nil {
return nil, err
}
if settings.HTTPClient != nil {
return nil, errors.New("transport/http: WithHTTPClient passed to NewTransport")
}
return newTransport(ctx, base, settings)
}
func newTransport(ctx context.Context, base http.RoundTripper, settings *internal.DialSettings) (http.RoundTripper, error) {
trans := base
trans = userAgentTransport{
base: trans,
userAgent: settings.UserAgent,
}
trans = addOCTransport(trans)
switch {
case settings.NoAuth:
// Do nothing.
case settings.APIKey != "":
trans = &transport.APIKey{
Transport: trans,
Key: settings.APIKey,
}
default:
creds, err := internal.Creds(ctx, settings)
if err != nil {
return nil, err
}
trans = &oauth2.Transport{
Base: trans,
Source: creds.TokenSource,
}
}
return trans, nil
}
func newSettings(opts []option.ClientOption) (*internal.DialSettings, error) {
var o internal.DialSettings
for _, opt := range opts {
opt.Apply(&o)
}
if err := o.Validate(); err != nil {
return nil, err
}
if o.GRPCConn != nil {
return nil, errors.New("unsupported gRPC connection specified")
}
return &o, nil
}
type userAgentTransport struct {
userAgent string
base http.RoundTripper
}
func (t userAgentTransport) RoundTrip(req *http.Request) (*http.Response, error) {
rt := t.base
if rt == nil {
return nil, errors.New("transport: no Transport specified")
}
if t.userAgent == "" {
return rt.RoundTrip(req)
}
newReq := *req
newReq.Header = make(http.Header)
for k, vv := range req.Header {
newReq.Header[k] = vv
}
// TODO(cbro): append to existing User-Agent header?
newReq.Header["User-Agent"] = []string{t.userAgent}
return rt.RoundTrip(&newReq)
}
// Set at init time by dial_appengine.go. If nil, we're not on App Engine.
var appengineUrlfetchHook func(context.Context) http.RoundTripper
// defaultBaseTransport returns the base HTTP transport.
// On App Engine, this is urlfetch.Transport, otherwise it's http.DefaultTransport.
func defaultBaseTransport(ctx context.Context) http.RoundTripper {
if appengineUrlfetchHook != nil {
return appengineUrlfetchHook(ctx)
}
return http.DefaultTransport
}
vendor/google.golang.org/api/transport/http/dial_appengine.go
Generated Vendored
+30
View File
@@ -0,0 +1,30 @@
// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build appengine
package http
import (
"net/http"
"golang.org/x/net/context"
"google.golang.org/appengine/urlfetch"
)
func init() {
appengineUrlfetchHook = func(ctx context.Context) http.RoundTripper {
return &urlfetch.Transport{Context: ctx}
}
}
vendor/google.golang.org/api/transport/http/go18.go
Generated Vendored
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build go1.8
package http
import (
"net/http"
"contrib.go.opencensus.io/exporter/stackdriver/propagation"
"go.opencensus.io/plugin/ochttp"
)
func addOCTransport(trans http.RoundTripper) http.RoundTripper {
return &ochttp.Transport{
Base: trans,
Propagation: &propagation.HTTPFormat{},
}
}
vendor/google.golang.org/api/transport/http/not_go18.go
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.8
package http
import "net/http"
func addOCTransport(trans http.RoundTripper) http.RoundTripper { return trans }
vendor/google.golang.org/api/transport/not_go19.go
Generated Vendored
+34
View File
@@ -0,0 +1,34 @@
// Copyright 2018 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// +build !go1.9
package transport
import (
"golang.org/x/net/context"
"golang.org/x/oauth2/google"
"google.golang.org/api/internal"
"google.golang.org/api/option"
)
// Creds constructs a google.DefaultCredentials from the information in the options,
// or obtains the default credentials in the same way as google.FindDefaultCredentials.
func Creds(ctx context.Context, opts ...option.ClientOption) (*google.DefaultCredentials, error) {
var ds internal.DialSettings
for _, opt := range opts {
opt.Apply(&ds)
}
return internal.Creds(ctx, &ds)
}