Activate xDS support in grpcurl (#137)

* use wrapped TransportCredentials instead of handling handshake in Dialer, so that grpc library will use correct :scheme
* support -authority for TLS conns; now effectively supercedes -servername flag

.gitignore

@@ -1 +1,2 @@
 dist/
+VERSION

.travis.yml

@@ -3,14 +3,19 @@ sudo: false
 
 matrix:
   include:
-    - go: "1.9"
-    - go: "1.10"
-      env: VET=1
-    - go: "1.11"
+    - go: 1.9.x
+    - go: 1.10.x
+    - go: 1.11.x
+      env:
+      - GO111MODULE=off
+      - VET=1
+    - go: 1.11.x
+      env: GO111MODULE=on
+    - go: 1.12.x
       env: GO111MODULE=off
-    - go: "1.11"
+    - go: 1.12.x
       env: GO111MODULE=on
     - go: tip
 
 script:
-  - if [[ "$VET" = 1 ]]; then make; else make deps test; fi
+  - if [[ "$VET" = 1 ]]; then make ci; else make deps test; fi

Dockerfile

@@ -0,0 +1,33 @@
+FROM golang:1.11.10-alpine as builder
+MAINTAINER FullStory Engineering
+
+# currently, a module build requires gcc (so Go tool can build
+# module-aware versions of std library; it ships only w/ the
+# non-module versions)
+RUN apk update && apk add --no-cache ca-certificates git gcc g++ libc-dev
+# create non-privileged group and user
+RUN addgroup -S grpcurl && adduser -S grpcurl -G grpcurl
+
+WORKDIR /tmp/fullstorydev/grpcurl
+# copy just the files/sources we need to build grpcurl
+COPY VERSION *.go go.* /tmp/fullstorydev/grpcurl/
+COPY cmd /tmp/fullstorydev/grpcurl/cmd
+# and build a completely static binary (so we can use
+# scratch as basis for the final image)
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+ENV GOARCH=amd64
+ENV GO111MODULE=on
+RUN go build -o /grpcurl \
+    -ldflags "-w -extldflags \"-static\" -X \"main.version=$(cat VERSION)\"" \
+    ./cmd/grpcurl
+
+# New FROM so we have a nice'n'tiny image
+FROM scratch
+WORKDIR /
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /grpcurl /bin/grpcurl
+USER grpcurl
+
+ENTRYPOINT ["/bin/grpcurl"]

Makefile

@@ -6,7 +6,7 @@ dev_build_version=$(shell git describe --tags --always --dirty)
 # they are just too noisy to be a requirement for a CI -- we don't even *want*
 # to fix some of the things they consider to be violations.
 .PHONY: ci
-ci: deps checkgofmt vet staticcheck unused ineffassign predeclared test
+ci: deps checkgofmt vet staticcheck ineffassign predeclared test
 
 .PHONY: deps
 deps:
@@ -25,6 +25,12 @@ release:
 	@GO111MODULE=off go get github.com/goreleaser/goreleaser
 	goreleaser --rm-dist
 
+.PHONY: docker
+docker:
+	@echo $(dev_build_version) > VERSION
+	docker build -t fullstorydev/grpcurl:$(dev_build_version) .
+	@rm VERSION
+
 .PHONY: checkgofmt
 checkgofmt:
 	gofmt -s -l .
@@ -47,12 +53,7 @@ vet:
 .PHONY: staticcheck
 staticcheck:
 	@go get honnef.co/go/tools/cmd/staticcheck
-	staticcheck -ignore github.com/fullstorydev/grpcurl/tls_settings_test.go:SA1019 ./...
-
-.PHONY: unused
-unused:
-	@go get honnef.co/go/tools/cmd/unused
-	unused ./...
+	staticcheck ./...
 
 .PHONY: ineffassign
 ineffassign:

README.md

@@ -49,6 +49,11 @@ files (containing compiled descriptors, produced by `protoc`) to `grpcurl`.
 
 Download the binary from the [releases](https://github.com/fullstorydev/grpcurl/releases) page.
 
+On macOS, `grpcurl` is available via Homebrew:
+```shell
+brew install grpcurl
+```
+
 ### From Source
 You can use the `go` tool to install `grpcurl`:
 ```shell
@@ -102,7 +107,7 @@ If you want to include `grpcurl` in a command pipeline, such as when using `jq`
 create a request body, you can use `-d @`, which tells `grpcurl` to read the actual
 request body from stdin:
 ```shell
-grpcurl -d @ grpc.server.com:443 my.custom.server.Service/Method <<<EOM
+grpcurl -d @ grpc.server.com:443 my.custom.server.Service/Method <<EOM
 {
   "id": 1234,
   "tags": [

cmd/grpcurl/grpcurl.go

@@ -8,7 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"strconv"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -23,6 +23,9 @@ import (
 	"google.golang.org/grpc/keepalive"
 	"google.golang.org/grpc/metadata"
 	reflectpb "google.golang.org/grpc/reflection/grpc_reflection_v1alpha"
+
+	// Register gzip compressor so compressed responses will work:
+	_ "google.golang.org/grpc/encoding/gzip"
 )
 
 var version = "dev build <no version set>"
@@ -32,39 +35,53 @@ var (
 
 	isUnixSocket func() bool // nil when run on non-unix platform
 
-	help = flag.Bool("help", false, prettify(`
+	flags = flag.NewFlagSet(os.Args[0], flag.ExitOnError)
+
+	help = flags.Bool("help", false, prettify(`
 		Print usage instructions and exit.`))
-	printVersion = flag.Bool("version", false, prettify(`
+	printVersion = flags.Bool("version", false, prettify(`
 		Print version.`))
-	plaintext = flag.Bool("plaintext", false, prettify(`
+	plaintext = flags.Bool("plaintext", false, prettify(`
 		Use plain-text HTTP/2 when connecting to server (no TLS).`))
-	insecure = flag.Bool("insecure", false, prettify(`
+	insecure = flags.Bool("insecure", false, prettify(`
 		Skip server certificate and domain verification. (NOT SECURE!) Not
 		valid with -plaintext option.`))
-	cacert = flag.String("cacert", "", prettify(`
+	cacert = flags.String("cacert", "", prettify(`
 		File containing trusted root certificates for verifying the server.
 		Ignored if -insecure is specified.`))
-	cert = flag.String("cert", "", prettify(`
+	cert = flags.String("cert", "", prettify(`
 		File containing client certificate (public key), to present to the
 		server. Not valid with -plaintext option. Must also provide -key option.`))
-	key = flag.String("key", "", prettify(`
+	key = flags.String("key", "", prettify(`
 		File containing client private key, to present to the server. Not valid
 		with -plaintext option. Must also provide -cert option.`))
-	protoset    multiString
-	protoFiles  multiString
-	importPaths multiString
-	addlHeaders multiString
-	rpcHeaders  multiString
-	reflHeaders multiString
-	authority   = flag.String("authority", "", prettify(`
-		Value of :authority pseudo-header to be use with underlying HTTP/2
-		requests. It defaults to the given address.`))
-	data = flag.String("d", "", prettify(`
+	protoset      multiString
+	protoFiles    multiString
+	importPaths   multiString
+	addlHeaders   multiString
+	rpcHeaders    multiString
+	reflHeaders   multiString
+	expandHeaders = flags.Bool("expand-headers", false, prettify(`
+		If set, headers may use '${NAME}' syntax to reference environment
+		variables. These will be expanded to the actual environment variable
+		value before sending to the server. For example, if there is an
+		environment variable defined like FOO=bar, then a header of
+		'key: ${FOO}' would expand to 'key: bar'. This applies to -H,
+		-rpc-header, and -reflect-header options. No other expansion/escaping is
+		performed. This can be used to supply credentials/secrets without having
+		to put them in command-line arguments.`))
+	authority = flags.String("authority", "", prettify(`
+		The authoritative name of the remote server. This value is passed as the
+		value of the ":authority" pseudo-header in the HTTP/2 protocol. When TLS
+		is used, this will also be used as the server name when verifying the
+		server's certificate. It defaults to the address that is provided in the
+		positional arguments.`))
+	data = flags.String("d", "", prettify(`
 		Data for request contents. If the value is '@' then the request contents
 		are read from stdin. For calls that accept a stream of requests, the
 		contents should include all such request messages concatenated together
 		(possibly delimited; see -format).`))
-	format = flag.String("format", "json", prettify(`
+	format = flags.String("format", "json", prettify(`
 		The format of request data. The allowed values are 'json' or 'text'. For
 		'json', the input data must be in JSON format. Multiple request values
 		may be concatenated (messages with a JSON representation other than
@@ -74,43 +91,58 @@ var (
 		ASCII character: 0x1E. The stream should not end in a record separator.
 		If it does, it will be interpreted as a final, blank message after the
 		separator.`))
-	connectTimeout = flag.String("connect-timeout", "", prettify(`
+	connectTimeout = flags.Float64("connect-timeout", 0, prettify(`
 		The maximum time, in seconds, to wait for connection to be established.
 		Defaults to 10 seconds.`))
-	keepaliveTime = flag.String("keepalive-time", "", prettify(`
+	keepaliveTime = flags.Float64("keepalive-time", 0, prettify(`
 		If present, the maximum idle time in seconds, after which a keepalive
 		probe is sent. If the connection remains idle and no keepalive response
 		is received for this same period then the connection is closed and the
 		operation fails.`))
-	maxTime = flag.String("max-time", "", prettify(`
-		The maximum total time the operation can take. This is useful for
-		preventing batch jobs that use grpcurl from hanging due to slow or bad
-		network links or due to incorrect stream method usage.`))
-	emitDefaults = flag.Bool("emit-defaults", false, prettify(`
+	maxTime = flags.Float64("max-time", 0, prettify(`
+		The maximum total time the operation can take, in seconds. This is
+		useful for preventing batch jobs that use grpcurl from hanging due to
+		slow or bad network links or due to incorrect stream method usage.`))
+	maxMsgSz = flags.Int("max-msg-sz", 0, prettify(`
+		The maximum encoded size of a response message, in bytes, that grpcurl
+		will accept. If not specified, defaults to 4,194,304 (4 megabytes).`))
+	emitDefaults = flags.Bool("emit-defaults", false, prettify(`
 		Emit default values for JSON-encoded responses.`))
-	msgTemplate = flag.Bool("msg-template", false, prettify(`
+	protosetOut = flags.String("protoset-out", "", prettify(`
+		The name of a file to be written that will contain a FileDescriptorSet
+		proto. With the list and describe verbs, the listed or described
+		elements and their transitive dependencies will be written to the named
+		file if this option is given. When invoking an RPC and this option is
+		given, the method being invoked and its transitive dependencies will be
+		included in the output file.`))
+	msgTemplate = flags.Bool("msg-template", false, prettify(`
 		When describing messages, show a template of input data.`))
-	verbose = flag.Bool("v", false, prettify(`
+	verbose = flags.Bool("v", false, prettify(`
 		Enable verbose output.`))
-	serverName = flag.String("servername", "", prettify(`
-		Override server name when validating TLS certificate.`))
+	serverName = flags.String("servername", "", prettify(`
+		Override server name when validating TLS certificate. This flag is
+		ignored if -plaintext or -insecure is used.
+		NOTE: Prefer -authority. This flag may be removed in the future. It is
+		an error to use both -authority and -servername (though this will be
+		permitted if they are both set to the same value, to increase backwards
+		compatibility with earlier releases that allowed both to be set).`))
 )
 
 func init() {
-	flag.Var(&addlHeaders, "H", prettify(`
+	flags.Var(&addlHeaders, "H", prettify(`
 		Additional headers in 'name: value' format. May specify more than one
 		via multiple flags. These headers will also be included in reflection
 		requests requests to a server.`))
-	flag.Var(&rpcHeaders, "rpc-header", prettify(`
+	flags.Var(&rpcHeaders, "rpc-header", prettify(`
 		Additional RPC headers in 'name: value' format. May specify more than
 		one via multiple flags. These headers will *only* be used when invoking
 		the requested RPC method. They are excluded from reflection requests.`))
-	flag.Var(&reflHeaders, "reflect-header", prettify(`
+	flags.Var(&reflHeaders, "reflect-header", prettify(`
 		Additional reflection headers in 'name: value' format. May specify more
 		than one via multiple flags. These headers will *only* be used during
 		reflection requests and will be excluded when invoking the requested RPC
 		method.`))
-	flag.Var(&protoset, "protoset", prettify(`
+	flags.Var(&protoset, "protoset", prettify(`
 		The name of a file containing an encoded FileDescriptorSet. This file's
 		contents will be used to determine the RPC schema instead of querying
 		for it from the remote server via the gRPC reflection API. When set: the
@@ -119,7 +151,7 @@ func init() {
 		symbols found in the given descriptors. May specify more than one via
 		multiple -protoset flags. It is an error to use both -protoset and
 		-proto flags.`))
-	flag.Var(&protoFiles, "proto", prettify(`
+	flags.Var(&protoFiles, "proto", prettify(`
 		The name of a proto source file. Source files given will be used to
 		determine the RPC schema instead of querying for it from the remote
 		server via the gRPC reflection API. When set: the 'list' action lists
@@ -129,7 +161,7 @@ func init() {
 		-proto flags. Imports will be resolved using the given -import-path
 		flags. Multiple proto files can be specified by specifying multiple
 		-proto flags. It is an error to use both -protoset and -proto flags.`))
-	flag.Var(&importPaths, "import-path", prettify(`
+	flags.Var(&importPaths, "import-path", prettify(`
 		The path to a directory from which proto sources can be imported, for
 		use with -proto flags. Multiple import paths can be configured by
 		specifying multiple -import-path flags. Paths will be searched in the
@@ -150,18 +182,30 @@ func (s *multiString) Set(value string) error {
 }
 
 func main() {
-	flag.CommandLine.Usage = usage
-	flag.Parse()
+	flags.Usage = usage
+	flags.Parse(os.Args[1:])
 	if *help {
 		usage()
 		os.Exit(0)
 	}
 	if *printVersion {
-		fmt.Fprintf(os.Stderr, "%s %s\n", os.Args[0], version)
+		fmt.Fprintf(os.Stderr, "%s %s\n", filepath.Base(os.Args[0]), version)
 		os.Exit(0)
 	}
 
 	// Do extra validation on arguments and figure out what user asked us to do.
+	if *connectTimeout < 0 {
+		fail(nil, "The -connect-timeout argument must not be negative.")
+	}
+	if *keepaliveTime < 0 {
+		fail(nil, "The -keepalive-time argument must not be negative.")
+	}
+	if *maxTime < 0 {
+		fail(nil, "The -max-time argument must not be negative.")
+	}
+	if *maxMsgSz < 0 {
+		fail(nil, "The -max-msg-sz argument must not be negative.")
+	}
 	if *plaintext && *insecure {
 		fail(nil, "The -plaintext and -insecure arguments are mutually exclusive.")
 	}
@@ -181,7 +225,7 @@ func main() {
 		warn("The -emit-defaults is only used when using json format.")
 	}
 
-	args := flag.Args()
+	args := flags.Args()
 
 	if len(args) == 0 {
 		fail(nil, "Too few arguments.")
@@ -246,40 +290,28 @@ func main() {
 	}
 
 	ctx := context.Background()
-	if *maxTime != "" {
-		t, err := strconv.ParseFloat(*maxTime, 64)
-		if err != nil {
-			fail(nil, "The -max-time argument must be a valid number.")
-		}
-		timeout := time.Duration(t * float64(time.Second))
+	if *maxTime > 0 {
+		timeout := time.Duration(*maxTime * float64(time.Second))
 		ctx, _ = context.WithTimeout(ctx, timeout)
 	}
 
 	dial := func() *grpc.ClientConn {
 		dialTime := 10 * time.Second
-		if *connectTimeout != "" {
-			t, err := strconv.ParseFloat(*connectTimeout, 64)
-			if err != nil {
-				fail(nil, "The -connect-timeout argument must be a valid number.")
-			}
-			dialTime = time.Duration(t * float64(time.Second))
+		if *connectTimeout > 0 {
+			dialTime = time.Duration(*connectTimeout * float64(time.Second))
 		}
 		ctx, cancel := context.WithTimeout(ctx, dialTime)
 		defer cancel()
 		var opts []grpc.DialOption
-		if *keepaliveTime != "" {
-			t, err := strconv.ParseFloat(*keepaliveTime, 64)
-			if err != nil {
-				fail(nil, "The -keepalive-time argument must be a valid number.")
-			}
-			timeout := time.Duration(t * float64(time.Second))
+		if *keepaliveTime > 0 {
+			timeout := time.Duration(*keepaliveTime * float64(time.Second))
 			opts = append(opts, grpc.WithKeepaliveParams(keepalive.ClientParameters{
 				Time:    timeout,
 				Timeout: timeout,
 			}))
 		}
-		if *authority != "" {
-			opts = append(opts, grpc.WithAuthority(*authority))
+		if *maxMsgSz > 0 {
+			opts = append(opts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(*maxMsgSz)))
 		}
 		var creds credentials.TransportCredentials
 		if !*plaintext {
@@ -288,11 +320,27 @@ func main() {
 			if err != nil {
 				fail(err, "Failed to configure transport credentials")
 			}
-			if *serverName != "" {
-				if err := creds.OverrideServerName(*serverName); err != nil {
-					fail(err, "Failed to override server name as %q", *serverName)
+
+			// can use either -servername or -authority; but not both
+			if *serverName != "" && *authority != "" {
+				if *serverName == *authority {
+					warn("Both -servername and -authority are present; prefer only -authority.")
+				} else {
+					fail(nil, "Cannot specify different values for -servername and -authority.")
 				}
 			}
+			overrideName := *serverName
+			if overrideName == "" {
+				overrideName = *authority
+			}
+
+			if overrideName != "" {
+				if err := creds.OverrideServerName(overrideName); err != nil {
+					fail(err, "Failed to override server name as %q", overrideName)
+				}
+			}
+		} else if *authority != "" {
+			opts = append(opts, grpc.WithAuthority(*authority))
 		}
 		network := "tcp"
 		if isUnixSocket != nil && isUnixSocket() {
@@ -305,6 +353,22 @@ func main() {
 		return cc
 	}
 
+	if *expandHeaders {
+		var err error
+		addlHeaders, err = grpcurl.ExpandHeaders(addlHeaders)
+		if err != nil {
+			fail(err, "Failed to expand additional headers")
+		}
+		rpcHeaders, err = grpcurl.ExpandHeaders(rpcHeaders)
+		if err != nil {
+			fail(err, "Failed to expand rpc headers")
+		}
+		reflHeaders, err = grpcurl.ExpandHeaders(reflHeaders)
+		if err != nil {
+			fail(err, "Failed to expand reflection headers")
+		}
+	}
+
 	var cc *grpc.ClientConn
 	var descSource grpcurl.DescriptorSource
 	var refClient *grpcreflect.Client
@@ -359,6 +423,9 @@ func main() {
 					fmt.Printf("%s\n", svc)
 				}
 			}
+			if err := writeProtoset(descSource, svcs...); err != nil {
+				fail(err, "Failed to write protoset to %s", *protosetOut)
+			}
 		} else {
 			methods, err := grpcurl.ListMethods(descSource, symbol)
 			if err != nil {
@@ -371,6 +438,9 @@ func main() {
 					fmt.Printf("%s\n", m)
 				}
 			}
+			if err := writeProtoset(descSource, symbol); err != nil {
+				fail(err, "Failed to write protoset to %s", *protosetOut)
+			}
 		}
 
 	} else if describe {
@@ -471,6 +541,9 @@ func main() {
 				fmt.Println(str)
 			}
 		}
+		if err := writeProtoset(descSource, symbols...); err != nil {
+			fail(err, "Failed to write protoset to %s", *protosetOut)
+		}
 
 	} else {
 		// Invoke an RPC
@@ -511,7 +584,7 @@ func main() {
 			fmt.Printf("Sent %d request%s and received %d response%s\n", reqCount, reqSuffix, h.NumResponses, respSuffix)
 		}
 		if h.Status.Code() != codes.OK {
-			fmt.Fprintf(os.Stderr, "ERROR:\n  Code: %s\n  Message: %s\n", h.Status.Code().String(), h.Status.Message())
+			grpcurl.PrintStatus(os.Stderr, h.Status, formatter)
 			exit(1)
 		}
 	}
@@ -546,7 +619,7 @@ path to the domain socket.
 
 Available flags:
 `, os.Args[0])
-	flag.PrintDefaults()
+	flags.PrintDefaults()
 }
 
 func prettify(docString string) string {
@@ -587,3 +660,15 @@ func fail(err error, msg string, args ...interface{}) {
 		exit(2)
 	}
 }
+
+func writeProtoset(descSource grpcurl.DescriptorSource, symbols ...string) error {
+	if *protosetOut == "" {
+		return nil
+	}
+	f, err := os.Create(*protosetOut)
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	return grpcurl.WriteProtoset(f, descSource, symbols...)
+}

cmd/grpcurl/unix.go

@@ -2,10 +2,8 @@
 
 package main
 
-import "flag"
-
 var (
-	unix = flag.Bool("unix", false, prettify(`
+	unix = flags.Bool("unix", false, prettify(`
 		Indicates that the server address is the path to a Unix domain socket.`))
 )
 

desc_source.go

@@ -3,6 +3,7 @@ package grpcurl
 import (
 	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"sync"
 
@@ -58,9 +59,14 @@ func DescriptorSourceFromProtoSets(fileNames ...string) (DescriptorSource, error
 // whose contents are Protocol Buffer source files. The given importPaths are used to locate
 // any imported files.
 func DescriptorSourceFromProtoFiles(importPaths []string, fileNames ...string) (DescriptorSource, error) {
+	fileNames, err := protoparse.ResolveFilenames(importPaths, fileNames...)
+	if err != nil {
+		return nil, err
+	}
 	p := protoparse.Parser{
-		ImportPaths:      importPaths,
-		InferImportPaths: len(importPaths) == 0,
+		ImportPaths:           importPaths,
+		InferImportPaths:      len(importPaths) == 0,
+		IncludeSourceCodeInfo: true,
 	}
 	fds, err := p.ParseFiles(fileNames...)
 	if err != nil {
@@ -109,7 +115,7 @@ func resolveFileDescriptor(unresolved map[string]*descpb.FileDescriptorProto, re
 	return result, nil
 }
 
-// DescriptorSourceFromFileDescriptorSet creates a DescriptorSource that is backed by the given
+// DescriptorSourceFromFileDescriptors creates a DescriptorSource that is backed by the given
 // file descriptors
 func DescriptorSourceFromFileDescriptors(files ...*desc.FileDescriptor) (DescriptorSource, error) {
 	fds := map[string]*desc.FileDescriptor{}
@@ -246,3 +252,53 @@ func reflectionSupport(err error) error {
 	}
 	return err
 }
+
+// WriteProtoset will use the given descriptor source to resolve all of the given
+// symbols and write a proto file descriptor set with their definitions to the
+// given output. The output will include descriptors for all files in which the
+// symbols are defined as well as their transitive dependencies.
+func WriteProtoset(out io.Writer, descSource DescriptorSource, symbols ...string) error {
+	// compute set of file descriptors
+	filenames := make([]string, 0, len(symbols))
+	fds := make(map[string]*desc.FileDescriptor, len(symbols))
+	for _, sym := range symbols {
+		d, err := descSource.FindSymbol(sym)
+		if err != nil {
+			return fmt.Errorf("failed to find descriptor for %q: %v", sym, err)
+		}
+		fd := d.GetFile()
+		if _, ok := fds[fd.GetName()]; !ok {
+			fds[fd.GetName()] = fd
+			filenames = append(filenames, fd.GetName())
+		}
+	}
+	// now expand that to include transitive dependencies in topologically sorted
+	// order (such that file always appears after its dependencies)
+	expandedFiles := make(map[string]struct{}, len(fds))
+	allFilesSlice := make([]*descpb.FileDescriptorProto, 0, len(fds))
+	for _, filename := range filenames {
+		allFilesSlice = addFilesToSet(allFilesSlice, expandedFiles, fds[filename])
+	}
+	// now we can serialize to file
+	b, err := proto.Marshal(&descpb.FileDescriptorSet{File: allFilesSlice})
+	if err != nil {
+		return fmt.Errorf("failed to serialize file descriptor set: %v", err)
+	}
+	if _, err := out.Write(b); err != nil {
+		return fmt.Errorf("failed to write file descriptor set: %v", err)
+	}
+	return nil
+}
+
+func addFilesToSet(allFiles []*descpb.FileDescriptorProto, expanded map[string]struct{}, fd *desc.FileDescriptor) []*descpb.FileDescriptorProto {
+	if _, ok := expanded[fd.GetName()]; ok {
+		// already seen this one
+		return allFiles
+	}
+	expanded[fd.GetName()] = struct{}{}
+	// add all dependencies first
+	for _, dep := range fd.GetDependencies() {
+		allFiles = addFilesToSet(allFiles, expanded, dep)
+	}
+	return append(allFiles, fd.AsFileDescriptorProto())
+}

desc_source_test.go

@@ -0,0 +1,62 @@
+package grpcurl
+
+import (
+	"bytes"
+	"io/ioutil"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/protoc-gen-go/descriptor"
+)
+
+func TestWriteProtoset(t *testing.T) {
+	exampleProtoset, err := loadProtoset("./testing/example.protoset")
+	if err != nil {
+		t.Fatalf("failed to load example.protoset: %v", err)
+	}
+	testProtoset, err := loadProtoset("./testing/test.protoset")
+	if err != nil {
+		t.Fatalf("failed to load test.protoset: %v", err)
+	}
+
+	mergedProtoset := &descriptor.FileDescriptorSet{
+		File: append(exampleProtoset.File, testProtoset.File...),
+	}
+
+	descSrc, err := DescriptorSourceFromFileDescriptorSet(mergedProtoset)
+	if err != nil {
+		t.Fatalf("failed to create descriptor source: %v", err)
+	}
+
+	checkWriteProtoset(t, descSrc, exampleProtoset, "TestService")
+	checkWriteProtoset(t, descSrc, testProtoset, "grpc.testing.TestService")
+	checkWriteProtoset(t, descSrc, mergedProtoset, "TestService", "grpc.testing.TestService")
+}
+
+func loadProtoset(path string) (*descriptor.FileDescriptorSet, error) {
+	b, err := ioutil.ReadFile(path)
+	if err != nil {
+		return nil, err
+	}
+	var protoset descriptor.FileDescriptorSet
+	if err := proto.Unmarshal(b, &protoset); err != nil {
+		return nil, err
+	}
+	return &protoset, nil
+}
+
+func checkWriteProtoset(t *testing.T, descSrc DescriptorSource, protoset *descriptor.FileDescriptorSet, symbols ...string) {
+	var buf bytes.Buffer
+	if err := WriteProtoset(&buf, descSrc, symbols...); err != nil {
+		t.Fatalf("failed to write protoset: %v", err)
+	}
+
+	var result descriptor.FileDescriptorSet
+	if err := proto.Unmarshal(buf.Bytes(), &result); err != nil {
+		t.Fatalf("failed to unmarshal written protoset: %v", err)
+	}
+
+	if !proto.Equal(protoset, &result) {
+		t.Fatalf("written protoset not equal to input:\nExpecting: %s\nActual: %s", protoset, &result)
+	}
+}

format.go

@@ -3,14 +3,19 @@ package grpcurl
 import (
 	"bufio"
 	"bytes"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
+	"reflect"
+	"strings"
+	"sync"
 
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"
 	"github.com/jhump/protoreflect/desc"
 	"github.com/jhump/protoreflect/dynamic"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 )
@@ -142,6 +147,8 @@ type textFormatter struct {
 	numFormatted int
 }
 
+var protoTextMarshaler = proto.TextMarshaler{ExpandAny: true}
+
 func (tf *textFormatter) format(m proto.Message) (string, error) {
 	var buf bytes.Buffer
 	if tf.useSeparator && tf.numFormatted > 0 {
@@ -166,7 +173,7 @@ func (tf *textFormatter) format(m proto.Message) (string, error) {
 		if _, err := buf.Write(b); err != nil {
 			return "", err
 		}
-	} else if err := proto.MarshalText(&buf, m); err != nil {
+	} else if err := protoTextMarshaler.Marshal(&buf, m); err != nil {
 		return "", err
 	}
 
@@ -188,24 +195,153 @@ const (
 	FormatText = Format("text")
 )
 
-func anyResolver(source DescriptorSource) (jsonpb.AnyResolver, error) {
-	// TODO: instead of pro-actively downloading file descriptors to
-	// build a dynamic resolver, it would be better if the resolver
-	// impl was lazy, and simply downloaded the descriptors as needed
-	// when asked to resolve a particular type URL
-
-	// best effort: build resolver with whatever files we can
-	// load, ignoring any errors
-	files, _ := GetAllFiles(source)
-
-	var er dynamic.ExtensionRegistry
-	for _, fd := range files {
-		er.AddExtensionsFromFile(fd)
-	}
-	mf := dynamic.NewMessageFactoryWithExtensionRegistry(&er)
-	return dynamic.AnyResolver(mf, files...), nil
+// AnyResolverFromDescriptorSource returns an AnyResolver that will search for
+// types using the given descriptor source.
+func AnyResolverFromDescriptorSource(source DescriptorSource) jsonpb.AnyResolver {
+	return &anyResolver{source: source}
 }
 
+// AnyResolverFromDescriptorSourceWithFallback returns an AnyResolver that will
+// search for types using the given descriptor source and then fallback to a
+// special message if the type is not found. The fallback type will render to
+// JSON with a "@type" property, just like an Any message, but also with a
+// custom "@value" property that includes the binary encoded payload.
+func AnyResolverFromDescriptorSourceWithFallback(source DescriptorSource) jsonpb.AnyResolver {
+	res := anyResolver{source: source}
+	return &anyResolverWithFallback{AnyResolver: &res}
+}
+
+type anyResolver struct {
+	source DescriptorSource
+
+	er dynamic.ExtensionRegistry
+
+	mu       sync.RWMutex
+	mf       *dynamic.MessageFactory
+	resolved map[string]func() proto.Message
+}
+
+func (r *anyResolver) Resolve(typeUrl string) (proto.Message, error) {
+	mname := typeUrl
+	if slash := strings.LastIndex(mname, "/"); slash >= 0 {
+		mname = mname[slash+1:]
+	}
+
+	r.mu.RLock()
+	factory := r.resolved[mname]
+	r.mu.RUnlock()
+
+	// already resolved?
+	if factory != nil {
+		return factory(), nil
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// double-check, in case we were racing with another goroutine
+	// that resolved this one
+	factory = r.resolved[mname]
+	if factory != nil {
+		return factory(), nil
+	}
+
+	// use descriptor source to resolve message type
+	d, err := r.source.FindSymbol(mname)
+	if err != nil {
+		return nil, err
+	}
+	md, ok := d.(*desc.MessageDescriptor)
+	if !ok {
+		return nil, fmt.Errorf("unknown message: %s", typeUrl)
+	}
+	// populate any extensions for this message, too
+	if exts, err := r.source.AllExtensionsForType(mname); err != nil {
+		return nil, err
+	} else if err := r.er.AddExtension(exts...); err != nil {
+		return nil, err
+	}
+
+	if r.mf == nil {
+		r.mf = dynamic.NewMessageFactoryWithExtensionRegistry(&r.er)
+	}
+
+	factory = func() proto.Message {
+		return r.mf.NewMessage(md)
+	}
+	if r.resolved == nil {
+		r.resolved = map[string]func() proto.Message{}
+	}
+	r.resolved[mname] = factory
+	return factory(), nil
+}
+
+// anyResolverWithFallback can provide a fallback value for unknown
+// messages that will format itself to JSON using an "@value" field
+// that has the base64-encoded data for the unknown message value.
+type anyResolverWithFallback struct {
+	jsonpb.AnyResolver
+}
+
+func (r anyResolverWithFallback) Resolve(typeUrl string) (proto.Message, error) {
+	msg, err := r.AnyResolver.Resolve(typeUrl)
+	if err == nil {
+		return msg, err
+	}
+
+	// Try "default" resolution logic. This mirrors the default behavior
+	// of jsonpb, which checks to see if the given message name is registered
+	// in the proto package.
+	mname := typeUrl
+	if slash := strings.LastIndex(mname, "/"); slash >= 0 {
+		mname = mname[slash+1:]
+	}
+	mt := proto.MessageType(mname)
+	if mt != nil {
+		return reflect.New(mt.Elem()).Interface().(proto.Message), nil
+	}
+
+	// finally, fallback to a special placeholder that can marshal itself
+	// to JSON using a special "@value" property to show base64-encoded
+	// data for the embedded message
+	return &unknownAny{TypeUrl: typeUrl, Error: fmt.Sprintf("%s is not recognized; see @value for raw binary message data", mname)}, nil
+}
+
+type unknownAny struct {
+	TypeUrl string `json:"@type"`
+	Error   string `json:"@error"`
+	Value   string `json:"@value"`
+}
+
+func (a *unknownAny) MarshalJSONPB(jsm *jsonpb.Marshaler) ([]byte, error) {
+	if jsm.Indent != "" {
+		return json.MarshalIndent(a, "", jsm.Indent)
+	}
+	return json.Marshal(a)
+}
+
+func (a *unknownAny) Unmarshal(b []byte) error {
+	a.Value = base64.StdEncoding.EncodeToString(b)
+	return nil
+}
+
+func (a *unknownAny) Reset() {
+	a.Value = ""
+}
+
+func (a *unknownAny) String() string {
+	b, err := a.MarshalJSONPB(&jsonpb.Marshaler{})
+	if err != nil {
+		return fmt.Sprintf("ERROR: %v", err.Error())
+	}
+	return string(b)
+}
+
+func (a *unknownAny) ProtoMessage() {
+}
+
+var _ proto.Message = (*unknownAny)(nil)
+
 // RequestParserAndFormatterFor returns a request parser and formatter for the
 // given format. The given descriptor source may be used for parsing message
 // data (if needed by the format). The flags emitJSONDefaultFields and
@@ -214,11 +350,8 @@ func anyResolver(source DescriptorSource) (jsonpb.AnyResolver, error) {
 func RequestParserAndFormatterFor(format Format, descSource DescriptorSource, emitJSONDefaultFields, includeTextSeparator bool, in io.Reader) (RequestParser, Formatter, error) {
 	switch format {
 	case FormatJSON:
-		resolver, err := anyResolver(descSource)
-		if err != nil {
-			return nil, nil, fmt.Errorf("error creating message resolver: %v", err)
-		}
-		return NewJSONRequestParser(in, resolver), NewJSONFormatter(emitJSONDefaultFields, resolver), nil
+		resolver := AnyResolverFromDescriptorSource(descSource)
+		return NewJSONRequestParser(in, resolver), NewJSONFormatter(emitJSONDefaultFields, anyResolverWithFallback{AnyResolver: resolver}), nil
 	case FormatText:
 		return NewTextRequestParser(in), NewTextFormatter(includeTextSeparator), nil
 	default:
@@ -295,3 +428,42 @@ func (h *DefaultEventHandler) OnReceiveTrailers(stat *status.Status, md metadata
 		fmt.Fprintf(h.out, "\nResponse trailers received:\n%s\n", MetadataToString(md))
 	}
 }
+
+// PrintStatus prints details about the given status to the given writer. The given
+// formatter is used to print any detail messages that may be included in the status.
+// If the given status has a code of OK, "OK" is printed and that is all. Otherwise,
+// "ERROR:" is printed along with a line showing the code, one showing the message
+// string, and each detail message if any are present. The detail messages will be
+// printed as proto text format or JSON, depending on the given formatter.
+func PrintStatus(w io.Writer, stat *status.Status, formatter Formatter) {
+	if stat.Code() == codes.OK {
+		fmt.Fprintln(w, "OK")
+		return
+	}
+	fmt.Fprintf(w, "ERROR:\n  Code: %s\n  Message: %s\n", stat.Code().String(), stat.Message())
+
+	statpb := stat.Proto()
+	if len(statpb.Details) > 0 {
+		fmt.Fprintf(w, "  Details:\n")
+		for i, det := range statpb.Details {
+			prefix := fmt.Sprintf("  %d)", i+1)
+			fmt.Fprintf(w, "%s\t", prefix)
+			prefix = strings.Repeat(" ", len(prefix)) + "\t"
+
+			output, err := formatter(det)
+			if err != nil {
+				fmt.Fprintf(w, "Error parsing detail message: %v\n", err)
+			} else {
+				lines := strings.Split(output, "\n")
+				for i, line := range lines {
+					if i == 0 {
+						// first line is already indented
+						fmt.Fprintf(w, "%s\n", line)
+					} else {
+						fmt.Fprintf(w, "%s%s\n", prefix, line)
+					}
+				}
+			}
+		}
+	}
+}

go.mod

@@ -1,8 +1,8 @@
 module github.com/fullstorydev/grpcurl
 
 require (
-	github.com/golang/protobuf v1.1.0
-	github.com/jhump/protoreflect v1.1.0
-	golang.org/x/net v0.0.0-20180530234432-1e491301e022
-	google.golang.org/grpc v1.12.0
+	github.com/golang/protobuf v1.3.5
+	github.com/jhump/protoreflect v1.5.0
+	golang.org/x/net v0.0.0-20190311183353-d8887717615a
+	google.golang.org/grpc v1.28.0
 )

go.sum

@@ -1,13 +1,58 @@
-github.com/golang/protobuf v1.1.0 h1:0iH4Ffd/meGoXqF2lSAhZHt8X+cPgkfn/cb6Cce5Vpc=
-github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/jhump/protoreflect v1.1.0 h1:h+zsMrsiq0vIl7yWmeowmd8e8VtnWk75U04GgXA2s6Y=
-github.com/jhump/protoreflect v1.1.0/go.mod h1:kG/zRVeS2M91gYaCvvUbPkMjjtFQS4qqjcPFzFkh2zE=
-golang.org/x/net v0.0.0-20180530234432-1e491301e022 h1:MVYFTUmVD3/+ERcvRRI+P/C2+WOUimXh+Pd8LVsklZ4=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/jhump/protoreflect v1.5.0 h1:NgpVT+dX71c8hZnxHof2M7QDK7QtohIJ7DYycjnkyfc=
+github.com/jhump/protoreflect v1.5.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-google.golang.org/genproto v0.0.0-20170818100345-ee236bd376b0 h1:jgaHBfsPDMBDKsth1hPtI1HcOyecWndWOFSGW21VgaM=
-google.golang.org/genproto v0.0.0-20170818100345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55 h1:gSJIx1SDwno+2ElGhA4+qG2zF97qiUzTM+rQ0klBOcE=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-google.golang.org/grpc v1.12.0 h1:Mm8atZtkT+P6R43n/dqNDWkPPu5BwRVu/1rJnJCeZH8=
-google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.28.0 h1:bO/TA4OxCOummhSf10siHuG7vJOiwh7SpRpFZDkOgl4=
+google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

grpcurl.go

@@ -15,9 +15,10 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net"
+	"os"
+	"regexp"
 	"sort"
 	"strings"
-	"time"
 
 	"github.com/golang/protobuf/proto"
 	descpb "github.com/golang/protobuf/protoc-gen-go/descriptor"
@@ -31,6 +32,7 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials"
 	"google.golang.org/grpc/metadata"
+	_ "google.golang.org/grpc/xds/experimental"
 )
 
 // ListServices uses the given descriptor source to return a sorted list of fully-qualified
@@ -162,6 +164,36 @@ func MetadataFromHeaders(headers []string) metadata.MD {
 	return md
 }
 
+var envVarRegex = regexp.MustCompile(`\${\w+}`)
+
+// ExpandHeaders expands environment variables contained in the header string.
+// If no corresponding environment variable is found an error is returned.
+// TODO: Add escaping for `${`
+func ExpandHeaders(headers []string) ([]string, error) {
+	expandedHeaders := make([]string, len(headers))
+	for idx, header := range headers {
+		if header == "" {
+			continue
+		}
+		results := envVarRegex.FindAllString(header, -1)
+		if len(results) == 0 {
+			expandedHeaders[idx] = headers[idx]
+			continue
+		}
+		expandedHeader := header
+		for _, result := range results {
+			envVarName := result[2 : len(result)-1] // strip leading `${` and trailing `}`
+			envVarValue, ok := os.LookupEnv(envVarName)
+			if !ok {
+				return nil, fmt.Errorf("header %q refers to missing environment variable %q", header, envVarName)
+			}
+			expandedHeader = strings.Replace(expandedHeader, result, envVarValue, -1)
+		}
+		expandedHeaders[idx] = expandedHeader
+	}
+	return expandedHeaders, nil
+}
+
 var base64Codecs = []*base64.Encoding{base64.StdEncoding, base64.URLEncoding, base64.RawStdEncoding, base64.RawURLEncoding}
 
 func decode(val string) (string, error) {
@@ -574,36 +606,44 @@ func BlockingDial(ctx context.Context, network, address string, creds credential
 		}
 	}
 
-	dialer := func(address string, timeout time.Duration) (net.Conn, error) {
-		ctx, cancel := context.WithTimeout(ctx, timeout)
-		defer cancel()
-
-		conn, err := (&net.Dialer{Cancel: ctx.Done()}).Dial(network, address)
+	// custom credentials and dialer will notify on error via the
+	// writeResult function
+	if creds != nil {
+		creds = &errSignalingCreds{
+			TransportCredentials: creds,
+			writeResult:          writeResult,
+		}
+	}
+	dialer := func(ctx context.Context, address string) (net.Conn, error) {
+		// NB: We *could* handle the TLS handshake ourselves, in the custom
+		// dialer (instead of customizing both the dialer and the credentials).
+		// But that requires using WithInsecure dial option (so that the gRPC
+		// library doesn't *also* try to do a handshake). And that would mean
+		// that the library would send the wrong ":scheme" metaheader to
+		// servers: it would send "http" instead of "https" because it is
+		// unaware that TLS is actually in use.
+		conn, err := (&net.Dialer{}).DialContext(ctx, network, address)
 		if err != nil {
 			writeResult(err)
-			return nil, err
 		}
-		if creds != nil {
-			conn, _, err = creds.ClientHandshake(ctx, address, conn)
-			if err != nil {
-				writeResult(err)
-				return nil, err
-			}
-		}
-		return conn, nil
+		return conn, err
 	}
 
 	// Even with grpc.FailOnNonTempDialError, this call will usually timeout in
 	// the face of TLS handshake errors. So we can't rely on grpc.WithBlock() to
 	// know when we're done. So we run it in a goroutine and then use result
-	// channel to either get the channel or fail-fast.
+	// channel to either get the connection or fail-fast.
 	go func() {
 		opts = append(opts,
 			grpc.WithBlock(),
 			grpc.FailOnNonTempDialError(true),
-			grpc.WithDialer(dialer),
-			grpc.WithInsecure(), // we are handling TLS, so tell grpc not to
+			grpc.WithContextDialer(dialer),
 		)
+		if creds == nil {
+			opts = append(opts, grpc.WithInsecure())
+		} else {
+			opts = append(opts, grpc.WithTransportCredentials(creds))
+		}
 		conn, err := grpc.DialContext(ctx, address, opts...)
 		var res interface{}
 		if err != nil {
@@ -624,3 +664,18 @@ func BlockingDial(ctx context.Context, network, address string, creds credential
 		return nil, ctx.Err()
 	}
 }
+
+// errSignalingCreds is a wrapper around a TransportCredentials value, but
+// it will use the writeResult function to notify on error.
+type errSignalingCreds struct {
+	credentials.TransportCredentials
+	writeResult func(res interface{})
+}
+
+func (c *errSignalingCreds) ClientHandshake(ctx context.Context, addr string, rawConn net.Conn) (net.Conn, credentials.AuthInfo, error) {
+	conn, auth, err := c.TransportCredentials.ClientHandshake(ctx, addr, rawConn)
+	if err != nil {
+		c.writeResult(err)
+	}
+	return conn, auth, err
+}

grpcurl_test.go

@@ -300,6 +300,33 @@ func TestGetAllFiles(t *testing.T) {
 	}
 }
 
+func TestExpandHeaders(t *testing.T) {
+	inHeaders := []string{"key1: ${value}", "key2: bar", "key3: ${woo", "key4: woo}", "key5: ${TEST}",
+		"key6: ${TEST_VAR}", "${TEST}: ${TEST_VAR}", "key8: ${EMPTY}"}
+	os.Setenv("value", "value")
+	os.Setenv("TEST", "value5")
+	os.Setenv("TEST_VAR", "value6")
+	os.Setenv("EMPTY", "")
+	expectedHeaders := map[string]bool{"key1: value": true, "key2: bar": true, "key3: ${woo": true, "key4: woo}": true,
+		"key5: value5": true, "key6: value6": true, "value5: value6": true, "key8: ": true}
+
+	outHeaders, err := ExpandHeaders(inHeaders)
+	if err != nil {
+		t.Errorf("The ExpandHeaders function generated an unexpected error %s", err)
+	}
+	for _, expandedHeader := range outHeaders {
+		if _, ok := expectedHeaders[expandedHeader]; !ok {
+			t.Errorf("The ExpandHeaders function has returned an unexpected header. Received unexpected header %s", expandedHeader)
+		}
+	}
+
+	badHeaders := []string{"key: ${DNE}"}
+	_, err = ExpandHeaders(badHeaders)
+	if err == nil {
+		t.Errorf("The ExpandHeaders function should return an error for missing environment variables %q", badHeaders)
+	}
+}
+
 func fileNames(files []*desc.FileDescriptor) []string {
 	names := make([]string, len(files))
 	for i, f := range files {

invoke.go

@@ -311,6 +311,7 @@ func invokeBidi(ctx context.Context, stub grpcdynamic.Stub, md *desc.MethodDescr
 				}
 				if err != nil {
 					err = fmt.Errorf("error getting request data: %v", err)
+					cancel()
 					break
 				}
 
@@ -321,7 +322,6 @@ func invokeBidi(ctx context.Context, stub grpcdynamic.Stub, md *desc.MethodDescr
 
 			if err != nil {
 				sendErr.Store(err)
-				cancel()
 			}
 		}()
 	}

testing/cmd/testserver/testserver.go

@@ -120,7 +120,7 @@ func unaryLogger(ctx context.Context, req interface{}, info *grpc.UnaryServerInf
 	} else {
 		code = codes.Unknown
 	}
-	grpclog.Infof("completed <%d>: %v (%d) %v\n", i, code, code, time.Now().Sub(start))
+	grpclog.Infof("completed <%d>: %v (%d) %v\n", i, code, code, time.Since(start))
 	return rsp, err
 }
 
@@ -135,7 +135,7 @@ func streamLogger(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServer
 	} else {
 		code = codes.Unknown
 	}
-	grpclog.Infof("completed <%d>: %v(%d) %v\n", i, code, code, time.Now().Sub(start))
+	grpclog.Infof("completed <%d>: %v(%d) %v\n", i, code, code, time.Since(start))
 	return err
 }
 

tls_settings_test.go

@@ -109,24 +109,42 @@ func TestBrokenTLS_ClientPlainText(t *testing.T) {
 	}
 
 	// client connection (usually) succeeds since client is not waiting for TLS handshake
-	e, err := createTestServerAndClient(serverCreds, nil)
-	if err != nil {
-		if strings.Contains(err.Error(), "deadline exceeded") {
-			// It is possible that connection never becomes healthy:
+	// (we try several times, but if we never get a connection and the error message is
+	// a known/expected possibility, we'll just bail)
+	var e testEnv
+	failCount := 0
+	for {
+		e, err = createTestServerAndClient(serverCreds, nil)
+		if err == nil {
+			// success!
+			defer e.Close()
+			break
+		}
+
+		if strings.Contains(err.Error(), "deadline exceeded") ||
+			strings.Contains(err.Error(), "use of closed network connection") {
+			// It is possible that the connection never becomes healthy:
 			//   1) grpc connects successfully
 			//   2) grpc client tries to send HTTP/2 preface and settings frame
 			//   3) server, expecting handshake, closes the connection
 			//   4) in the client, the write fails, so the connection never
 			//      becomes ready
-			// More often than not, the connection becomes ready (presumably
-			// the write to the socket succeeds before the server closes the
-			// connection). But when it does not, it is possible to observe
-			// timeouts when setting up the connection.
-			return
+			// The client will attempt to reconnect on transient errors, so
+			// may eventually bump into the connect time limit. This used to
+			// result in a "deadline exceeded" error, but more recent versions
+			// of the grpc library report any underlying I/O error instead, so
+			// we also check for "use of closed network connection".
+			failCount++
+			if failCount > 5 {
+				return // bail...
+			}
+			// we'll try again
+
+		} else {
+			// some other error occurred, so we'll consider that a test failure
+			t.Fatalf("failed to setup server and client: %v", err)
 		}
-		t.Fatalf("failed to setup server and client: %v", err)
 	}
-	defer e.Close()
 
 	// but request fails because server closes connection upon seeing request
 	// bytes that are not a TLS handshake
@@ -285,7 +303,7 @@ func simpleTest(t *testing.T, cc *grpc.ClientConn) {
 	cl := grpc_testing.NewTestServiceClient(cc)
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 	defer cancel()
-	_, err := cl.UnaryCall(ctx, &grpc_testing.SimpleRequest{}, grpc.FailFast(false))
+	_, err := cl.UnaryCall(ctx, &grpc_testing.SimpleRequest{}, grpc.WaitForReady(true))
 	if err != nil {
 		t.Errorf("simple RPC failed: %v", err)
 	}