fix go.mod for Go 1.13 (#110)

To better support printing of google.protobuf.Any messages (error details), this
also makes a few other changes:
1. Allows printing of unresolvable Any messages using an "@value" field in JSON output
   that has the base64-encoded embedded message data.
2. Improves support for "-format text" to show expanded Any messages if possible.
   (Due to limitations in underlying proto package, this will usually *not* be
   that helpful. But this should greatly improve with v2 of the go protobuf API.)
3. Addresses a TODO in existing AnyResolver code to lazily fetch descriptors
   as needed instead of having to download all files eagerly.

.gitignore

@@ -1 +1,2 @@
 dist/
+VERSION

.travis.yml

@@ -5,13 +5,17 @@ matrix:
   include:
     - go: "1.9"
     - go: "1.10"
-      env: VET=1
     - go: "1.11"
-      env: GO111MODULE=off
+      env:
+      - GO111MODULE=off
+      - VET=1
     - go: "1.11"
       env: GO111MODULE=on
     - go: "1.12"
+      env: GO111MODULE=off
+    - go: "1.12"
+      env: GO111MODULE=on
     - go: tip
 
 script:
-  - if [[ "$VET" = 1 ]]; then make; else make deps test; fi
+  - if [[ "$VET" = 1 ]]; then make ci; else make deps test; fi

Dockerfile

@@ -0,0 +1,33 @@
+FROM golang:1.11.10-alpine as builder
+MAINTAINER FullStory Engineering
+
+# currently, a module build requires gcc (so Go tool can build
+# module-aware versions of std library; it ships only w/ the
+# non-module versions)
+RUN apk update && apk add --no-cache ca-certificates git gcc g++ libc-dev
+# create non-privileged group and user
+RUN addgroup -S grpcurl && adduser -S grpcurl -G grpcurl
+
+WORKDIR /tmp/fullstorydev/grpcurl
+# copy just the files/sources we need to build grpcurl
+COPY VERSION *.go go.* /tmp/fullstorydev/grpcurl/
+COPY cmd /tmp/fullstorydev/grpcurl/cmd
+# and build a completely static binary (so we can use
+# scratch as basis for the final image)
+ENV CGO_ENABLED=0
+ENV GOOS=linux
+ENV GOARCH=amd64
+ENV GO111MODULE=on
+RUN go build -o /grpcurl \
+    -ldflags "-w -extldflags \"-static\" -X \"main.version=$(cat VERSION)\"" \
+    ./cmd/grpcurl
+
+# New FROM so we have a nice'n'tiny image
+FROM scratch
+WORKDIR /
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt
+COPY --from=builder /etc/passwd /etc/passwd
+COPY --from=builder /grpcurl /bin/grpcurl
+USER grpcurl
+
+ENTRYPOINT ["/bin/grpcurl"]

Makefile

@@ -6,7 +6,7 @@ dev_build_version=$(shell git describe --tags --always --dirty)
 # they are just too noisy to be a requirement for a CI -- we don't even *want*
 # to fix some of the things they consider to be violations.
 .PHONY: ci
-ci: deps checkgofmt vet staticcheck unused ineffassign predeclared test
+ci: deps checkgofmt vet staticcheck ineffassign predeclared test
 
 .PHONY: deps
 deps:
@@ -25,6 +25,12 @@ release:
 	@GO111MODULE=off go get github.com/goreleaser/goreleaser
 	goreleaser --rm-dist
 
+.PHONY: docker
+docker:
+	@echo $(dev_build_version) > VERSION
+	docker build -t fullstorydev/grpcurl:$(dev_build_version) .
+	@rm VERSION
+
 .PHONY: checkgofmt
 checkgofmt:
 	gofmt -s -l .
@@ -47,12 +53,7 @@ vet:
 .PHONY: staticcheck
 staticcheck:
 	@go get honnef.co/go/tools/cmd/staticcheck
-	staticcheck -ignore github.com/fullstorydev/grpcurl/tls_settings_test.go:SA1019 ./...
+	staticcheck ./...
-
-.PHONY: unused
-unused:
-	@go get honnef.co/go/tools/cmd/unused
-	unused ./...
 
 .PHONY: ineffassign
 ineffassign:

cmd/grpcurl/grpcurl.go

@@ -8,6 +8,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 
@@ -161,7 +162,7 @@ func main() {
 		os.Exit(0)
 	}
 	if *printVersion {
-		fmt.Fprintf(os.Stderr, "%s %s\n", os.Args[0], version)
+		fmt.Fprintf(os.Stderr, "%s %s\n", filepath.Base(os.Args[0]), version)
 		os.Exit(0)
 	}
 
@@ -518,7 +519,7 @@ func main() {
 			fmt.Printf("Sent %d request%s and received %d response%s\n", reqCount, reqSuffix, h.NumResponses, respSuffix)
 		}
 		if h.Status.Code() != codes.OK {
-			fmt.Fprintf(os.Stderr, "ERROR:\n  Code: %s\n  Message: %s\n", h.Status.Code().String(), h.Status.Message())
+			grpcurl.PrintStatus(os.Stderr, h.Status, formatter)
 			exit(1)
 		}
 	}

desc_source.go

@@ -58,9 +58,14 @@ func DescriptorSourceFromProtoSets(fileNames ...string) (DescriptorSource, error
 // whose contents are Protocol Buffer source files. The given importPaths are used to locate
 // any imported files.
 func DescriptorSourceFromProtoFiles(importPaths []string, fileNames ...string) (DescriptorSource, error) {
+	fileNames, err := protoparse.ResolveFilenames(importPaths, fileNames...)
+	if err != nil {
+		return nil, err
+	}
 	p := protoparse.Parser{
-		ImportPaths:      importPaths,
+		ImportPaths:           importPaths,
-		InferImportPaths: len(importPaths) == 0,
+		InferImportPaths:      len(importPaths) == 0,
+		IncludeSourceCodeInfo: true,
 	}
 	fds, err := p.ParseFiles(fileNames...)
 	if err != nil {
@@ -109,7 +114,7 @@ func resolveFileDescriptor(unresolved map[string]*descpb.FileDescriptorProto, re
 	return result, nil
 }
 
-// DescriptorSourceFromFileDescriptorSet creates a DescriptorSource that is backed by the given
+// DescriptorSourceFromFileDescriptors creates a DescriptorSource that is backed by the given
 // file descriptors
 func DescriptorSourceFromFileDescriptors(files ...*desc.FileDescriptor) (DescriptorSource, error) {
 	fds := map[string]*desc.FileDescriptor{}

format.go

@@ -3,14 +3,19 @@ package grpcurl
 import (
 	"bufio"
 	"bytes"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"io"
+	"reflect"
+	"strings"
+	"sync"
 
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/golang/protobuf/proto"
 	"github.com/jhump/protoreflect/desc"
 	"github.com/jhump/protoreflect/dynamic"
+	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 )
@@ -142,6 +147,8 @@ type textFormatter struct {
 	numFormatted int
 }
 
+var protoTextMarshaler = proto.TextMarshaler{ExpandAny: true}
+
 func (tf *textFormatter) format(m proto.Message) (string, error) {
 	var buf bytes.Buffer
 	if tf.useSeparator && tf.numFormatted > 0 {
@@ -166,7 +173,7 @@ func (tf *textFormatter) format(m proto.Message) (string, error) {
 		if _, err := buf.Write(b); err != nil {
 			return "", err
 		}
-	} else if err := proto.MarshalText(&buf, m); err != nil {
+	} else if err := protoTextMarshaler.Marshal(&buf, m); err != nil {
 		return "", err
 	}
 
@@ -188,24 +195,153 @@ const (
 	FormatText = Format("text")
 )
 
-func anyResolver(source DescriptorSource) (jsonpb.AnyResolver, error) {
+// AnyResolverFromDescriptorSource returns an AnyResolver that will search for
-	// TODO: instead of pro-actively downloading file descriptors to
+// types using the given descriptor source.
-	// build a dynamic resolver, it would be better if the resolver
+func AnyResolverFromDescriptorSource(source DescriptorSource) jsonpb.AnyResolver {
-	// impl was lazy, and simply downloaded the descriptors as needed
+	return &anyResolver{source: source}
-	// when asked to resolve a particular type URL
-
-	// best effort: build resolver with whatever files we can
-	// load, ignoring any errors
-	files, _ := GetAllFiles(source)
-
-	var er dynamic.ExtensionRegistry
-	for _, fd := range files {
-		er.AddExtensionsFromFile(fd)
-	}
-	mf := dynamic.NewMessageFactoryWithExtensionRegistry(&er)
-	return dynamic.AnyResolver(mf, files...), nil
 }
 
+// AnyResolverFromDescriptorSourceWithFallback returns an AnyResolver that will
+// search for types using the given descriptor source and then fallback to a
+// special message if the type is not found. The fallback type will render to
+// JSON with a "@type" property, just like an Any message, but also with a
+// custom "@value" property that includes the binary encoded payload.
+func AnyResolverFromDescriptorSourceWithFallback(source DescriptorSource) jsonpb.AnyResolver {
+	res := anyResolver{source: source}
+	return &anyResolverWithFallback{AnyResolver: &res}
+}
+
+type anyResolver struct {
+	source DescriptorSource
+
+	er dynamic.ExtensionRegistry
+
+	mu       sync.RWMutex
+	mf       *dynamic.MessageFactory
+	resolved map[string]func() proto.Message
+}
+
+func (r *anyResolver) Resolve(typeUrl string) (proto.Message, error) {
+	mname := typeUrl
+	if slash := strings.LastIndex(mname, "/"); slash >= 0 {
+		mname = mname[slash+1:]
+	}
+
+	r.mu.RLock()
+	factory := r.resolved[mname]
+	r.mu.RUnlock()
+
+	// already resolved?
+	if factory != nil {
+		return factory(), nil
+	}
+
+	r.mu.Lock()
+	defer r.mu.Unlock()
+
+	// double-check, in case we were racing with another goroutine
+	// that resolved this one
+	factory = r.resolved[mname]
+	if factory != nil {
+		return factory(), nil
+	}
+
+	// use descriptor source to resolve message type
+	d, err := r.source.FindSymbol(mname)
+	if err != nil {
+		return nil, err
+	}
+	md, ok := d.(*desc.MessageDescriptor)
+	if !ok {
+		return nil, fmt.Errorf("unknown message: %s", typeUrl)
+	}
+	// populate any extensions for this message, too
+	if exts, err := r.source.AllExtensionsForType(mname); err != nil {
+		return nil, err
+	} else if err := r.er.AddExtension(exts...); err != nil {
+		return nil, err
+	}
+
+	if r.mf == nil {
+		r.mf = dynamic.NewMessageFactoryWithExtensionRegistry(&r.er)
+	}
+
+	factory = func() proto.Message {
+		return r.mf.NewMessage(md)
+	}
+	if r.resolved == nil {
+		r.resolved = map[string]func() proto.Message{}
+	}
+	r.resolved[mname] = factory
+	return factory(), nil
+}
+
+// anyResolverWithFallback can provide a fallback value for unknown
+// messages that will format itself to JSON using an "@value" field
+// that has the base64-encoded data for the unknown message value.
+type anyResolverWithFallback struct {
+	jsonpb.AnyResolver
+}
+
+func (r anyResolverWithFallback) Resolve(typeUrl string) (proto.Message, error) {
+	msg, err := r.AnyResolver.Resolve(typeUrl)
+	if err == nil {
+		return msg, err
+	}
+
+	// Try "default" resolution logic. This mirrors the default behavior
+	// of jsonpb, which checks to see if the given message name is registered
+	// in the proto package.
+	mname := typeUrl
+	if slash := strings.LastIndex(mname, "/"); slash >= 0 {
+		mname = mname[slash+1:]
+	}
+	mt := proto.MessageType(mname)
+	if mt != nil {
+		return reflect.New(mt.Elem()).Interface().(proto.Message), nil
+	}
+
+	// finally, fallback to a special placeholder that can marshal itself
+	// to JSON using a special "@value" property to show base64-encoded
+	// data for the embedded message
+	return &unknownAny{TypeUrl: typeUrl, Error: fmt.Sprintf("%s is not recognized; see @value for raw binary message data", mname)}, nil
+}
+
+type unknownAny struct {
+	TypeUrl string `json:"@type"`
+	Error   string `json:"@error"`
+	Value   string `json:"@value"`
+}
+
+func (a *unknownAny) MarshalJSONPB(jsm *jsonpb.Marshaler) ([]byte, error) {
+	if jsm.Indent != "" {
+		return json.MarshalIndent(a, "", jsm.Indent)
+	}
+	return json.Marshal(a)
+}
+
+func (a *unknownAny) Unmarshal(b []byte) error {
+	a.Value = base64.StdEncoding.EncodeToString(b)
+	return nil
+}
+
+func (a *unknownAny) Reset() {
+	a.Value = ""
+}
+
+func (a *unknownAny) String() string {
+	b, err := a.MarshalJSONPB(&jsonpb.Marshaler{})
+	if err != nil {
+		return fmt.Sprintf("ERROR: %v", err.Error())
+	}
+	return string(b)
+}
+
+func (a *unknownAny) ProtoMessage() {
+}
+
+var _ proto.Message = (*unknownAny)(nil)
+
 // RequestParserAndFormatterFor returns a request parser and formatter for the
 // given format. The given descriptor source may be used for parsing message
 // data (if needed by the format). The flags emitJSONDefaultFields and
@@ -214,11 +350,8 @@ func anyResolver(source DescriptorSource) (jsonpb.AnyResolver, error) {
 func RequestParserAndFormatterFor(format Format, descSource DescriptorSource, emitJSONDefaultFields, includeTextSeparator bool, in io.Reader) (RequestParser, Formatter, error) {
 	switch format {
 	case FormatJSON:
-		resolver, err := anyResolver(descSource)
+		resolver := AnyResolverFromDescriptorSource(descSource)
-		if err != nil {
+		return NewJSONRequestParser(in, resolver), NewJSONFormatter(emitJSONDefaultFields, anyResolverWithFallback{AnyResolver: resolver}), nil
-			return nil, nil, fmt.Errorf("error creating message resolver: %v", err)
-		}
-		return NewJSONRequestParser(in, resolver), NewJSONFormatter(emitJSONDefaultFields, resolver), nil
 	case FormatText:
 		return NewTextRequestParser(in), NewTextFormatter(includeTextSeparator), nil
 	default:
@@ -295,3 +428,42 @@ func (h *DefaultEventHandler) OnReceiveTrailers(stat *status.Status, md metadata
 		fmt.Fprintf(h.out, "\nResponse trailers received:\n%s\n", MetadataToString(md))
 	}
 }
+
+// PrintStatus prints details about the given status to the given writer. The given
+// formatter is used to print any detail messages that may be included in the status.
+// If the given status has a code of OK, "OK" is printed and that is all. Otherwise,
+// "ERROR:" is printed along with a line showing the code, one showing the message
+// string, and each detail message if any are present. The detail messages will be
+// printed as proto text format or JSON, depending on the given formatter.
+func PrintStatus(w io.Writer, stat *status.Status, formatter Formatter) {
+	if stat.Code() == codes.OK {
+		fmt.Fprintln(w, "OK")
+		return
+	}
+	fmt.Fprintf(w, "ERROR:\n  Code: %s\n  Message: %s\n", stat.Code().String(), stat.Message())
+
+	statpb := stat.Proto()
+	if len(statpb.Details) > 0 {
+		fmt.Fprintf(w, "  Details:\n")
+		for i, det := range statpb.Details {
+			prefix := fmt.Sprintf("  %d)", i+1)
+			fmt.Fprintf(w, "%s\t", prefix)
+			prefix = strings.Repeat(" ", len(prefix)) + "\t"
+
+			output, err := formatter(det)
+			if err != nil {
+				fmt.Fprintf(w, "Error parsing detail message: %v\n", err)
+			} else {
+				lines := strings.Split(output, "\n")
+				for i, line := range lines {
+					if i == 0 {
+						// first line is already indented
+						fmt.Fprintf(w, "%s\n", line)
+					} else {
+						fmt.Fprintf(w, "%s%s\n", prefix, line)
+					}
+				}
+			}
+		}
+	}
+}

go.mod

@@ -1,8 +1,8 @@
 module github.com/fullstorydev/grpcurl
 
 require (
-	github.com/golang/protobuf v1.2.0
+	github.com/golang/protobuf v1.3.1
-	github.com/jhump/protoreflect v1.2.0
+	github.com/jhump/protoreflect v1.5.0
-	golang.org/x/net v0.0.0-20180826012351-8a410e7b638d
+	golang.org/x/net v0.0.0-20190311183353-d8887717615a
-	google.golang.org/grpc v1.19.0
+	google.golang.org/grpc v1.21.0
 )

go.sum

@@ -3,26 +3,28 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
-github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-github.com/jhump/protoreflect v1.2.0 h1:W4HDXsqxgmE9OUmEXWaU2ZoUap/A29J5hW+zRXWbdC8=
+github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
-github.com/jhump/protoreflect v1.2.0/go.mod h1:kG/zRVeS2M91gYaCvvUbPkMjjtFQS4qqjcPFzFkh2zE=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
-golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/jhump/protoreflect v1.5.0 h1:NgpVT+dX71c8hZnxHof2M7QDK7QtohIJ7DYycjnkyfc=
+github.com/jhump/protoreflect v1.5.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d h1:g9qWBGx4puODJTMVyoPrpoxPFgVGd+z1DZwjfRu4d0I=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
-golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
-google.golang.org/genproto v0.0.0-20170818100345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20170818010345-ee236bd376b0/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
-google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
+google.golang.org/grpc v1.21.0 h1:G+97AoqBnmZIT91cLG/EkCoK9NSelj64P8bOHHNmGn0=
-google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

tls_settings_test.go

@@ -109,24 +109,42 @@ func TestBrokenTLS_ClientPlainText(t *testing.T) {
 	}
 
 	// client connection (usually) succeeds since client is not waiting for TLS handshake
-	e, err := createTestServerAndClient(serverCreds, nil)
+	// (we try several times, but if we never get a connection and the error message is
-	if err != nil {
+	// a known/expected possibility, we'll just bail)
-		if strings.Contains(err.Error(), "deadline exceeded") {
+	var e testEnv
-			// It is possible that connection never becomes healthy:
+	failCount := 0
+	for {
+		e, err = createTestServerAndClient(serverCreds, nil)
+		if err == nil {
+			// success!
+			defer e.Close()
+			break
+		}
+
+		if strings.Contains(err.Error(), "deadline exceeded") ||
+			strings.Contains(err.Error(), "use of closed network connection") {
+			// It is possible that the connection never becomes healthy:
 			//   1) grpc connects successfully
 			//   2) grpc client tries to send HTTP/2 preface and settings frame
 			//   3) server, expecting handshake, closes the connection
 			//   4) in the client, the write fails, so the connection never
 			//      becomes ready
-			// More often than not, the connection becomes ready (presumably
+			// The client will attempt to reconnect on transient errors, so
-			// the write to the socket succeeds before the server closes the
+			// may eventually bump into the connect time limit. This used to
-			// connection). But when it does not, it is possible to observe
+			// result in a "deadline exceeded" error, but more recent versions
-			// timeouts when setting up the connection.
+			// of the grpc library report any underlying I/O error instead, so
-			return
+			// we also check for "use of closed network connection".
+			failCount++
+			if failCount > 5 {
+				return // bail...
+			}
+			// we'll try again
+
+		} else {
+			// some other error occurred, so we'll consider that a test failure
+			t.Fatalf("failed to setup server and client: %v", err)
 		}
-		t.Fatalf("failed to setup server and client: %v", err)
 	}
-	defer e.Close()
 
 	// but request fails because server closes connection upon seeing request
 	// bytes that are not a TLS handshake
@@ -285,7 +303,7 @@ func simpleTest(t *testing.T, cc *grpc.ClientConn) {
 	cl := grpc_testing.NewTestServiceClient(cc)
 	ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
 	defer cancel()
-	_, err := cl.UnaryCall(ctx, &grpc_testing.SimpleRequest{}, grpc.FailFast(false))
+	_, err := cl.UnaryCall(ctx, &grpc_testing.SimpleRequest{}, grpc.WaitForReady(true))
 	if err != nil {
 		t.Errorf("simple RPC failed: %v", err)
 	}