From f0723c62736815edefe386d02cd5dd79f270a3dd Mon Sep 17 00:00:00 2001
From: Joshua Humphries <jh@fullstory.com>
Date: Mon, 25 Mar 2019 10:34:32 -0400
Subject: [PATCH] fix flaky test (#93)

---
 tls_settings_test.go | 40 +++++++++++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/tls_settings_test.go b/tls_settings_test.go
index e520c27..82ecd76 100644
--- a/tls_settings_test.go
+++ b/tls_settings_test.go
@@ -109,24 +109,42 @@ func TestBrokenTLS_ClientPlainText(t *testing.T) {
 	}
 
 	// client connection (usually) succeeds since client is not waiting for TLS handshake
-	e, err := createTestServerAndClient(serverCreds, nil)
-	if err != nil {
-		if strings.Contains(err.Error(), "deadline exceeded") {
-			// It is possible that connection never becomes healthy:
+	// (we try several times, but if we never get a connection and the error message is
+	// a known/expected possibility, we'll just bail)
+	var e testEnv
+	failCount := 0
+	for {
+		e, err = createTestServerAndClient(serverCreds, nil)
+		if err == nil {
+			// success!
+			defer e.Close()
+			break
+		}
+
+		if strings.Contains(err.Error(), "deadline exceeded") ||
+			strings.Contains(err.Error(), "use of closed network connection") {
+			// It is possible that the connection never becomes healthy:
 			//   1) grpc connects successfully
 			//   2) grpc client tries to send HTTP/2 preface and settings frame
 			//   3) server, expecting handshake, closes the connection
 			//   4) in the client, the write fails, so the connection never
 			//      becomes ready
-			// More often than not, the connection becomes ready (presumably
-			// the write to the socket succeeds before the server closes the
-			// connection). But when it does not, it is possible to observe
-			// timeouts when setting up the connection.
-			return
+			// The client will attempt to reconnect on transient errors, so
+			// may eventually bump into the connect time limit. This used to
+			// result in a "deadline exceeded" error, but more recent versions
+			// of the grpc library report any underlying I/O error instead, so
+			// we also check for "use of closed network connection".
+			failCount++
+			if failCount > 5 {
+				return // bail...
+			}
+			// we'll try again
+
+		} else {
+			// some other error occurred, so we'll consider that a test failure
+			t.Fatalf("failed to setup server and client: %v", err)
 		}
-		t.Fatalf("failed to setup server and client: %v", err)
 	}
-	defer e.Close()
 
 	// but request fails because server closes connection upon seeing request
 	// bytes that are not a TLS handshake