From 27330490f32f9d10d6da0515dcf876f7f6cfccc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Wei=C3=9Fe?= <66256922+daniel-weisse@users.noreply.github.com> Date: Fri, 8 Mar 2024 13:15:06 +0100 Subject: [PATCH] cli: retry auth handshake deadline exceeded errors in CLI and Terraform (#2976) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Daniel Weiße --- internal/grpc/retry/retry.go | 17 ++++++++++++++--- internal/grpc/retry/retry_test.go | 8 ++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/internal/grpc/retry/retry.go b/internal/grpc/retry/retry.go index 9d03279a4..b7457fc1f 100644 --- a/internal/grpc/retry/retry.go +++ b/internal/grpc/retry/retry.go @@ -16,9 +16,10 @@ import ( ) const ( - authEOFErr = `connection error: desc = "transport: authentication handshake failed: EOF"` - authReadTCPErr = `connection error: desc = "transport: authentication handshake failed: read tcp` - authHandshakeErr = `connection error: desc = "transport: authentication handshake failed` + authEOFErr = `connection error: desc = "transport: authentication handshake failed: EOF"` + authReadTCPErr = `connection error: desc = "transport: authentication handshake failed: read tcp` + authHandshakeErr = `connection error: desc = "transport: authentication handshake failed` + authHandshakeDeadlineExceededErr = `connection error: desc = "transport: authentication handshake failed: context deadline exceeded` ) // grpcErr is the error type that is returned by the grpc client. @@ -57,6 +58,11 @@ func ServiceIsUnavailable(err error) bool { return true } + // retry if the handshake deadline was exceeded + if strings.HasPrefix(statusErr.Message(), authHandshakeDeadlineExceededErr) { + return true + } + return !strings.HasPrefix(statusErr.Message(), authHandshakeErr) } @@ -76,6 +82,11 @@ func LoadbalancerIsNotReady(err error) bool { return false } + // retry if the handshake deadline was exceeded + if strings.HasPrefix(statusErr.Message(), authHandshakeDeadlineExceededErr) { + return true + } + // retry if GCP proxy LB isn't fully available yet return strings.HasPrefix(statusErr.Message(), authReadTCPErr) } diff --git a/internal/grpc/retry/retry_test.go b/internal/grpc/retry/retry_test.go index a1b44dce4..5e51e4bb0 100644 --- a/internal/grpc/retry/retry_test.go +++ b/internal/grpc/retry/retry_test.go @@ -43,6 +43,10 @@ func TestServiceIsUnavailable(t *testing.T) { err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`), wantUnavailable: true, }, + "handshake deadline exceeded error": { + err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`), + wantUnavailable: true, + }, "wrapped error": { err: fmt.Errorf("some wrapping: %w", status.Error(codes.Unavailable, "error")), wantUnavailable: true, @@ -82,6 +86,10 @@ func TestLoadbalancerIsNotReady(t *testing.T) { err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`), wantNotReady: true, }, + "handshake deadline exceeded error": { + err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`), + wantNotReady: true, + }, "normal unavailable error": { err: status.Error(codes.Unavailable, "error"), },