cli: retry auth handshake deadline exceeded errors in CLI and Terraform (#2976)

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2024-03-08 13:15:06 +01:00 committed by GitHub
parent 52e4e64316
commit 27330490f3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 22 additions and 3 deletions

View File

@ -16,9 +16,10 @@ import (
) )
const ( const (
authEOFErr = `connection error: desc = "transport: authentication handshake failed: EOF"` authEOFErr = `connection error: desc = "transport: authentication handshake failed: EOF"`
authReadTCPErr = `connection error: desc = "transport: authentication handshake failed: read tcp` authReadTCPErr = `connection error: desc = "transport: authentication handshake failed: read tcp`
authHandshakeErr = `connection error: desc = "transport: authentication handshake failed` authHandshakeErr = `connection error: desc = "transport: authentication handshake failed`
authHandshakeDeadlineExceededErr = `connection error: desc = "transport: authentication handshake failed: context deadline exceeded`
) )
// grpcErr is the error type that is returned by the grpc client. // grpcErr is the error type that is returned by the grpc client.
@ -57,6 +58,11 @@ func ServiceIsUnavailable(err error) bool {
return true return true
} }
// retry if the handshake deadline was exceeded
if strings.HasPrefix(statusErr.Message(), authHandshakeDeadlineExceededErr) {
return true
}
return !strings.HasPrefix(statusErr.Message(), authHandshakeErr) return !strings.HasPrefix(statusErr.Message(), authHandshakeErr)
} }
@ -76,6 +82,11 @@ func LoadbalancerIsNotReady(err error) bool {
return false return false
} }
// retry if the handshake deadline was exceeded
if strings.HasPrefix(statusErr.Message(), authHandshakeDeadlineExceededErr) {
return true
}
// retry if GCP proxy LB isn't fully available yet // retry if GCP proxy LB isn't fully available yet
return strings.HasPrefix(statusErr.Message(), authReadTCPErr) return strings.HasPrefix(statusErr.Message(), authReadTCPErr)
} }

View File

@ -43,6 +43,10 @@ func TestServiceIsUnavailable(t *testing.T) {
err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`), err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`),
wantUnavailable: true, wantUnavailable: true,
}, },
"handshake deadline exceeded error": {
err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`),
wantUnavailable: true,
},
"wrapped error": { "wrapped error": {
err: fmt.Errorf("some wrapping: %w", status.Error(codes.Unavailable, "error")), err: fmt.Errorf("some wrapping: %w", status.Error(codes.Unavailable, "error")),
wantUnavailable: true, wantUnavailable: true,
@ -82,6 +86,10 @@ func TestLoadbalancerIsNotReady(t *testing.T) {
err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`), err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`),
wantNotReady: true, wantNotReady: true,
}, },
"handshake deadline exceeded error": {
err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`),
wantNotReady: true,
},
"normal unavailable error": { "normal unavailable error": {
err: status.Error(codes.Unavailable, "error"), err: status.Error(codes.Unavailable, "error"),
}, },