cli: retry auth handshake deadline exceeded errors in CLI and Terraform (#2976)

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2024-03-08 13:15:06 +01:00 committed by GitHub
parent 52e4e64316
commit 27330490f3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 22 additions and 3 deletions

View File

@ -16,9 +16,10 @@ import (
)
const (
authEOFErr = `connection error: desc = "transport: authentication handshake failed: EOF"`
authReadTCPErr = `connection error: desc = "transport: authentication handshake failed: read tcp`
authHandshakeErr = `connection error: desc = "transport: authentication handshake failed`
authEOFErr = `connection error: desc = "transport: authentication handshake failed: EOF"`
authReadTCPErr = `connection error: desc = "transport: authentication handshake failed: read tcp`
authHandshakeErr = `connection error: desc = "transport: authentication handshake failed`
authHandshakeDeadlineExceededErr = `connection error: desc = "transport: authentication handshake failed: context deadline exceeded`
)
// grpcErr is the error type that is returned by the grpc client.
@ -57,6 +58,11 @@ func ServiceIsUnavailable(err error) bool {
return true
}
// retry if the handshake deadline was exceeded
if strings.HasPrefix(statusErr.Message(), authHandshakeDeadlineExceededErr) {
return true
}
return !strings.HasPrefix(statusErr.Message(), authHandshakeErr)
}
@ -76,6 +82,11 @@ func LoadbalancerIsNotReady(err error) bool {
return false
}
// retry if the handshake deadline was exceeded
if strings.HasPrefix(statusErr.Message(), authHandshakeDeadlineExceededErr) {
return true
}
// retry if GCP proxy LB isn't fully available yet
return strings.HasPrefix(statusErr.Message(), authReadTCPErr)
}

View File

@ -43,6 +43,10 @@ func TestServiceIsUnavailable(t *testing.T) {
err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`),
wantUnavailable: true,
},
"handshake deadline exceeded error": {
err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`),
wantUnavailable: true,
},
"wrapped error": {
err: fmt.Errorf("some wrapping: %w", status.Error(codes.Unavailable, "error")),
wantUnavailable: true,
@ -82,6 +86,10 @@ func TestLoadbalancerIsNotReady(t *testing.T) {
err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: read tcp error"`),
wantNotReady: true,
},
"handshake deadline exceeded error": {
err: status.Error(codes.Unavailable, `connection error: desc = "transport: authentication handshake failed: context deadline exceeded"`),
wantNotReady: true,
},
"normal unavailable error": {
err: status.Error(codes.Unavailable, "error"),
},