debugd: Correctly handle direct coordinator upload if coordinator was uploaded previously (file already exists)

Signed-off-by: Malte Poll <mp@edgeless.systems>
This commit is contained in:
Malte Poll 2022-04-20 16:11:48 +02:00 committed by Malte Poll
parent f025afce98
commit 78af3b173f
5 changed files with 84 additions and 62 deletions

View File

@ -149,15 +149,20 @@ func deployOnEndpoint(ctx context.Context, in deployOnEndpointInput) error {
if err != nil {
return fmt.Errorf("starting coordinator upload to instance %v failed: %w", in.debugdEndpoint, err)
}
if err := in.reader.ReadStream(in.coordinatorPath, stream, debugd.Chunksize, true); err != nil {
return fmt.Errorf("streaming coordinator to instance failed: %w", err)
streamErr := in.reader.ReadStream(in.coordinatorPath, stream, debugd.Chunksize, true)
uploadResponse, closeErr := stream.CloseAndRecv()
if closeErr != nil {
return fmt.Errorf("closing upload stream after uploading coordinator to %v failed: %w", in.debugdEndpoint, closeErr)
}
uploadResponse, err := stream.CloseAndRecv()
if uploadResponse.Status != pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_SUCCESS || err != nil {
return fmt.Errorf("uploading coordinator to instance %v failed: %v / %w", in.debugdEndpoint, uploadResponse, err)
if uploadResponse.Status == pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_FILE_EXISTS {
log.Println("Coordinator was already uploaded")
return nil
}
if uploadResponse.Status != pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_SUCCESS || streamErr != nil {
return fmt.Errorf("uploading coordinator to instance %v failed: %v / %w", in.debugdEndpoint, uploadResponse, streamErr)
}
log.Println("Uploaded coordinator")
return nil
}

View File

@ -2,6 +2,9 @@ package server
import (
"context"
"errors"
"fmt"
"io/fs"
"log"
"net"
"sync"
@ -53,31 +56,38 @@ func (s *debugdServer) UploadCoordinator(stream pb.Debugd_UploadCoordinatorServe
Unit: debugd.CoordinatorSystemdUnitName,
Action: deploy.Stop,
})
log.Println("Starting coordinator upload")
if err := s.streamer.WriteStream(debugd.CoordinatorDeployFilename, stream, true); err != nil {
log.Printf("Uploading coordinator failed: %v\n", err)
return stream.SendAndClose(&pb.UploadCoordinatorResponse{
Status: pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_UPLOAD_FAILED,
})
}
log.Println("Successfully uploaded coordinator")
// after the upload succeeds, try to restart the coordinator
restartAction := deploy.ServiceManagerRequest{
Unit: debugd.CoordinatorSystemdUnitName,
Action: deploy.Restart,
}
var responseStatus pb.UploadCoordinatorStatus
defer func() {
if err := s.serviceManager.SystemdAction(stream.Context(), restartAction); err != nil {
log.Printf("Starting uploaded coordinator failed: %v\n", err)
return stream.SendAndClose(&pb.UploadCoordinatorResponse{
Status: pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_START_FAILED,
if responseStatus == pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_SUCCESS {
responseStatus = pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_START_FAILED
}
}
stream.SendAndClose(&pb.UploadCoordinatorResponse{
Status: responseStatus,
})
}()
log.Println("Starting coordinator upload")
if err := s.streamer.WriteStream(debugd.CoordinatorDeployFilename, stream, true); err != nil {
if errors.Is(err, fs.ErrExist) {
// coordinator was already uploaded
log.Println("Coordinator already uploaded")
responseStatus = pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_FILE_EXISTS
return nil
}
log.Printf("Uploading coordinator failed: %v\n", err)
responseStatus = pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_UPLOAD_FAILED
return fmt.Errorf("uploading coordinator failed: %w", err)
}
return stream.SendAndClose(&pb.UploadCoordinatorResponse{
Status: pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_SUCCESS,
})
log.Println("Successfully uploaded coordinator")
responseStatus = pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_SUCCESS
return nil
}
// DownloadCoordinator streams the local coordinator binary to other instances.

View File

@ -125,7 +125,7 @@ func TestUploadCoordinator(t *testing.T) {
writeStreamErr: errors.New("recv error"),
},
expectedResponseStatus: pb.UploadCoordinatorStatus_UPLOAD_COORDINATOR_UPLOAD_FAILED,
expectFile: true,
expectErr: true,
},
"starting coordinator fails": {
uploadChunks: [][]byte{

View File

@ -72,6 +72,7 @@ const (
UploadCoordinatorStatus_UPLOAD_COORDINATOR_SUCCESS UploadCoordinatorStatus = 0
UploadCoordinatorStatus_UPLOAD_COORDINATOR_UPLOAD_FAILED UploadCoordinatorStatus = 1
UploadCoordinatorStatus_UPLOAD_COORDINATOR_START_FAILED UploadCoordinatorStatus = 2
UploadCoordinatorStatus_UPLOAD_COORDINATOR_FILE_EXISTS UploadCoordinatorStatus = 3
)
// Enum value maps for UploadCoordinatorStatus.
@ -80,11 +81,13 @@ var (
0: "UPLOAD_COORDINATOR_SUCCESS",
1: "UPLOAD_COORDINATOR_UPLOAD_FAILED",
2: "UPLOAD_COORDINATOR_START_FAILED",
3: "UPLOAD_COORDINATOR_FILE_EXISTS",
}
UploadCoordinatorStatus_value = map[string]int32{
"UPLOAD_COORDINATOR_SUCCESS": 0,
"UPLOAD_COORDINATOR_UPLOAD_FAILED": 1,
"UPLOAD_COORDINATOR_START_FAILED": 2,
"UPLOAD_COORDINATOR_FILE_EXISTS": 3,
}
)
@ -641,7 +644,7 @@ var file_debugd_proto_rawDesc = []byte{
0x49, 0x5a, 0x45, 0x44, 0x5f, 0x4b, 0x45, 0x59, 0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53,
0x53, 0x10, 0x00, 0x12, 0x22, 0x0a, 0x1e, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x41, 0x55,
0x54, 0x48, 0x4f, 0x52, 0x49, 0x5a, 0x45, 0x44, 0x5f, 0x4b, 0x45, 0x59, 0x53, 0x5f, 0x46, 0x41,
0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x2a, 0x84, 0x01, 0x0a, 0x17, 0x55, 0x70, 0x6c, 0x6f,
0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x2a, 0xa8, 0x01, 0x0a, 0x17, 0x55, 0x70, 0x6c, 0x6f,
0x61, 0x64, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x53, 0x74, 0x61,
0x74, 0x75, 0x73, 0x12, 0x1e, 0x0a, 0x1a, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x43, 0x4f,
0x4f, 0x52, 0x44, 0x49, 0x4e, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53,
@ -649,42 +652,45 @@ var file_debugd_proto_rawDesc = []byte{
0x4f, 0x52, 0x44, 0x49, 0x4e, 0x41, 0x54, 0x4f, 0x52, 0x5f, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44,
0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x01, 0x12, 0x23, 0x0a, 0x1f, 0x55, 0x50, 0x4c,
0x4f, 0x41, 0x44, 0x5f, 0x43, 0x4f, 0x4f, 0x52, 0x44, 0x49, 0x4e, 0x41, 0x54, 0x4f, 0x52, 0x5f,
0x53, 0x54, 0x41, 0x52, 0x54, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x2a, 0x75,
0x0a, 0x1f, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x64, 0x53,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x53, 0x74, 0x61, 0x74, 0x75,
0x73, 0x12, 0x28, 0x0a, 0x24, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x53, 0x59, 0x53, 0x54,
0x45, 0x4d, 0x44, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x54,
0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12, 0x28, 0x0a, 0x24, 0x55,
0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x44, 0x5f, 0x53, 0x45,
0x52, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x54, 0x53, 0x5f, 0x46, 0x41, 0x49, 0x4c,
0x55, 0x52, 0x45, 0x10, 0x01, 0x32, 0xf9, 0x02, 0x0a, 0x06, 0x44, 0x65, 0x62, 0x75, 0x67, 0x64,
0x12, 0x63, 0x0a, 0x14, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72,
0x69, 0x7a, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x23, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67,
0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a,
0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x24, 0x2e,
0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x75, 0x74,
0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x11, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x43,
0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x0d, 0x2e, 0x64, 0x65, 0x62,
0x75, 0x67, 0x64, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x21, 0x2e, 0x64, 0x65, 0x62, 0x75,
0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e,
0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x28, 0x01,
0x12, 0x4c, 0x0a, 0x13, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6f, 0x72,
0x64, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x22, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64,
0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e,
0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x0d, 0x2e, 0x64, 0x65,
0x62, 0x75, 0x67, 0x64, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x22, 0x00, 0x30, 0x01, 0x12, 0x71,
0x0a, 0x18, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x53, 0x65,
0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x12, 0x28, 0x2e, 0x64, 0x65, 0x62,
0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d,
0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x71,
0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70,
0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69,
0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22,
0x00, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f,
0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73,
0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64,
0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x53, 0x54, 0x41, 0x52, 0x54, 0x5f, 0x46, 0x41, 0x49, 0x4c, 0x45, 0x44, 0x10, 0x02, 0x12, 0x22,
0x0a, 0x1e, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x43, 0x4f, 0x4f, 0x52, 0x44, 0x49, 0x4e,
0x41, 0x54, 0x4f, 0x52, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x5f, 0x45, 0x58, 0x49, 0x53, 0x54, 0x53,
0x10, 0x03, 0x2a, 0x75, 0x0a, 0x1f, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74,
0x65, 0x6d, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x53,
0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x28, 0x0a, 0x24, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f,
0x53, 0x59, 0x53, 0x54, 0x45, 0x4d, 0x44, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x43, 0x45, 0x5f,
0x55, 0x4e, 0x49, 0x54, 0x53, 0x5f, 0x53, 0x55, 0x43, 0x43, 0x45, 0x53, 0x53, 0x10, 0x00, 0x12,
0x28, 0x0a, 0x24, 0x55, 0x50, 0x4c, 0x4f, 0x41, 0x44, 0x5f, 0x53, 0x59, 0x53, 0x54, 0x45, 0x4d,
0x44, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x43, 0x45, 0x5f, 0x55, 0x4e, 0x49, 0x54, 0x53, 0x5f,
0x46, 0x41, 0x49, 0x4c, 0x55, 0x52, 0x45, 0x10, 0x01, 0x32, 0xf9, 0x02, 0x0a, 0x06, 0x44, 0x65,
0x62, 0x75, 0x67, 0x64, 0x12, 0x63, 0x0a, 0x14, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x75,
0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x12, 0x23, 0x2e, 0x64,
0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x41, 0x75, 0x74, 0x68,
0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73,
0x74, 0x1a, 0x24, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61,
0x64, 0x41, 0x75, 0x74, 0x68, 0x6f, 0x72, 0x69, 0x7a, 0x65, 0x64, 0x4b, 0x65, 0x79, 0x73, 0x52,
0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x49, 0x0a, 0x11, 0x55, 0x70, 0x6c,
0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x0d,
0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x1a, 0x21, 0x2e,
0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6f,
0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65,
0x22, 0x00, 0x28, 0x01, 0x12, 0x4c, 0x0a, 0x13, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64,
0x43, 0x6f, 0x6f, 0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x12, 0x22, 0x2e, 0x64, 0x65,
0x62, 0x75, 0x67, 0x64, 0x2e, 0x44, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x43, 0x6f, 0x6f,
0x72, 0x64, 0x69, 0x6e, 0x61, 0x74, 0x6f, 0x72, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a,
0x0d, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x22, 0x00,
0x30, 0x01, 0x12, 0x71, 0x0a, 0x18, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74,
0x65, 0x6d, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x12, 0x28,
0x2e, 0x64, 0x65, 0x62, 0x75, 0x67, 0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79,
0x73, 0x74, 0x65, 0x6d, 0x64, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74,
0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x29, 0x2e, 0x64, 0x65, 0x62, 0x75, 0x67,
0x64, 0x2e, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x53, 0x79, 0x73, 0x74, 0x65, 0x6d, 0x64, 0x53,
0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x55, 0x6e, 0x69, 0x74, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f,
0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f,
0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x64, 0x65,
0x62, 0x75, 0x67, 0x64, 0x2f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x62, 0x06, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x33,
}
var (

View File

@ -43,6 +43,7 @@ enum UploadCoordinatorStatus {
UPLOAD_COORDINATOR_SUCCESS = 0;
UPLOAD_COORDINATOR_UPLOAD_FAILED = 1;
UPLOAD_COORDINATOR_START_FAILED = 2;
UPLOAD_COORDINATOR_FILE_EXISTS = 3;
}
message ServiceUnit {