Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-05-20 07:00:32 -04:00
Code Issues Projects Releases Packages Wiki Activity

AB#2261 Add loadbalancer for control-plane recovery (#151)

Browse source 
Signed-off-by: Daniel Weiße <dw@edgeless.systems>
This commit is contained in:
Daniel Weiße 2022-09-14 13:25:42 +02:00 • committed by GitHub
parent 273d89e002
commit e367e1a68b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 418 additions and 243 deletions
Download patch file Download diff file Expand all files Collapse all files

123
cli/internal/cmd/recover.go
View file

@ -8,26 +8,28 @@ package cmd
import (
"context"
"errors"
"fmt"
"io"
"net"
"time"
"github.com/edgelesssys/constellation/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/cli/internal/proto"
"github.com/edgelesssys/constellation/internal/atls"
"github.com/edgelesssys/constellation/disk-mapper/recoverproto"
"github.com/edgelesssys/constellation/internal/attestation"
"github.com/edgelesssys/constellation/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/crypto"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/grpc/dialer"
grpcRetry "github.com/edgelesssys/constellation/internal/grpc/retry"
"github.com/edgelesssys/constellation/internal/retry"
"github.com/edgelesssys/constellation/internal/state"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"go.uber.org/multierr"
)
type recoveryClient interface {
Connect(endpoint string, validators atls.Validator) error
Recover(ctx context.Context, masterSecret, salt []byte) error
io.Closer
}
// NewRecoverCmd returns a new cobra.Command for the recover command.
func NewRecoverCmd() *cobra.Command {
cmd := &cobra.Command{
@ -46,12 +48,13 @@ func NewRecoverCmd() *cobra.Command {
func runRecover(cmd *cobra.Command, _ []string) error {
fileHandler := file.NewHandler(afero.NewOsFs())
recoveryClient := &proto.RecoverClient{}
defer recoveryClient.Close()
return recover(cmd, fileHandler, recoveryClient)
newDialer := func(validator *cloudcmd.Validator) *dialer.Dialer {
return dialer.New(nil, validator.V(cmd), &net.Dialer{})
}
return recover(cmd, fileHandler, newDialer)
}
func recover(cmd *cobra.Command, fileHandler file.Handler, recoveryClient recoveryClient) error {
func recover(cmd *cobra.Command, fileHandler file.Handler, newDialer func(validator *cloudcmd.Validator) *dialer.Dialer) error {
flags, err := parseRecoverFlags(cmd)
if err != nil {
return err
@ -73,29 +76,103 @@ func recover(cmd *cobra.Command, fileHandler file.Handler, recoveryClient recove
return fmt.Errorf("reading and validating config: %w", err)
}
validators, err := cloudcmd.NewValidator(provider, config)
validator, err := cloudcmd.NewValidator(provider, config)
if err != nil {
return err
}
if err := recoveryClient.Connect(flags.endpoint, validators.V(cmd)); err != nil {
return err
}
if err := recoveryClient.Recover(cmd.Context(), masterSecret.Key, masterSecret.Salt); err != nil {
return err
if err := recoverCall(cmd.Context(), newDialer(validator), flags.endpoint, masterSecret.Key, masterSecret.Salt); err != nil {
return fmt.Errorf("recovering cluster: %w", err)
}
cmd.Println("Pushed recovery key.")
return nil
}
func recoverCall(ctx context.Context, dialer grpcDialer, endpoint string, key, salt []byte) error {
measurementSecret, err := attestation.DeriveMeasurementSecret(key, salt)
if err != nil {
return err
}
doer := &recoverDoer{
dialer: dialer,
endpoint: endpoint,
getDiskKey: getStateDiskKeyFunc(key, salt),
measurementSecret: measurementSecret,
}
retrier := retry.NewIntervalRetrier(doer, 30*time.Second, grpcRetry.ServiceIsUnavailable)
if err := retrier.Do(ctx); err != nil {
return err
}
return nil
}
type recoverDoer struct {
dialer grpcDialer
endpoint string
measurementSecret []byte
getDiskKey func(uuid string) (key []byte, err error)
}
func (d *recoverDoer) Do(ctx context.Context) (retErr error) {
conn, err := d.dialer.Dial(ctx, d.endpoint)
if err != nil {
return fmt.Errorf("dialing recovery server: %w", err)
}
defer conn.Close()
// set up streaming client
protoClient := recoverproto.NewAPIClient(conn)
recoverclient, err := protoClient.Recover(ctx)
if err != nil {
return err
}
defer func() {
if err := recoverclient.CloseSend(); err != nil {
multierr.AppendInto(&retErr, err)
}
}()
// send measurement secret as first message
if err := recoverclient.Send(&recoverproto.RecoverMessage{
Request: &recoverproto.RecoverMessage_MeasurementSecret{
MeasurementSecret: d.measurementSecret,
},
}); err != nil {
return err
}
// receive disk uuid
res, err := recoverclient.Recv()
if err != nil {
return err
}
stateDiskKey, err := d.getDiskKey(res.DiskUuid)
if err != nil {
return err
}
// send disk key
if err := recoverclient.Send(&recoverproto.RecoverMessage{
Request: &recoverproto.RecoverMessage_StateDiskKey{
StateDiskKey: stateDiskKey,
},
}); err != nil {
return err
}
if _, err := recoverclient.Recv(); err != nil && !errors.Is(err, io.EOF) {
return err
}
return nil
}
func parseRecoverFlags(cmd *cobra.Command) (recoverFlags, error) {
endpoint, err := cmd.Flags().GetString("endpoint")
if err != nil {
return recoverFlags{}, fmt.Errorf("parsing endpoint argument: %w", err)
}
endpoint, err = addPortIfMissing(endpoint, constants.BootstrapperPort)
endpoint, err = addPortIfMissing(endpoint, constants.RecoveryPort)
if err != nil {
return recoverFlags{}, fmt.Errorf("validating endpoint argument: %w", err)
}
@ -122,3 +199,9 @@ type recoverFlags struct {
secretPath string
configPath string
}
func getStateDiskKeyFunc(masterKey, salt []byte) func(uuid string) ([]byte, error) {
return func(uuid string) ([]byte, error) {
return crypto.DeriveKey(masterKey, salt, []byte(crypto.HKDFInfoPrefix+uuid), crypto.StateDiskKeyLength)
}
}

267
cli/internal/cmd/recover_test.go
View file

@ -10,18 +10,25 @@ import (
"bytes"
"context"
"errors"
"net"
"strconv"
"testing"
"github.com/edgelesssys/constellation/internal/atls"
"github.com/edgelesssys/constellation/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/disk-mapper/recoverproto"
"github.com/edgelesssys/constellation/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/internal/config"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/crypto/testvector"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/internal/grpc/dialer"
"github.com/edgelesssys/constellation/internal/grpc/testdialer"
"github.com/edgelesssys/constellation/internal/state"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)
func TestRecoverCmdArgumentValidation(t *testing.T) {
@ -52,10 +59,25 @@ func TestRecoverCmdArgumentValidation(t *testing.T) {
func TestRecover(t *testing.T) {
validState := state.ConstellationState{CloudProvider: "GCP"}
invalidCSPState := state.ConstellationState{CloudProvider: "invalid"}
successActions := []func(stream recoverproto.API_RecoverServer) error{
func(stream recoverproto.API_RecoverServer) error {
_, err := stream.Recv()
return err
},
func(stream recoverproto.API_RecoverServer) error {
return stream.Send(&recoverproto.RecoverResponse{
DiskUuid: "00000000-0000-0000-0000-000000000000",
})
},
func(stream recoverproto.API_RecoverServer) error {
_, err := stream.Recv()
return err
},
}
testCases := map[string]struct {
existingState state.ConstellationState
client *stubRecoveryClient
recoverServerAPI *stubRecoveryServer
masterSecret testvector.HKDF
endpointFlag string
masterSecretFlag string
@ -64,46 +86,36 @@ func TestRecover(t *testing.T) {
wantErr bool
}{
"works": {
existingState: validState,
client: &stubRecoveryClient{},
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
existingState: validState,
recoverServerAPI: &stubRecoveryServer{actions: successActions},
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
},
"missing flags": {
wantErr: true,
recoverServerAPI: &stubRecoveryServer{actions: successActions},
wantErr: true,
},
"missing config": {
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
configFlag: "nonexistent-config",
wantErr: true,
recoverServerAPI: &stubRecoveryServer{actions: successActions},
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
configFlag: "nonexistent-config",
wantErr: true,
},
"missing state": {
existingState: validState,
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
stateless: true,
wantErr: true,
existingState: validState,
recoverServerAPI: &stubRecoveryServer{actions: successActions},
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
stateless: true,
wantErr: true,
},
"invalid cloud provider": {
existingState: invalidCSPState,
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
wantErr: true,
},
"connect fails": {
existingState: validState,
client: &stubRecoveryClient{connectErr: errors.New("connect failed")},
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
wantErr: true,
},
"pushing state key fails": {
existingState: validState,
client: &stubRecoveryClient{pushStateDiskKeyErr: errors.New("pushing key failed")},
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
wantErr: true,
existingState: invalidCSPState,
recoverServerAPI: &stubRecoveryServer{actions: successActions},
endpointFlag: "192.0.2.1",
masterSecret: testvector.HKDFZero,
wantErr: true,
},
}
@ -113,6 +125,7 @@ func TestRecover(t *testing.T) {
require := require.New(t)
cmd := NewRecoverCmd()
cmd.SetContext(context.Background())
cmd.Flags().String("config", constants.ConfigFilename, "") // register persistent flag manually
out := &bytes.Buffer{}
cmd.SetOut(out)
@ -133,12 +146,32 @@ func TestRecover(t *testing.T) {
config := defaultConfigWithExpectedMeasurements(t, config.Default(), cloudprovider.FromString(tc.existingState.CloudProvider))
require.NoError(fileHandler.WriteYAML(constants.ConfigFilename, config))
require.NoError(fileHandler.WriteJSON("constellation-mastersecret.json", masterSecret{Key: tc.masterSecret.Secret, Salt: tc.masterSecret.Salt}, file.OptNone))
require.NoError(fileHandler.WriteJSON(
"constellation-mastersecret.json",
masterSecret{Key: tc.masterSecret.Secret, Salt: tc.masterSecret.Salt},
file.OptNone,
))
if !tc.stateless {
require.NoError(fileHandler.WriteJSON(constants.StateFilename, tc.existingState, file.OptNone))
require.NoError(fileHandler.WriteJSON(
constants.StateFilename,
tc.existingState,
file.OptNone,
))
}
err := recover(cmd, fileHandler, tc.client)
netDialer := testdialer.NewBufconnDialer()
newDialer := func(*cloudcmd.Validator) *dialer.Dialer {
return dialer.New(nil, nil, netDialer)
}
serverCreds := atlscredentials.New(nil, nil)
recoverServer := grpc.NewServer(grpc.Creds(serverCreds))
recoverproto.RegisterAPIServer(recoverServer, tc.recoverServerAPI)
listener := netDialer.GetListener(net.JoinHostPort("192.0.2.1", strconv.Itoa(constants.RecoveryPort)))
go recoverServer.Serve(listener)
defer recoverServer.GracefulStop()
err := recover(cmd, fileHandler, newDialer)
if tc.wantErr {
assert.Error(err)
@ -201,26 +234,156 @@ func TestParseRecoverFlags(t *testing.T) {
}
}
type stubRecoveryClient struct {
conn bool
connectErr error
closeErr error
pushStateDiskKeyErr error
func TestDoRecovery(t *testing.T) {
someErr := errors.New("error")
testCases := map[string]struct {
recoveryServer *stubRecoveryServer
wantErr bool
}{
"success": {
recoveryServer: &stubRecoveryServer{actions: []func(stream recoverproto.API_RecoverServer) error{
func(stream recoverproto.API_RecoverServer) error {
_, err := stream.Recv()
return err
},
func(stream recoverproto.API_RecoverServer) error {
return stream.Send(&recoverproto.RecoverResponse{
DiskUuid: "00000000-0000-0000-0000-000000000000",
})
},
func(stream recoverproto.API_RecoverServer) error {
_, err := stream.Recv()
return err
},
}},
},
"error on first recv": {
recoveryServer: &stubRecoveryServer{actions: []func(stream recoverproto.API_RecoverServer) error{
func(stream recoverproto.API_RecoverServer) error {
return someErr
},
}},
wantErr: true,
},
"error on send": {
recoveryServer: &stubRecoveryServer{actions: []func(stream recoverproto.API_RecoverServer) error{
func(stream recoverproto.API_RecoverServer) error {
_, err := stream.Recv()
return err
},
func(stream recoverproto.API_RecoverServer) error {
return someErr
},
}},
wantErr: true,
},
"error on second recv": {
recoveryServer: &stubRecoveryServer{actions: []func(stream recoverproto.API_RecoverServer) error{
func(stream recoverproto.API_RecoverServer) error {
_, err := stream.Recv()
return err
},
func(stream recoverproto.API_RecoverServer) error {
return stream.Send(&recoverproto.RecoverResponse{
DiskUuid: "00000000-0000-0000-0000-000000000000",
})
},
func(stream recoverproto.API_RecoverServer) error {
return someErr
},
}},
wantErr: true,
},
"final message is an error": {
recoveryServer: &stubRecoveryServer{actions: []func(stream recoverproto.API_RecoverServer) error{
func(stream recoverproto.API_RecoverServer) error {
_, err := stream.Recv()
return err
},
func(stream recoverproto.API_RecoverServer) error {
return stream.Send(&recoverproto.RecoverResponse{
DiskUuid: "00000000-0000-0000-0000-000000000000",
})
},
func(stream recoverproto.API_RecoverServer) error {
_, err := stream.Recv()
return err
},
func(stream recoverproto.API_RecoverServer) error {
return someErr
},
}},
wantErr: true,
},
}
pushStateDiskKeyKey []byte
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
netDialer := testdialer.NewBufconnDialer()
serverCreds := atlscredentials.New(nil, nil)
recoverServer := grpc.NewServer(grpc.Creds(serverCreds))
recoverproto.RegisterAPIServer(recoverServer, tc.recoveryServer)
addr := net.JoinHostPort("192.0.2.1", strconv.Itoa(constants.RecoveryPort))
listener := netDialer.GetListener(addr)
go recoverServer.Serve(listener)
defer recoverServer.GracefulStop()
recoverDoer := &recoverDoer{
dialer: dialer.New(nil, nil, netDialer),
endpoint: addr,
measurementSecret: []byte("measurement-secret"),
getDiskKey: func(string) ([]byte, error) {
return []byte("disk-key"), nil
},
}
err := recoverDoer.Do(context.Background())
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
}
})
}
}
func (c *stubRecoveryClient) Connect(string, atls.Validator) error {
c.conn = true
return c.connectErr
func TestDeriveStateDiskKey(t *testing.T) {
testCases := map[string]struct {
masterSecret testvector.HKDF
}{
"all zero": {
masterSecret: testvector.HKDFZero,
},
"all 0xff": {
masterSecret: testvector.HKDF0xFF,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
getKeyFunc := getStateDiskKeyFunc(tc.masterSecret.Secret, tc.masterSecret.Salt)
stateDiskKey, err := getKeyFunc(tc.masterSecret.Info)
assert.NoError(err)
assert.Equal(tc.masterSecret.Output, stateDiskKey)
})
}
}
func (c *stubRecoveryClient) Close() error {
c.conn = false
return c.closeErr
type stubRecoveryServer struct {
actions []func(recoverproto.API_RecoverServer) error
recoverproto.UnimplementedAPIServer
}
func (c *stubRecoveryClient) Recover(_ context.Context, stateDiskKey, _ []byte) error {
c.pushStateDiskKeyKey = stateDiskKey
return c.pushStateDiskKeyErr
func (s *stubRecoveryServer) Recover(stream recoverproto.API_RecoverServer) error {
for _, action := range s.actions {
if err := action(stream); err != nil {
return err
}
}
return nil
}