Persist Node State to disk after node activation

Signed-off-by: Malte Poll <mp@edgeless.systems>

coordinator/core/core.go

@@ -230,6 +230,21 @@ func (c *Core) Initialize() (nodeActivated bool, err error) {
 	return nodeActivated, nil
 }
 
+// PersistNodeState persists node state to disk.
+func (c *Core) PersistNodeState(role role.Role, ownerID []byte, clusterID []byte) error {
+	vpnPrivKey, err := c.vpn.GetPrivateKey()
+	if err != nil {
+		return fmt.Errorf("failed to retrieve VPN private key: %w", err)
+	}
+	nodeState := nodestate.NodeState{
+		Role:       role,
+		VPNPrivKey: vpnPrivKey,
+		OwnerID:    ownerID,
+		ClusterID:  clusterID,
+	}
+	return nodeState.ToFile(c.fileHandler)
+}
+
 // SetUpKMS sets the Coordinators key management service and key encryption key ID.
 // Creates a new key encryption key in the KMS, if requested.
 // Otherwise the KEK is assumed to already exist in the KMS.

coordinator/core/core_test.go

@@ -245,6 +245,64 @@ func TestInitialize(t *testing.T) {
 	}
 }
 
+func TestPersistNodeState(t *testing.T) {
+	testCases := map[string]struct {
+		vpn            VPN
+		touchStateFile bool
+		errExpected    bool
+	}{
+		"persisting works": {
+			vpn: &stubVPN{
+				privateKey: []byte("private-key"),
+			},
+		},
+		"retrieving VPN key fails": {
+			vpn: &stubVPN{
+				getPrivateKeyErr: errors.New("error"),
+			},
+			errExpected: true,
+		},
+		"writing node state over existing file fails": {
+			vpn: &stubVPN{
+				privateKey: []byte("private-key"),
+			},
+			touchStateFile: true,
+			errExpected:    true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			fs := afero.NewMemMapFs()
+			fileHandler := file.NewHandler(fs)
+			if tc.touchStateFile {
+				file, err := fs.Create("/run/state/constellation/node_state.json")
+				require.NoError(err)
+				require.NoError(file.Close())
+			}
+			core, err := NewCore(tc.vpn, nil, nil, nil, nil, nil, zaptest.NewLogger(t), nil, nil, fileHandler)
+			require.NoError(err)
+			err = core.PersistNodeState(role.Coordinator, []byte("owner-id"), []byte("cluster-id"))
+			if tc.errExpected {
+				assert.Error(err)
+				return
+			}
+			require.NoError(err)
+			nodeState, err := nodestate.FromFile(fileHandler)
+			assert.NoError(err)
+			assert.Equal(nodestate.NodeState{
+				Role:       role.Coordinator,
+				VPNPrivKey: []byte("private-key"),
+				OwnerID:    []byte("owner-id"),
+				ClusterID:  []byte("cluster-id"),
+			}, *nodeState)
+		})
+	}
+}
+
 type fakeStoreFactory struct {
 	store store.Store
 }

coordinator/nodestate/nodestate.go

@@ -14,6 +14,8 @@ const nodeStatePath = "/run/state/constellation/node_state.json"
 type NodeState struct {
 	Role       role.Role
 	VPNPrivKey []byte
+	OwnerID    []byte
+	ClusterID  []byte
 }
 
 // FromFile reads a NodeState from disk.
@@ -27,5 +29,5 @@ func FromFile(fileHandler file.Handler) (*NodeState, error) {
 
 // ToFile writes a NodeState to disk.
 func (nodeState *NodeState) ToFile(fileHandler file.Handler) error {
-	return fileHandler.WriteJSON(nodeStatePath, nodeState, false, true)
+	return fileHandler.WriteJSON(nodeStatePath, nodeState, file.OptMkdirAll)
 }

coordinator/nodestate/nodestate_test.go

@@ -18,10 +18,12 @@ func TestFromFile(t *testing.T) {
 		errExpected   bool
 	}{
 		"nodestate exists": {
-			fileContents: `{	"Role": "Coordinator",	"VPNPrivKey": "dGVzdA=="	}`,
+			fileContents: `{	"Role": "Coordinator",	"VPNPrivKey": "dGVzdA==", "OwnerID": "T3duZXJJRA==", "ClusterID": "Q2x1c3RlcklE"	}`,
 			expectedState: &NodeState{
 				Role:       role.Coordinator,
 				VPNPrivKey: []byte("test"),
+				OwnerID:    []byte("OwnerID"),
+				ClusterID:  []byte("ClusterID"),
 			},
 		},
 		"nodestate file does not exist": {
@@ -62,10 +64,14 @@ func TestToFile(t *testing.T) {
 			state: &NodeState{
 				Role:       role.Coordinator,
 				VPNPrivKey: []byte("test"),
+				OwnerID:    []byte("OwnerID"),
+				ClusterID:  []byte("ClusterID"),
 			},
 			expectedFile: `{
 	"Role": "Coordinator",
-	"VPNPrivKey": "dGVzdA=="
+	"VPNPrivKey": "dGVzdA==",
+	"OwnerID": "T3duZXJJRA==",
+	"ClusterID": "Q2x1c3RlcklE"
 }`,
 		},
 		"file exists already": {
@@ -80,8 +86,8 @@ func TestToFile(t *testing.T) {
 			require := require.New(t)
 
 			fs := afero.NewMemMapFs()
-			require.NoError(fs.MkdirAll(filepath.Dir(nodeStatePath), 0o755))
 			if tc.precreateFile {
+				require.NoError(fs.MkdirAll(filepath.Dir(nodeStatePath), 0o755))
 				require.NoError(afero.WriteFile(fs, nodeStatePath, []byte("pre-existing"), 0o644))
 			}
 			fileHandler := file.NewHandler(fs)

coordinator/pubapi/coord.go

@@ -111,6 +111,11 @@ func (a *API) ActivateAsCoordinator(in *pubproto.ActivateAsCoordinatorRequest, s
 		return status.Errorf(codes.Internal, "%v", err)
 	}
 
+	// persist node state on disk
+	if err := a.core.PersistNodeState(role.Coordinator, ownerID, clusterID); err != nil {
+		return status.Errorf(codes.Internal, "%v", err)
+	}
+
 	// This effectively gives code execution, so we do this last.
 	adminVPNIP, err := a.core.AddAdmin(in.AdminVpnPubKey)
 	if err != nil {

coordinator/pubapi/coord_test.go

@@ -169,6 +169,7 @@ func TestActivateAsCoordinator(t *testing.T) {
 			assert.Equal(tc.expectedPeers, core.peers)
 			assert.Equal(autoscalingNodeGroups, core.autoscalingNodeGroups)
 			assert.Equal(keyEncryptionKeyID, core.kekID)
+			assert.Equal([]role.Role{role.Coordinator}, core.persistNodeStateRoles)
 		})
 	}
 }

coordinator/pubapi/core.go

@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/edgelesssys/constellation/coordinator/peer"
+	"github.com/edgelesssys/constellation/coordinator/role"
 	"github.com/edgelesssys/constellation/coordinator/state"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
@@ -17,6 +18,7 @@ type Core interface {
 	GetNextNodeIP() (string, error)
 	SwitchToPersistentStore() error
 	GetIDs(masterSecret []byte) (ownerID []byte, clusterID []byte, err error)
+	PersistNodeState(role role.Role, ownerID []byte, clusterID []byte) error
 	SetUpKMS(ctx context.Context, storageURI, kmsURI, kekID string, useExisting bool) error
 	GetDataKey(ctx context.Context, keyID string, length int) ([]byte, error)
 

coordinator/pubapi/core_test.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/edgelesssys/constellation/coordinator/peer"
+	"github.com/edgelesssys/constellation/coordinator/role"
 	"github.com/edgelesssys/constellation/coordinator/state"
 	kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
 )
@@ -26,6 +27,8 @@ type fakeCore struct {
 	autoscalingNodeGroups      []string
 	joinArgs                   []kubeadm.BootstrapTokenDiscovery
 	joinClusterErr             error
+	persistNodeStateRoles      []role.Role
+	persistNodeStateErr        error
 	kekID                      string
 	dataKey                    []byte
 	getDataKeyErr              error
@@ -108,6 +111,11 @@ func (c *fakeCore) JoinCluster(args kubeadm.BootstrapTokenDiscovery) error {
 	return c.joinClusterErr
 }
 
+func (c *fakeCore) PersistNodeState(role role.Role, ownerID []byte, clusterID []byte) error {
+	c.persistNodeStateRoles = append(c.persistNodeStateRoles, role)
+	return c.persistNodeStateErr
+}
+
 func (c *fakeCore) SetUpKMS(ctx context.Context, storageURI, kmsURI, kekID string, useExisting bool) error {
 	c.kekID = kekID
 	return nil

coordinator/pubapi/node.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/edgelesssys/constellation/coordinator/peer"
 	"github.com/edgelesssys/constellation/coordinator/pubapi/pubproto"
+	"github.com/edgelesssys/constellation/coordinator/role"
 	"github.com/edgelesssys/constellation/coordinator/state"
 	"github.com/edgelesssys/constellation/coordinator/vpnapi/vpnproto"
 	"go.uber.org/zap"
@@ -58,6 +59,11 @@ func (a *API) ActivateAsNode(ctx context.Context, in *pubproto.ActivateAsNodeReq
 		return nil, status.Errorf(codes.Internal, "%v", err)
 	}
 
+	// persist node state on disk
+	if err := a.core.PersistNodeState(role.Node, in.OwnerId, in.ClusterId); err != nil {
+		return nil, status.Errorf(codes.Internal, "%v", err)
+	}
+
 	// regularly get (peer) updates from Coordinator
 	a.wgClose.Add(1)
 	go a.updateLoop()

coordinator/pubapi/node_test.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/edgelesssys/constellation/coordinator/peer"
 	"github.com/edgelesssys/constellation/coordinator/pubapi/pubproto"
+	"github.com/edgelesssys/constellation/coordinator/role"
 	"github.com/edgelesssys/constellation/coordinator/state"
 	"github.com/edgelesssys/constellation/coordinator/util/testdialer"
 	"github.com/edgelesssys/constellation/coordinator/vpnapi/vpnproto"
@@ -121,6 +122,7 @@ func TestActivateAsNode(t *testing.T) {
 				require.Len(core.updatedPeers, 1)
 			}
 			assert.Equal(tc.initialPeers, core.updatedPeers[0])
+			assert.Equal([]role.Role{role.Node}, core.persistNodeStateRoles)
 		})
 	}
 }