Delete Coordinator core and apis

This commit is contained in:
katexochen 2022-06-21 17:59:12 +02:00 committed by Paul Meyer
parent e534c6a338
commit 32f1f5fd3e
93 changed files with 1824 additions and 16487 deletions

View file

@ -1,6 +1,7 @@
package cmd
import (
"context"
"encoding/base64"
"errors"
"fmt"
@ -9,29 +10,26 @@ import (
"net"
"strconv"
"text/tabwriter"
"time"
"github.com/edgelesssys/constellation/cli/internal/azure"
"github.com/edgelesssys/constellation/cli/internal/cloudcmd"
"github.com/edgelesssys/constellation/cli/internal/gcp"
"github.com/edgelesssys/constellation/cli/internal/proto"
"github.com/edgelesssys/constellation/cli/internal/vpn"
"github.com/edgelesssys/constellation/coordinator/pubapi/pubproto"
coordinatorstate "github.com/edgelesssys/constellation/coordinator/state"
"github.com/edgelesssys/constellation/coordinator/initproto"
"github.com/edgelesssys/constellation/coordinator/kms"
"github.com/edgelesssys/constellation/coordinator/util"
"github.com/edgelesssys/constellation/internal/atls"
"github.com/edgelesssys/constellation/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/internal/cloud/cloudtypes"
"github.com/edgelesssys/constellation/internal/config"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/deploy/ssh"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/grpc/dialer"
"github.com/edgelesssys/constellation/internal/grpc/retry"
"github.com/edgelesssys/constellation/internal/state"
"github.com/edgelesssys/constellation/internal/statuswaiter"
"github.com/kr/text"
wgquick "github.com/nmiculinic/wg-quick-go"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
"google.golang.org/grpc"
)
// NewInitCmd returns a new cobra.Command for the init command.
@ -44,10 +42,7 @@ func NewInitCmd() *cobra.Command {
Args: cobra.ExactArgs(0),
RunE: runInitialize,
}
cmd.Flags().String("privatekey", "", "path to your private key")
cmd.Flags().String("master-secret", "", "path to base64-encoded master secret")
cmd.Flags().Bool("wg-autoconfig", false, "enable automatic configuration of WireGuard interface")
must(cmd.Flags().MarkHidden("wg-autoconfig"))
cmd.Flags().Bool("autoscale", false, "enable Kubernetes cluster-autoscaler")
return cmd
}
@ -55,19 +50,15 @@ func NewInitCmd() *cobra.Command {
// runInitialize runs the initialize command.
func runInitialize(cmd *cobra.Command, args []string) error {
fileHandler := file.NewHandler(afero.NewOsFs())
vpnHandler := vpn.NewConfigHandler()
serviceAccountCreator := cloudcmd.NewServiceAccountCreator()
waiter := statuswaiter.New()
protoClient := &proto.Client{}
defer protoClient.Close()
return initialize(cmd, protoClient, serviceAccountCreator, fileHandler, waiter, vpnHandler)
dialer := dialer.New(nil, nil, &net.Dialer{})
return initialize(cmd, dialer, serviceAccountCreator, fileHandler)
}
// initialize initializes a Constellation. Coordinator instances are activated as contole-plane nodes and will
// themself activate the other peers as workers.
func initialize(cmd *cobra.Command, protCl protoClient, serviceAccCreator serviceAccountCreator,
fileHandler file.Handler, waiter statusWaiter, vpnHandler vpnHandler,
func initialize(cmd *cobra.Command, dialer grpcDialer, serviceAccCreator serviceAccountCreator,
fileHandler file.Handler,
) error {
flags, err := evalFlagArgs(cmd, fileHandler)
if err != nil {
@ -117,153 +108,79 @@ func initialize(cmd *cobra.Command, protCl protoClient, serviceAccCreator servic
return err
}
endpoints := ipsToEndpoints(append(coordinators.PublicIPs(), nodes.PublicIPs()...), strconv.Itoa(constants.CoordinatorPort))
cmd.Println("Waiting for cloud provider resource creation and boot ...")
if err := waiter.InitializeValidators(validators.V()); err != nil {
return err
}
if err := waiter.WaitForAll(cmd.Context(), endpoints, coordinatorstate.AcceptingInit); err != nil {
return fmt.Errorf("waiting for all peers status: %w", err)
}
var autoscalingNodeGroups []string
if flags.autoscale {
autoscalingNodeGroups = append(autoscalingNodeGroups, nodes.GroupID)
}
input := activationInput{
coordinatorPubIP: coordinators.PublicIPs()[0],
pubKey: flags.userPubKey,
masterSecret: flags.masterSecret,
nodePrivIPs: nodes.PrivateIPs(),
coordinatorPrivIPs: coordinators.PrivateIPs()[1:],
autoscalingNodeGroups: autoscalingNodeGroups,
cloudServiceAccountURI: serviceAccount,
sshUserKeys: ssh.ToProtoSlice(sshUsers),
req := &initproto.InitRequest{
AutoscalingNodeGroups: autoscalingNodeGroups,
MasterSecret: flags.masterSecret,
KmsUri: kms.ClusterKMSURI,
StorageUri: kms.NoStoreURI,
KeyEncryptionKeyId: "",
UseExistingKek: false,
CloudServiceAccountUri: serviceAccount,
KubernetesVersion: "1.23.6",
SshUserKeys: ssh.ToProtoSlice(sshUsers),
}
result, err := activate(cmd, protCl, input, validators.V())
resp, err := initCall(cmd.Context(), dialer, coordinators.PublicIPs()[0], req)
if err != nil {
return err
}
err = result.writeOutput(cmd.OutOrStdout(), fileHandler)
if err != nil {
if err := writeOutput(resp, cmd.OutOrStdout(), fileHandler); err != nil {
return err
}
vpnConfig, err := vpnHandler.Create(result.coordinatorPubKey, result.coordinatorPubIP, string(flags.userPrivKey), result.clientVpnIP, constants.WireguardAdminMTU)
if err != nil {
return err
}
if err := writeWGQuickFile(fileHandler, vpnHandler, vpnConfig); err != nil {
return fmt.Errorf("writing wg-quick file: %w", err)
}
if flags.autoconfigureWG {
if err := vpnHandler.Apply(vpnConfig); err != nil {
return fmt.Errorf("configuring WireGuard: %w", err)
}
}
return nil
}
func activate(cmd *cobra.Command, client protoClient, input activationInput,
validators []atls.Validator,
) (activationResult, error) {
err := client.Connect(net.JoinHostPort(input.coordinatorPubIP, strconv.Itoa(constants.CoordinatorPort)), validators)
if err != nil {
return activationResult{}, err
func initCall(ctx context.Context, dialer grpcDialer, ip string, req *initproto.InitRequest) (*initproto.InitResponse, error) {
doer := &initDoer{
dialer: dialer,
endpoint: net.JoinHostPort(ip, strconv.Itoa(constants.CoordinatorPort)),
req: req,
}
respCl, err := client.Activate(cmd.Context(), input.pubKey, input.masterSecret, input.nodePrivIPs, input.coordinatorPrivIPs, input.autoscalingNodeGroups, input.cloudServiceAccountURI, input.sshUserKeys)
if err != nil {
return activationResult{}, err
retryer := retry.NewIntervalRetryer(doer, 30*time.Second)
if err := retryer.Do(ctx); err != nil {
return nil, err
}
indentOut := text.NewIndentWriter(cmd.OutOrStdout(), []byte{'\t'})
cmd.Println("Activating the cluster ...")
if err := respCl.WriteLogStream(indentOut); err != nil {
return activationResult{}, err
}
clientVpnIp, err := respCl.GetClientVpnIp()
if err != nil {
return activationResult{}, err
}
coordinatorPubKey, err := respCl.GetCoordinatorVpnKey()
if err != nil {
return activationResult{}, err
}
kubeconfig, err := respCl.GetKubeconfig()
if err != nil {
return activationResult{}, err
}
ownerID, err := respCl.GetOwnerID()
if err != nil {
return activationResult{}, err
}
clusterID, err := respCl.GetClusterID()
if err != nil {
return activationResult{}, err
}
return activationResult{
clientVpnIP: clientVpnIp,
coordinatorPubKey: coordinatorPubKey,
coordinatorPubIP: input.coordinatorPubIP,
kubeconfig: kubeconfig,
ownerID: ownerID,
clusterID: clusterID,
}, nil
return doer.resp, nil
}
type activationInput struct {
coordinatorPubIP string
pubKey []byte
masterSecret []byte
nodePrivIPs []string
coordinatorPrivIPs []string
autoscalingNodeGroups []string
cloudServiceAccountURI string
sshUserKeys []*pubproto.SSHUserKey
type initDoer struct {
dialer grpcDialer
endpoint string
req *initproto.InitRequest
resp *initproto.InitResponse
}
type activationResult struct {
clientVpnIP string
coordinatorPubKey string
coordinatorPubIP string
kubeconfig string
ownerID string
clusterID string
}
// writeWGQuickFile writes the wg-quick file to the default path.
func writeWGQuickFile(fileHandler file.Handler, vpnHandler vpnHandler, vpnConfig *wgquick.Config) error {
data, err := vpnHandler.Marshal(vpnConfig)
func (d *initDoer) Do(ctx context.Context) error {
conn, err := d.dialer.Dial(ctx, d.endpoint)
if err != nil {
return fmt.Errorf("dialing init server: %w", err)
}
protoClient := initproto.NewAPIClient(conn)
resp, err := protoClient.Init(ctx, d.req)
if err != nil {
return fmt.Errorf("marshalling VPN config: %w", err)
}
return fileHandler.Write(constants.WGQuickConfigFilename, data, file.OptNone)
d.resp = resp
return nil
}
func (r activationResult) writeOutput(wr io.Writer, fileHandler file.Handler) error {
func writeOutput(resp *initproto.InitResponse, wr io.Writer, fileHandler file.Handler) error {
fmt.Fprint(wr, "Your Constellation cluster was successfully initialized.\n\n")
tw := tabwriter.NewWriter(wr, 0, 0, 2, ' ', 0)
writeRow(tw, "Your WireGuard IP", r.clientVpnIP)
writeRow(tw, "Control plane's public IP", r.coordinatorPubIP)
writeRow(tw, "Control plane's public key", r.coordinatorPubKey)
writeRow(tw, "Constellation cluster's owner identifier", r.ownerID)
writeRow(tw, "Constellation cluster's unique identifier", r.clusterID)
writeRow(tw, "WireGuard configuration file", constants.WGQuickConfigFilename)
writeRow(tw, "Constellation cluster's owner identifier", string(resp.OwnerId))
writeRow(tw, "Constellation cluster's unique identifier", string(resp.ClusterId))
writeRow(tw, "Kubernetes configuration", constants.AdminConfFilename)
tw.Flush()
fmt.Fprintln(wr)
if err := fileHandler.Write(constants.AdminConfFilename, []byte(r.kubeconfig), file.OptNone); err != nil {
if err := fileHandler.Write(constants.AdminConfFilename, resp.Kubeconfig, file.OptNone); err != nil {
return fmt.Errorf("write kubeconfig: %w", err)
}
@ -273,7 +190,6 @@ func (r activationResult) writeOutput(wr io.Writer, fileHandler file.Handler) er
}
fmt.Fprintln(wr, "You can now connect to your cluster by executing:")
fmt.Fprintf(wr, "\twg-quick up ./%s\n", constants.WGQuickConfigFilename)
fmt.Fprintf(wr, "\texport KUBECONFIG=\"$PWD/%s\"\n", constants.AdminConfFilename)
return nil
}
@ -285,18 +201,6 @@ func writeRow(wr io.Writer, col1 string, col2 string) {
// evalFlagArgs gets the flag values and does preprocessing of these values like
// reading the content from file path flags and deriving other values from flag combinations.
func evalFlagArgs(cmd *cobra.Command, fileHandler file.Handler) (initFlags, error) {
userPrivKeyPath, err := cmd.Flags().GetString("privatekey")
if err != nil {
return initFlags{}, fmt.Errorf("parsing privatekey path argument: %w", err)
}
userPrivKey, userPubKey, err := readOrGenerateVPNKey(fileHandler, userPrivKeyPath)
if err != nil {
return initFlags{}, err
}
autoconfigureWG, err := cmd.Flags().GetBool("wg-autoconfig")
if err != nil {
return initFlags{}, fmt.Errorf("parsing wg-autoconfig argument: %w", err)
}
masterSecretPath, err := cmd.Flags().GetString("master-secret")
if err != nil {
return initFlags{}, fmt.Errorf("parsing master-secret path argument: %w", err)
@ -315,58 +219,17 @@ func evalFlagArgs(cmd *cobra.Command, fileHandler file.Handler) (initFlags, erro
}
return initFlags{
configPath: configPath,
userPrivKey: userPrivKey,
userPubKey: userPubKey,
autoconfigureWG: autoconfigureWG,
autoscale: autoscale,
masterSecret: masterSecret,
configPath: configPath,
autoscale: autoscale,
masterSecret: masterSecret,
}, nil
}
// initFlags are the resulting values of flag preprocessing.
type initFlags struct {
configPath string
userPrivKey []byte
userPubKey []byte
masterSecret []byte
autoconfigureWG bool
autoscale bool
}
func readOrGenerateVPNKey(fileHandler file.Handler, privKeyPath string) (privKey, pubKey []byte, err error) {
var privKeyParsed wgtypes.Key
if privKeyPath == "" {
privKeyParsed, err = wgtypes.GeneratePrivateKey()
if err != nil {
return nil, nil, fmt.Errorf("generating WireGuard private key: %w", err)
}
privKey = []byte(privKeyParsed.String())
} else {
privKey, err = fileHandler.Read(privKeyPath)
if err != nil {
return nil, nil, fmt.Errorf("reading the VPN private key: %w", err)
}
privKeyParsed, err = wgtypes.ParseKey(string(privKey))
if err != nil {
return nil, nil, fmt.Errorf("parsing the WireGuard private key: %w", err)
}
}
pubKey = []byte(privKeyParsed.PublicKey().String())
return privKey, pubKey, nil
}
func ipsToEndpoints(ips []string, port string) []string {
var endpoints []string
for _, ip := range ips {
if ip == "" {
continue
}
endpoints = append(endpoints, net.JoinHostPort(ip, port))
}
return endpoints
configPath string
masterSecret []byte
autoscale bool
}
// readOrGenerateMasterSecret reads a base64 encoded master secret from file or generates a new 32 byte secret.
@ -491,3 +354,7 @@ func initCompletion(cmd *cobra.Command, args []string, toComplete string) ([]str
}
return []string{}, cobra.ShellCompDirectiveDefault
}
type grpcDialer interface {
Dial(ctx context.Context, target string) (*grpc.ClientConn, error)
}

View file

@ -5,21 +5,25 @@ import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"net"
"strconv"
"strings"
"testing"
"time"
"github.com/edgelesssys/constellation/coordinator/initproto"
"github.com/edgelesssys/constellation/internal/cloud/cloudtypes"
"github.com/edgelesssys/constellation/internal/constants"
"github.com/edgelesssys/constellation/internal/file"
"github.com/edgelesssys/constellation/internal/grpc/atlscredentials"
"github.com/edgelesssys/constellation/internal/grpc/dialer"
"github.com/edgelesssys/constellation/internal/grpc/testdialer"
"github.com/edgelesssys/constellation/internal/state"
wgquick "github.com/nmiculinic/wg-quick-go"
"github.com/spf13/afero"
"github.com/spf13/cobra"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/grpc"
)
func TestInitArgumentValidation(t *testing.T) {
@ -32,7 +36,6 @@ func TestInitArgumentValidation(t *testing.T) {
}
func TestInitialize(t *testing.T) {
testKey := base64.StdEncoding.EncodeToString([]byte("32bytesWireGuardKeyForTheTesting"))
testGcpState := state.ConstellationState{
CloudProvider: "GCP",
GCPNodes: cloudtypes.Instances{
@ -64,221 +67,73 @@ func TestInitialize(t *testing.T) {
"id-c": {PrivateIP: "192.0.2.1", PublicIP: "192.0.2.1"},
},
}
testActivationResps := []fakeActivationRespMessage{
{log: "testlog1"},
{log: "testlog2"},
{
kubeconfig: "kubeconfig",
clientVpnIp: "192.0.2.2",
coordinatorVpnKey: testKey,
ownerID: "ownerID",
clusterID: "clusterID",
},
{log: "testlog3"},
testInitResp := &initproto.InitResponse{
Kubeconfig: []byte("kubeconfig"),
OwnerId: []byte("ownerID"),
ClusterId: []byte("clusterID"),
}
someErr := errors.New("failed")
// someErr := errors.New("failed")
testCases := map[string]struct {
existingState state.ConstellationState
client protoClient
serviceAccountCreator stubServiceAccountCreator
waiter statusWaiter
privKey string
vpnHandler vpnHandler
initVPN bool
initServerAPI *stubInitServer
setAutoscaleFlag bool
wantErr bool
}{
"initialize some gcp instances": {
existingState: testGcpState,
client: &fakeProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
vpnHandler: &stubVPNHandler{},
privKey: testKey,
initServerAPI: &stubInitServer{initResp: testInitResp},
},
"initialize some azure instances": {
existingState: testAzureState,
client: &fakeProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
vpnHandler: &stubVPNHandler{},
privKey: testKey,
initServerAPI: &stubInitServer{initResp: testInitResp},
},
"initialize some qemu instances": {
existingState: testQemuState,
client: &fakeProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
vpnHandler: &stubVPNHandler{},
privKey: testKey,
initServerAPI: &stubInitServer{initResp: testInitResp},
},
"initialize vpn": {
existingState: testAzureState,
client: &fakeProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
vpnHandler: &stubVPNHandler{},
initVPN: true,
privKey: testKey,
"initialize gcp with autoscaling": {
existingState: testGcpState,
initServerAPI: &stubInitServer{initResp: testInitResp},
setAutoscaleFlag: true,
},
"invalid initialize vpn": {
existingState: testAzureState,
client: &fakeProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
vpnHandler: &stubVPNHandler{applyErr: someErr},
initVPN: true,
privKey: testKey,
wantErr: true,
},
"invalid create vpn config": {
existingState: testAzureState,
client: &fakeProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
vpnHandler: &stubVPNHandler{createErr: someErr},
initVPN: true,
privKey: testKey,
wantErr: true,
},
"invalid write vpn config": {
existingState: testAzureState,
client: &fakeProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
vpnHandler: &stubVPNHandler{marshalErr: someErr},
initVPN: true,
privKey: testKey,
wantErr: true,
},
"no state exists": {
existingState: state.ConstellationState{},
client: &stubProtoClient{},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"no instances to pick one": {
existingState: state.ConstellationState{GCPNodes: cloudtypes.Instances{}},
client: &stubProtoClient{},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"public key to short": {
existingState: testGcpState,
client: &stubProtoClient{},
waiter: &stubStatusWaiter{},
privKey: base64.StdEncoding.EncodeToString([]byte("tooShortKey")),
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"public key to long": {
existingState: testGcpState,
client: &stubProtoClient{},
waiter: &stubStatusWaiter{},
privKey: base64.StdEncoding.EncodeToString([]byte("thisWireguardKeyIsToLongAndHasTooManyBytes")),
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"public key not base64": {
existingState: testGcpState,
client: &stubProtoClient{},
waiter: &stubStatusWaiter{},
privKey: "this is not base64 encoded",
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail Connect": {
existingState: testGcpState,
client: &stubProtoClient{connectErr: someErr},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail Activate": {
existingState: testGcpState,
client: &stubProtoClient{activateErr: someErr},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail respClient WriteLogStream": {
existingState: testGcpState,
client: &stubProtoClient{respClient: &stubActivationRespClient{writeLogStreamErr: someErr}},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail respClient getKubeconfig": {
existingState: testGcpState,
client: &stubProtoClient{respClient: &stubActivationRespClient{getKubeconfigErr: someErr}},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail respClient getCoordinatorVpnKey": {
existingState: testGcpState,
client: &stubProtoClient{respClient: &stubActivationRespClient{getCoordinatorVpnKeyErr: someErr}},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail respClient getClientVpnIp": {
existingState: testGcpState,
client: &stubProtoClient{respClient: &stubActivationRespClient{getClientVpnIpErr: someErr}},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail respClient getOwnerID": {
existingState: testGcpState,
client: &stubProtoClient{respClient: &stubActivationRespClient{getOwnerIDErr: someErr}},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail respClient getClusterID": {
existingState: testGcpState,
client: &stubProtoClient{respClient: &stubActivationRespClient{getClusterIDErr: someErr}},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail to wait for required status": {
existingState: testGcpState,
client: &stubProtoClient{},
waiter: &stubStatusWaiter{waitForAllErr: someErr},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
},
"fail to create service account": {
existingState: testGcpState,
client: &stubProtoClient{},
serviceAccountCreator: stubServiceAccountCreator{createErr: someErr},
waiter: &stubStatusWaiter{},
privKey: testKey,
vpnHandler: &stubVPNHandler{},
wantErr: true,
"initialize azure with autoscaling": {
existingState: testAzureState,
initServerAPI: &stubInitServer{initResp: testInitResp},
setAutoscaleFlag: true,
},
// "no state exists": {
// existingState: state.ConstellationState{},
// initServerAPI: &stubInitServer{},
// wantErr: true,
// },
// "no instances to pick one": {
// existingState: state.ConstellationState{GCPNodes: cloudtypes.Instances{}},
// initServerAPI: &stubInitServer{},
// wantErr: true,
// },
// "fail Connect": {
// existingState: testGcpState,
// initServerAPI: &stubInitServer{},
// wantErr: true,
// },
// "fail Activate": {
// existingState: testGcpState,
// initServerAPI: &stubInitServer{},
// wantErr: true,
// },
// "fail to wait for required status": {
// existingState: testGcpState,
// initServerAPI: &stubInitServer{},
// wantErr: true,
// },
// "fail to create service account": {
// existingState: testGcpState,
// initServerAPI: &stubInitServer{},
// serviceAccountCreator: stubServiceAccountCreator{createErr: someErr},
// wantErr: true,
// },
}
for name, tc := range testCases {
@ -286,6 +141,16 @@ func TestInitialize(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
netDialer := testdialer.NewBufconnDialer()
dialer := dialer.New(nil, nil, netDialer)
serverCreds := atlscredentials.New(nil, nil)
initServer := grpc.NewServer(grpc.Creds(serverCreds))
initproto.RegisterAPIServer(initServer, tc.initServerAPI)
port := strconv.Itoa(constants.CoordinatorPort)
listener := netDialer.GetListener(net.JoinHostPort("192.0.2.1", port))
go initServer.Serve(listener)
defer initServer.GracefulStop()
cmd := NewInitCmd()
var out bytes.Buffer
cmd.SetOut(&out)
@ -295,29 +160,27 @@ func TestInitialize(t *testing.T) {
fs := afero.NewMemMapFs()
fileHandler := file.NewHandler(fs)
require.NoError(fileHandler.WriteJSON(constants.StateFilename, tc.existingState, file.OptNone))
// Write key file to filesystem and set path in flag.
require.NoError(afero.Afero{Fs: fs}.WriteFile("privK", []byte(tc.privKey), 0o600))
require.NoError(cmd.Flags().Set("privatekey", "privK"))
if tc.initVPN {
require.NoError(cmd.Flags().Set("wg-autoconfig", "true"))
}
require.NoError(cmd.Flags().Set("autoscale", strconv.FormatBool(tc.setAutoscaleFlag)))
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
defer cancel()
cmd.SetContext(ctx)
err := initialize(cmd, tc.client, &tc.serviceAccountCreator, fileHandler, tc.waiter, tc.vpnHandler)
err := initialize(cmd, dialer, &tc.serviceAccountCreator, fileHandler)
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Contains(out.String(), "192.0.2.2")
assert.Contains(out.String(), "ownerID")
assert.Contains(out.String(), "clusterID")
if tc.setAutoscaleFlag {
assert.Len(tc.initServerAPI.activateAutoscalingNodeGroups, 1)
} else {
require.NoError(err)
assert.Equal(tc.initVPN, tc.vpnHandler.(*stubVPNHandler).configured)
assert.Contains(out.String(), "192.0.2.2")
assert.Contains(out.String(), "ownerID")
assert.Contains(out.String(), "clusterID")
assert.Len(tc.initServerAPI.activateAutoscalingNodeGroups, 0)
}
})
}
@ -326,11 +189,10 @@ func TestInitialize(t *testing.T) {
func TestWriteOutput(t *testing.T) {
assert := assert.New(t)
result := activationResult{
clientVpnIP: "foo-qq",
coordinatorPubKey: "bar-qq",
coordinatorPubIP: "baz-qq",
kubeconfig: "foo-bar-baz-qq",
resp := &initproto.InitResponse{
OwnerId: []byte("ownerID"),
ClusterId: []byte("clusterID"),
Kubeconfig: []byte("kubeconfig"),
}
expectedIdFile := clusterIDsFile{
@ -343,13 +205,12 @@ func TestWriteOutput(t *testing.T) {
testFs := afero.NewMemMapFs()
fileHandler := file.NewHandler(testFs)
err := result.writeOutput(&out, fileHandler)
err := writeOutput(resp, &out, fileHandler)
assert.NoError(err)
assert.Contains(out.String(), result.clientVpnIP)
assert.Contains(out.String(), result.coordinatorPubIP)
assert.Contains(out.String(), result.coordinatorPubKey)
assert.Contains(out.String(), result.clusterID)
assert.Contains(out.String(), result.ownerID)
assert.Contains(out.String(), resp.OwnerId)
assert.Contains(out.String(), resp.ClusterId)
assert.Contains(out.String(), constants.AdminConfFilename)
assert.Equal(resp.Kubeconfig, string(adminConf))
afs := afero.Afero{Fs: testFs}
adminConf, err := afs.ReadFile(constants.AdminConfFilename)
@ -364,15 +225,6 @@ func TestWriteOutput(t *testing.T) {
assert.Equal(expectedIdFile, testIdFile)
}
func TestIpsToEndpoints(t *testing.T) {
assert := assert.New(t)
ips := []string{"192.0.2.1", "192.0.2.2", "", "192.0.2.3"}
port := "8080"
endpoints := ipsToEndpoints(ips, port)
assert.Equal([]string{"192.0.2.1:8080", "192.0.2.2:8080", "192.0.2.3:8080"}, endpoints)
}
func TestInitCompletion(t *testing.T) {
testCases := map[string]struct {
args []string
@ -412,27 +264,7 @@ func TestInitCompletion(t *testing.T) {
}
}
func TestReadOrGenerateVPNKey(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
testKey := []byte(base64.StdEncoding.EncodeToString([]byte("32bytesWireGuardKeyForTheTesting")))
fileHandler := file.NewHandler(afero.NewMemMapFs())
require.NoError(fileHandler.Write("testKey", testKey, file.OptNone))
privK, pubK, err := readOrGenerateVPNKey(fileHandler, "testKey")
assert.NoError(err)
assert.Equal(testKey, privK)
assert.NotEmpty(pubK)
// no path provided
privK, pubK, err = readOrGenerateVPNKey(fileHandler, "")
assert.NoError(err)
assert.NotEmpty(privK)
assert.NotEmpty(pubK)
}
func TestReadOrGenerateMasterSecret(t *testing.T) {
func TestReadOrGeneratedMasterSecret(t *testing.T) {
testCases := map[string]struct {
filename string
filecontent string
@ -523,157 +355,16 @@ func TestReadOrGenerateMasterSecret(t *testing.T) {
}
}
func TestAutoscaleFlag(t *testing.T) {
testKey := base64.StdEncoding.EncodeToString([]byte("32bytesWireGuardKeyForTheTesting"))
testGcpState := state.ConstellationState{
CloudProvider: "gcp",
GCPNodes: cloudtypes.Instances{
"id-0": {PrivateIP: "192.0.2.1", PublicIP: "192.0.2.1"},
"id-1": {PrivateIP: "192.0.2.1", PublicIP: "192.0.2.1"},
},
GCPCoordinators: cloudtypes.Instances{
"id-c": {PrivateIP: "192.0.2.1", PublicIP: "192.0.2.1"},
},
}
testAzureState := state.ConstellationState{
CloudProvider: "azure",
AzureNodes: cloudtypes.Instances{
"id-0": {PrivateIP: "192.0.2.1", PublicIP: "192.0.2.1"},
"id-1": {PrivateIP: "192.0.2.1", PublicIP: "192.0.2.1"},
},
AzureCoordinators: cloudtypes.Instances{
"id-c": {PrivateIP: "192.0.2.1", PublicIP: "192.0.2.1"},
},
AzureResourceGroup: "test",
}
testActivationResps := []fakeActivationRespMessage{
{log: "testlog1"},
{log: "testlog2"},
{
kubeconfig: "kubeconfig",
clientVpnIp: "192.0.2.2",
coordinatorVpnKey: testKey,
ownerID: "ownerID",
clusterID: "clusterID",
},
{log: "testlog3"},
}
type stubInitServer struct {
initResp *initproto.InitResponse
initErr error
testCases := map[string]struct {
autoscaleFlag bool
existingState state.ConstellationState
client *stubProtoClient
serviceAccountCreator stubServiceAccountCreator
waiter statusWaiter
privKey string
}{
"initialize some gcp instances without autoscale flag": {
autoscaleFlag: false,
existingState: testGcpState,
client: &stubProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
privKey: testKey,
},
"initialize some azure instances without autoscale flag": {
autoscaleFlag: false,
existingState: testAzureState,
client: &stubProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
privKey: testKey,
},
"initialize some gcp instances with autoscale flag": {
autoscaleFlag: true,
existingState: testGcpState,
client: &stubProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
privKey: testKey,
},
"initialize some azure instances with autoscale flag": {
autoscaleFlag: true,
existingState: testAzureState,
client: &stubProtoClient{
respClient: &fakeActivationRespClient{responses: testActivationResps},
},
waiter: &stubStatusWaiter{},
privKey: testKey,
},
}
activateAutoscalingNodeGroups []string
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
cmd := NewInitCmd()
var out bytes.Buffer
cmd.SetOut(&out)
var errOut bytes.Buffer
cmd.SetErr(&errOut)
cmd.Flags().String("config", "", "") // register persisten flag manually
fs := afero.NewMemMapFs()
fileHandler := file.NewHandler(fs)
vpnHandler := stubVPNHandler{}
require.NoError(fileHandler.WriteJSON(constants.StateFilename, tc.existingState, file.OptNone))
// Write key file to filesystem and set path in flag.
require.NoError(afero.Afero{Fs: fs}.WriteFile("privK", []byte(tc.privKey), 0o600))
require.NoError(cmd.Flags().Set("privatekey", "privK"))
require.NoError(cmd.Flags().Set("autoscale", strconv.FormatBool(tc.autoscaleFlag)))
require.NoError(initialize(cmd, tc.client, &tc.serviceAccountCreator, fileHandler, tc.waiter, &vpnHandler))
if tc.autoscaleFlag {
assert.Len(tc.client.activateAutoscalingNodeGroups, 1)
} else {
assert.Len(tc.client.activateAutoscalingNodeGroups, 0)
}
})
}
initproto.UnimplementedAPIServer
}
func TestWriteWGQuickFile(t *testing.T) {
testCases := map[string]struct {
fileHandler file.Handler
vpnHandler *stubVPNHandler
vpnConfig *wgquick.Config
wantErr bool
}{
"write wg quick file": {
fileHandler: file.NewHandler(afero.NewMemMapFs()),
vpnHandler: &stubVPNHandler{marshalRes: "config"},
},
"marshal failed": {
fileHandler: file.NewHandler(afero.NewMemMapFs()),
vpnHandler: &stubVPNHandler{marshalErr: errors.New("some err")},
wantErr: true,
},
"write fails": {
fileHandler: file.NewHandler(afero.NewReadOnlyFs(afero.NewMemMapFs())),
vpnHandler: &stubVPNHandler{marshalRes: "config"},
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
err := writeWGQuickFile(tc.fileHandler, tc.vpnHandler, tc.vpnConfig)
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
file, err := tc.fileHandler.Read(constants.WGQuickConfigFilename)
assert.NoError(err)
assert.Contains(string(file), tc.vpnHandler.marshalRes)
}
})
}
func (s *stubInitServer) Init(ctx context.Context, req *initproto.InitRequest) (*initproto.InitResponse, error) {
s.activateAutoscalingNodeGroups = req.AutoscalingNodeGroups
return s.initResp, s.initErr
}

View file

@ -1,17 +0,0 @@
package cmd
import (
"context"
"github.com/edgelesssys/constellation/cli/internal/proto"
"github.com/edgelesssys/constellation/coordinator/pubapi/pubproto"
"github.com/edgelesssys/constellation/coordinator/state"
"github.com/edgelesssys/constellation/internal/atls"
)
type protoClient interface {
Connect(endpoint string, validators []atls.Validator) error
Close() error
GetState(ctx context.Context) (state.State, error)
Activate(ctx context.Context, userPublicKey, masterSecret []byte, nodeIPs, coordinatorIPs, autoscalingNodeGroups []string, cloudServiceAccountURI string, sshUsers []*pubproto.SSHUserKey) (proto.ActivationResponseClient, error)
}

View file

@ -1,225 +0,0 @@
package cmd
import (
"context"
"errors"
"fmt"
"io"
"github.com/edgelesssys/constellation/cli/internal/proto"
"github.com/edgelesssys/constellation/coordinator/pubapi/pubproto"
"github.com/edgelesssys/constellation/coordinator/state"
"github.com/edgelesssys/constellation/internal/atls"
)
type stubProtoClient struct {
conn bool
respClient proto.ActivationResponseClient
connectErr error
closeErr error
getStateErr error
activateErr error
getStateState state.State
activateUserPublicKey []byte
activateMasterSecret []byte
activateNodeIPs []string
activateCoordinatorIPs []string
activateAutoscalingNodeGroups []string
cloudServiceAccountURI string
sshUserKeys []*pubproto.SSHUserKey
}
func (c *stubProtoClient) Connect(_ string, _ []atls.Validator) error {
c.conn = true
return c.connectErr
}
func (c *stubProtoClient) Close() error {
c.conn = false
return c.closeErr
}
func (c *stubProtoClient) GetState(_ context.Context) (state.State, error) {
return c.getStateState, c.getStateErr
}
func (c *stubProtoClient) Activate(ctx context.Context, userPublicKey, masterSecret []byte, nodeIPs, coordinatorIPs []string, autoscalingNodeGroups []string, cloudServiceAccountURI string, sshUserKeys []*pubproto.SSHUserKey) (proto.ActivationResponseClient, error) {
c.activateUserPublicKey = userPublicKey
c.activateMasterSecret = masterSecret
c.activateNodeIPs = nodeIPs
c.activateCoordinatorIPs = coordinatorIPs
c.activateAutoscalingNodeGroups = autoscalingNodeGroups
c.cloudServiceAccountURI = cloudServiceAccountURI
c.sshUserKeys = sshUserKeys
return c.respClient, c.activateErr
}
func (c *stubProtoClient) ActivateAdditionalCoordinators(ctx context.Context, ips []string) error {
return c.activateErr
}
type stubActivationRespClient struct {
nextLogErr *error
getKubeconfigErr error
getCoordinatorVpnKeyErr error
getClientVpnIpErr error
getOwnerIDErr error
getClusterIDErr error
writeLogStreamErr error
}
func (s *stubActivationRespClient) NextLog() (string, error) {
if s.nextLogErr == nil {
return "", io.EOF
}
return "", *s.nextLogErr
}
func (s *stubActivationRespClient) WriteLogStream(io.Writer) error {
return s.writeLogStreamErr
}
func (s *stubActivationRespClient) GetKubeconfig() (string, error) {
return "", s.getKubeconfigErr
}
func (s *stubActivationRespClient) GetCoordinatorVpnKey() (string, error) {
return "", s.getCoordinatorVpnKeyErr
}
func (s *stubActivationRespClient) GetClientVpnIp() (string, error) {
return "", s.getClientVpnIpErr
}
func (s *stubActivationRespClient) GetOwnerID() (string, error) {
return "", s.getOwnerIDErr
}
func (s *stubActivationRespClient) GetClusterID() (string, error) {
return "", s.getClusterIDErr
}
type fakeProtoClient struct {
conn bool
respClient proto.ActivationResponseClient
}
func (c *fakeProtoClient) Connect(endpoint string, validators []atls.Validator) error {
if endpoint == "" {
return errors.New("endpoint is empty")
}
if len(validators) == 0 {
return errors.New("validators is empty")
}
c.conn = true
return nil
}
func (c *fakeProtoClient) Close() error {
c.conn = false
return nil
}
func (c *fakeProtoClient) GetState(_ context.Context) (state.State, error) {
if !c.conn {
return state.Uninitialized, errors.New("client is not connected")
}
return state.IsNode, nil
}
func (c *fakeProtoClient) Activate(ctx context.Context, userPublicKey, masterSecret []byte, nodeIPs, coordinatorIPs, autoscalingNodeGroups []string, cloudServiceAccountURI string, sshUserKeys []*pubproto.SSHUserKey) (proto.ActivationResponseClient, error) {
if !c.conn {
return nil, errors.New("client is not connected")
}
return c.respClient, nil
}
func (c *fakeProtoClient) ActivateAdditionalCoordinators(ctx context.Context, ips []string) error {
if !c.conn {
return errors.New("client is not connected")
}
return nil
}
type fakeActivationRespClient struct {
responses []fakeActivationRespMessage
kubeconfig string
coordinatorVpnKey string
clientVpnIp string
ownerID string
clusterID string
}
func (c *fakeActivationRespClient) NextLog() (string, error) {
for len(c.responses) > 0 {
resp := c.responses[0]
c.responses = c.responses[1:]
if len(resp.log) > 0 {
return resp.log, nil
}
c.kubeconfig = resp.kubeconfig
c.coordinatorVpnKey = resp.coordinatorVpnKey
c.clientVpnIp = resp.clientVpnIp
c.ownerID = resp.ownerID
c.clusterID = resp.clusterID
}
return "", io.EOF
}
func (c *fakeActivationRespClient) WriteLogStream(w io.Writer) error {
log, err := c.NextLog()
for err == nil {
fmt.Fprint(w, log)
log, err = c.NextLog()
}
if !errors.Is(err, io.EOF) {
return err
}
return nil
}
func (c *fakeActivationRespClient) GetKubeconfig() (string, error) {
if c.kubeconfig == "" {
return "", errors.New("kubeconfig is empty")
}
return c.kubeconfig, nil
}
func (c *fakeActivationRespClient) GetCoordinatorVpnKey() (string, error) {
if c.coordinatorVpnKey == "" {
return "", errors.New("control-plane public VPN key is empty")
}
return c.coordinatorVpnKey, nil
}
func (c *fakeActivationRespClient) GetClientVpnIp() (string, error) {
if c.clientVpnIp == "" {
return "", errors.New("client VPN IP is empty")
}
return c.clientVpnIp, nil
}
func (c *fakeActivationRespClient) GetOwnerID() (string, error) {
if c.ownerID == "" {
return "", errors.New("init secret is empty")
}
return c.ownerID, nil
}
func (c *fakeActivationRespClient) GetClusterID() (string, error) {
if c.clusterID == "" {
return "", errors.New("cluster identifier is empty")
}
return c.clusterID, nil
}
type fakeActivationRespMessage struct {
log string
kubeconfig string
coordinatorVpnKey string
clientVpnIp string
ownerID string
clusterID string
}

View file

@ -1,13 +0,0 @@
package cmd
import (
"context"
"github.com/edgelesssys/constellation/coordinator/state"
"github.com/edgelesssys/constellation/internal/atls"
)
type statusWaiter interface {
InitializeValidators([]atls.Validator) error
WaitForAll(ctx context.Context, endpoints []string, status ...state.State) error
}

View file

@ -1,27 +0,0 @@
package cmd
import (
"context"
"errors"
"github.com/edgelesssys/constellation/coordinator/state"
"github.com/edgelesssys/constellation/internal/atls"
)
type stubStatusWaiter struct {
initialized bool
initializeErr error
waitForAllErr error
}
func (s *stubStatusWaiter) InitializeValidators([]atls.Validator) error {
s.initialized = true
return s.initializeErr
}
func (s *stubStatusWaiter) WaitForAll(ctx context.Context, endpoints []string, status ...state.State) error {
if !s.initialized {
return errors.New("waiter not initialized")
}
return s.waitForAllErr
}