constellation/cli/gcp/client/instances.go
2022-06-02 13:08:25 +02:00

414 lines
13 KiB
Go

package client
import (
"context"
"errors"
"fmt"
"strings"
"time"
"github.com/edgelesssys/constellation/cli/gcp"
"github.com/edgelesssys/constellation/coordinator/role"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
"google.golang.org/protobuf/proto"
)
// CreateInstances creates instances (virtual machines) on Google Compute Engine.
//
// A separate managed instance group is created for coordinators and nodes, the function
// waits until the instances are up and stores the public and private IPs of the instances
// in the client. If the client's network must be set before instances can be created.
func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput) error {
if c.network == "" {
return errors.New("client has no network")
}
ops := []Operation{}
nodeTemplateInput := insertInstanceTemplateInput{
Name: c.name + "-worker-" + c.uid,
Network: c.network,
SecondarySubnetworkRangeName: c.secondarySubnetworkRange,
Subnetwork: c.subnetwork,
ImageId: input.ImageId,
InstanceType: input.InstanceType,
StateDiskSizeGB: int64(input.StateDiskSizeGB),
Role: role.Node.String(),
KubeEnv: input.KubeEnv,
Project: c.project,
Zone: c.zone,
Region: c.region,
UID: c.uid,
}
op, err := c.insertInstanceTemplate(ctx, nodeTemplateInput)
if err != nil {
return fmt.Errorf("inserting instanceTemplate failed: %w", err)
}
ops = append(ops, op)
c.nodeTemplate = nodeTemplateInput.Name
coordinatorTemplateInput := insertInstanceTemplateInput{
Name: c.name + "-control-plane-" + c.uid,
Network: c.network,
Subnetwork: c.subnetwork,
SecondarySubnetworkRangeName: c.secondarySubnetworkRange,
ImageId: input.ImageId,
InstanceType: input.InstanceType,
StateDiskSizeGB: int64(input.StateDiskSizeGB),
Role: role.Coordinator.String(),
KubeEnv: input.KubeEnv,
Project: c.project,
Zone: c.zone,
Region: c.region,
UID: c.uid,
}
op, err = c.insertInstanceTemplate(ctx, coordinatorTemplateInput)
if err != nil {
return fmt.Errorf("inserting instanceTemplate failed: %w", err)
}
ops = append(ops, op)
c.coordinatorTemplate = coordinatorTemplateInput.Name
if err := c.waitForOperations(ctx, ops); err != nil {
return err
}
ops = []Operation{}
coordinatorGroupInput := instanceGroupManagerInput{
Count: input.CountCoordinators,
Name: strings.Join([]string{c.name, "control-plane", c.uid}, "-"),
Template: c.coordinatorTemplate,
UID: c.uid,
Project: c.project,
Zone: c.zone,
}
op, err = c.insertInstanceGroupManger(ctx, coordinatorGroupInput)
if err != nil {
return fmt.Errorf("inserting instanceGroupManager failed: %w", err)
}
ops = append(ops, op)
c.coordinatorInstanceGroup = coordinatorGroupInput.Name
nodeGroupInput := instanceGroupManagerInput{
Count: input.CountNodes,
Name: strings.Join([]string{c.name, "worker", c.uid}, "-"),
Template: c.nodeTemplate,
UID: c.uid,
Project: c.project,
Zone: c.zone,
}
op, err = c.insertInstanceGroupManger(ctx, nodeGroupInput)
if err != nil {
return fmt.Errorf("inserting instanceGroupManager failed: %w", err)
}
ops = append(ops, op)
c.nodesInstanceGroup = nodeGroupInput.Name
if err := c.waitForOperations(ctx, ops); err != nil {
return err
}
if err := c.waitForInstanceGroupScaling(ctx, c.nodesInstanceGroup); err != nil {
return fmt.Errorf("waiting for instanceGroupScaling failed: %w", err)
}
if err := c.waitForInstanceGroupScaling(ctx, c.coordinatorInstanceGroup); err != nil {
return fmt.Errorf("waiting for instanceGroupScaling failed: %w", err)
}
if err := c.getInstanceIPs(ctx, c.nodesInstanceGroup, c.nodes); err != nil {
return fmt.Errorf("failed to get instanceIPs: %w", err)
}
if err := c.getInstanceIPs(ctx, c.coordinatorInstanceGroup, c.coordinators); err != nil {
return fmt.Errorf("failed to get instanceIPs: %w", err)
}
return nil
}
// TerminateInstances terminates the clients instances.
func (c *Client) TerminateInstances(ctx context.Context) error {
ops := []Operation{}
if c.nodesInstanceGroup != "" {
op, err := c.deleteInstanceGroupManager(ctx, c.nodesInstanceGroup)
if err != nil {
return fmt.Errorf("deleting instanceGroupManager '%s' failed: %w", c.nodesInstanceGroup, err)
}
ops = append(ops, op)
c.nodesInstanceGroup = ""
c.nodes = make(gcp.Instances)
}
if c.coordinatorInstanceGroup != "" {
op, err := c.deleteInstanceGroupManager(ctx, c.coordinatorInstanceGroup)
if err != nil {
return fmt.Errorf("deleting instanceGroupManager '%s' failed: %w", c.coordinatorInstanceGroup, err)
}
ops = append(ops, op)
c.coordinatorInstanceGroup = ""
c.coordinators = make(gcp.Instances)
}
if err := c.waitForOperations(ctx, ops); err != nil {
return err
}
ops = []Operation{}
if c.nodeTemplate != "" {
op, err := c.deleteInstanceTemplate(ctx, c.nodeTemplate)
if err != nil {
return fmt.Errorf("deleting instanceTemplate failed: %w", err)
}
ops = append(ops, op)
c.nodeTemplate = ""
}
if c.coordinatorTemplate != "" {
op, err := c.deleteInstanceTemplate(ctx, c.coordinatorTemplate)
if err != nil {
return fmt.Errorf("deleting instanceTemplate failed: %w", err)
}
ops = append(ops, op)
c.coordinatorTemplate = ""
}
return c.waitForOperations(ctx, ops)
}
func (c *Client) insertInstanceTemplate(ctx context.Context, input insertInstanceTemplateInput) (Operation, error) {
req := input.insertInstanceTemplateRequest()
return c.instanceTemplateAPI.Insert(ctx, req)
}
func (c *Client) deleteInstanceTemplate(ctx context.Context, name string) (Operation, error) {
req := &computepb.DeleteInstanceTemplateRequest{
InstanceTemplate: name,
Project: c.project,
}
return c.instanceTemplateAPI.Delete(ctx, req)
}
func (c *Client) insertInstanceGroupManger(ctx context.Context, input instanceGroupManagerInput) (Operation, error) {
req := input.InsertInstanceGroupManagerRequest()
return c.instanceGroupManagersAPI.Insert(ctx, &req)
}
func (c *Client) deleteInstanceGroupManager(ctx context.Context, instanceGroupManagerName string) (Operation, error) {
req := &computepb.DeleteInstanceGroupManagerRequest{
InstanceGroupManager: instanceGroupManagerName,
Project: c.project,
Zone: c.zone,
}
return c.instanceGroupManagersAPI.Delete(ctx, req)
}
func (c *Client) waitForInstanceGroupScaling(ctx context.Context, groupId string) error {
for {
if err := ctx.Err(); err != nil {
return err
}
listReq := &computepb.ListManagedInstancesInstanceGroupManagersRequest{
InstanceGroupManager: groupId,
Project: c.project,
Zone: c.zone,
}
it := c.instanceGroupManagersAPI.ListManagedInstances(ctx, listReq)
for {
resp, err := it.Next()
if errors.Is(err, iterator.Done) {
return nil
}
if err != nil {
return err
}
if resp.CurrentAction == nil {
return errors.New("currentAction is nil")
}
if *resp.CurrentAction != computepb.ManagedInstance_NONE.String() {
time.Sleep(5 * time.Second)
break
}
}
}
}
// getInstanceIPs requests the IPs of the client's instances.
func (c *Client) getInstanceIPs(ctx context.Context, groupId string, list gcp.Instances) error {
req := &computepb.ListInstancesRequest{
Filter: proto.String("name=" + groupId + "*"),
Project: c.project,
Zone: c.zone,
}
it := c.instanceAPI.List(ctx, req)
for {
resp, err := it.Next()
if errors.Is(err, iterator.Done) {
return nil
}
if err != nil {
return err
}
if resp.Name == nil {
return errors.New("instance name is nil pointer")
}
if len(resp.NetworkInterfaces) == 0 {
return errors.New("network interface is empty")
}
if resp.NetworkInterfaces[0].NetworkIP == nil {
return errors.New("networkIP is nil")
}
if len(resp.NetworkInterfaces[0].AccessConfigs) == 0 {
return errors.New("access configs is empty")
}
if resp.NetworkInterfaces[0].AccessConfigs[0].NatIP == nil {
return errors.New("natIP is nil")
}
instance := gcp.Instance{
PrivateIP: *resp.NetworkInterfaces[0].NetworkIP,
PublicIP: *resp.NetworkInterfaces[0].AccessConfigs[0].NatIP,
}
list[*resp.Name] = instance
}
}
type instanceGroupManagerInput struct {
Count int
Name string
Template string
Project string
Zone string
UID string
}
func (i *instanceGroupManagerInput) InsertInstanceGroupManagerRequest() computepb.InsertInstanceGroupManagerRequest {
return computepb.InsertInstanceGroupManagerRequest{
InstanceGroupManagerResource: &computepb.InstanceGroupManager{
BaseInstanceName: proto.String(i.Name),
InstanceTemplate: proto.String("projects/" + i.Project + "/global/instanceTemplates/" + i.Template),
Name: proto.String(i.Name),
TargetSize: proto.Int32(int32(i.Count)),
},
Project: i.Project,
Zone: i.Zone,
}
}
// CreateInstancesInput is the input for a CreatInstances operation.
type CreateInstancesInput struct {
CountNodes int
CountCoordinators int
ImageId string
InstanceType string
StateDiskSizeGB int
KubeEnv string
}
type insertInstanceTemplateInput struct {
Name string
Network string
Subnetwork string
SecondarySubnetworkRangeName string
ImageId string
InstanceType string
StateDiskSizeGB int64
Role string
KubeEnv string
Project string
Zone string
Region string
UID string
}
func (i insertInstanceTemplateInput) insertInstanceTemplateRequest() *computepb.InsertInstanceTemplateRequest {
req := computepb.InsertInstanceTemplateRequest{
InstanceTemplateResource: &computepb.InstanceTemplate{
Description: proto.String("This instance belongs to a Constellation cluster."),
Name: proto.String(i.Name),
Properties: &computepb.InstanceProperties{
ConfidentialInstanceConfig: &computepb.ConfidentialInstanceConfig{
EnableConfidentialCompute: proto.Bool(true),
},
Description: proto.String("This instance belongs to a Constellation cluster."),
Disks: []*computepb.AttachedDisk{
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
DiskSizeGb: proto.Int64(10),
SourceImage: proto.String(i.ImageId),
},
AutoDelete: proto.Bool(true),
Boot: proto.Bool(true),
Mode: proto.String(computepb.AttachedDisk_READ_WRITE.String()),
},
{
InitializeParams: &computepb.AttachedDiskInitializeParams{
DiskSizeGb: proto.Int64(i.StateDiskSizeGB),
},
AutoDelete: proto.Bool(true),
DeviceName: proto.String("state-disk"),
Mode: proto.String(computepb.AttachedDisk_READ_WRITE.String()),
Type: proto.String(computepb.AttachedDisk_PERSISTENT.String()),
},
},
MachineType: proto.String(i.InstanceType),
Metadata: &computepb.Metadata{
Items: []*computepb.Items{
{
Key: proto.String("kube-env"),
Value: proto.String(i.KubeEnv),
},
{
Key: proto.String("constellation-uid"),
Value: proto.String(i.UID),
},
{
Key: proto.String("constellation-role"),
Value: proto.String(i.Role),
},
},
},
NetworkInterfaces: []*computepb.NetworkInterface{
{
Network: proto.String("projects/" + i.Project + "/global/networks/" + i.Network),
Subnetwork: proto.String("regions/" + i.Region + "/subnetworks/" + i.Subnetwork),
AccessConfigs: []*computepb.AccessConfig{
{Type: proto.String(computepb.AccessConfig_ONE_TO_ONE_NAT.String())},
},
},
},
Scheduling: &computepb.Scheduling{
OnHostMaintenance: proto.String(computepb.Scheduling_TERMINATE.String()),
},
ServiceAccounts: []*computepb.ServiceAccount{
{
Scopes: []string{
"https://www.googleapis.com/auth/compute",
"https://www.googleapis.com/auth/servicecontrol",
"https://www.googleapis.com/auth/service.management.readonly",
"https://www.googleapis.com/auth/devstorage.read_only",
"https://www.googleapis.com/auth/logging.write",
"https://www.googleapis.com/auth/monitoring.write",
"https://www.googleapis.com/auth/trace.append",
},
},
},
ShieldedInstanceConfig: &computepb.ShieldedInstanceConfig{
EnableIntegrityMonitoring: proto.Bool(true),
EnableSecureBoot: proto.Bool(true),
EnableVtpm: proto.Bool(true),
},
Tags: &computepb.Tags{
Items: []string{"constellation-" + i.UID},
},
},
},
Project: i.Project,
}
// if there is an secondary IP range defined, we use it as an alias IP range
if i.SecondarySubnetworkRangeName != "" {
req.InstanceTemplateResource.Properties.NetworkInterfaces[0].AliasIpRanges = []*computepb.AliasIpRange{
{
IpCidrRange: proto.String("/24"),
SubnetworkRangeName: proto.String(i.SecondarySubnetworkRangeName),
},
}
}
return &req
}