/* Copyright (c) Edgeless Systems GmbH SPDX-License-Identifier: AGPL-3.0-only */ package client import ( "context" "errors" "fmt" "strings" "time" "github.com/edgelesssys/constellation/internal/cloud/cloudtypes" "github.com/edgelesssys/constellation/internal/constants" "github.com/edgelesssys/constellation/internal/role" "google.golang.org/api/iterator" computepb "google.golang.org/genproto/googleapis/cloud/compute/v1" "google.golang.org/protobuf/proto" ) // CreateInstances creates instances (virtual machines) on Google Compute Engine. // // A separate managed instance group is created for control planes and workers, the function // waits until the instances are up and stores the public and private IPs of the instances // in the client. If the client's network must be set before instances can be created. func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput) error { if c.network == "" { return errors.New("client has no network") } ops := []Operation{} workerTemplateInput := insertInstanceTemplateInput{ Name: c.buildResourceName("worker"), Network: c.network, SecondarySubnetworkRangeName: c.secondarySubnetworkRange, Subnetwork: c.subnetwork, ImageID: input.ImageID, InstanceType: input.InstanceType, StateDiskSizeGB: int64(input.StateDiskSizeGB), StateDiskType: input.StateDiskType, Role: role.Worker.String(), KubeEnv: input.KubeEnv, Project: c.project, Zone: c.zone, Region: c.region, UID: c.uid, } op, err := c.insertInstanceTemplate(ctx, workerTemplateInput) if err != nil { return fmt.Errorf("inserting instanceTemplate: %w", err) } ops = append(ops, op) c.workerTemplate = workerTemplateInput.Name controlPlaneTemplateInput := insertInstanceTemplateInput{ Name: c.buildResourceName("control-plane"), Network: c.network, Subnetwork: c.subnetwork, SecondarySubnetworkRangeName: c.secondarySubnetworkRange, ImageID: input.ImageID, InstanceType: input.InstanceType, StateDiskSizeGB: int64(input.StateDiskSizeGB), StateDiskType: input.StateDiskType, Role: role.ControlPlane.String(), KubeEnv: input.KubeEnv, Project: c.project, Zone: c.zone, Region: c.region, UID: c.uid, } op, err = c.insertInstanceTemplate(ctx, controlPlaneTemplateInput) if err != nil { return fmt.Errorf("inserting instanceTemplate: %w", err) } ops = append(ops, op) c.controlPlaneTemplate = controlPlaneTemplateInput.Name if err := c.waitForOperations(ctx, ops); err != nil { return err } ops = []Operation{} controlPlaneGroupInput := instanceGroupManagerInput{ Count: input.CountControlPlanes, Name: strings.Join([]string{c.name, "control-plane", c.uid}, "-"), NamedPorts: []*computepb.NamedPort{ {Name: proto.String("kubernetes"), Port: proto.Int32(constants.KubernetesPort)}, {Name: proto.String("debugd"), Port: proto.Int32(constants.DebugdPort)}, {Name: proto.String("bootstrapper"), Port: proto.Int32(constants.BootstrapperPort)}, {Name: proto.String("verify"), Port: proto.Int32(constants.VerifyServiceNodePortGRPC)}, {Name: proto.String("konnectivity"), Port: proto.Int32(constants.KonnectivityPort)}, {Name: proto.String("recovery"), Port: proto.Int32(constants.RecoveryPort)}, }, Template: c.controlPlaneTemplate, UID: c.uid, Project: c.project, Zone: c.zone, } op, err = c.insertInstanceGroupManger(ctx, controlPlaneGroupInput) if err != nil { return fmt.Errorf("inserting instanceGroupManager: %w", err) } ops = append(ops, op) c.controlPlaneInstanceGroup = controlPlaneGroupInput.Name workerGroupInput := instanceGroupManagerInput{ Count: input.CountWorkers, Name: strings.Join([]string{c.name, "worker", c.uid}, "-"), Template: c.workerTemplate, UID: c.uid, Project: c.project, Zone: c.zone, } op, err = c.insertInstanceGroupManger(ctx, workerGroupInput) if err != nil { return fmt.Errorf("inserting instanceGroupManager: %w", err) } ops = append(ops, op) c.workerInstanceGroup = workerGroupInput.Name if err := c.waitForOperations(ctx, ops); err != nil { return err } if err := c.waitForInstanceGroupScaling(ctx, c.workerInstanceGroup); err != nil { return fmt.Errorf("waiting for instanceGroupScaling: %w", err) } if err := c.waitForInstanceGroupScaling(ctx, c.controlPlaneInstanceGroup); err != nil { return fmt.Errorf("waiting for instanceGroupScaling: %w", err) } if err := c.getInstanceIPs(ctx, c.workerInstanceGroup, c.workers); err != nil { return fmt.Errorf("getting instanceIPs: %w", err) } if err := c.getInstanceIPs(ctx, c.controlPlaneInstanceGroup, c.controlPlanes); err != nil { return fmt.Errorf("getting instanceIPs: %w", err) } return nil } // TerminateInstances terminates the clients instances. func (c *Client) TerminateInstances(ctx context.Context) error { ops := []Operation{} if c.workerInstanceGroup != "" { op, err := c.deleteInstanceGroupManager(ctx, c.workerInstanceGroup) if err != nil && !isNotFoundError(err) { return fmt.Errorf("deleting instanceGroupManager '%s': %w", c.workerInstanceGroup, err) } if err == nil { ops = append(ops, op) } c.workerInstanceGroup = "" c.workers = make(cloudtypes.Instances) } if c.controlPlaneInstanceGroup != "" { op, err := c.deleteInstanceGroupManager(ctx, c.controlPlaneInstanceGroup) if err != nil && !isNotFoundError(err) { return fmt.Errorf("deleting instanceGroupManager '%s': %w", c.controlPlaneInstanceGroup, err) } if err == nil { ops = append(ops, op) } c.controlPlaneInstanceGroup = "" c.controlPlanes = make(cloudtypes.Instances) } if err := c.waitForOperations(ctx, ops); err != nil { return err } ops = []Operation{} if c.workerTemplate != "" { op, err := c.deleteInstanceTemplate(ctx, c.workerTemplate) if err != nil && !isNotFoundError(err) { return fmt.Errorf("deleting instanceTemplate: %w", err) } if err == nil { ops = append(ops, op) } c.workerTemplate = "" } if c.controlPlaneTemplate != "" { op, err := c.deleteInstanceTemplate(ctx, c.controlPlaneTemplate) if err != nil && !isNotFoundError(err) { return fmt.Errorf("deleting instanceTemplate: %w", err) } if err == nil { ops = append(ops, op) } c.controlPlaneTemplate = "" } return c.waitForOperations(ctx, ops) } func (c *Client) insertInstanceTemplate(ctx context.Context, input insertInstanceTemplateInput) (Operation, error) { req := input.insertInstanceTemplateRequest() return c.instanceTemplateAPI.Insert(ctx, req) } func (c *Client) deleteInstanceTemplate(ctx context.Context, name string) (Operation, error) { req := &computepb.DeleteInstanceTemplateRequest{ InstanceTemplate: name, Project: c.project, } return c.instanceTemplateAPI.Delete(ctx, req) } func (c *Client) insertInstanceGroupManger(ctx context.Context, input instanceGroupManagerInput) (Operation, error) { req := input.InsertInstanceGroupManagerRequest() return c.instanceGroupManagersAPI.Insert(ctx, &req) } func (c *Client) deleteInstanceGroupManager(ctx context.Context, instanceGroupManagerName string) (Operation, error) { req := &computepb.DeleteInstanceGroupManagerRequest{ InstanceGroupManager: instanceGroupManagerName, Project: c.project, Zone: c.zone, } return c.instanceGroupManagersAPI.Delete(ctx, req) } func (c *Client) waitForInstanceGroupScaling(ctx context.Context, groupID string) error { for { if err := ctx.Err(); err != nil { return err } listReq := &computepb.ListManagedInstancesInstanceGroupManagersRequest{ InstanceGroupManager: groupID, Project: c.project, Zone: c.zone, } it := c.instanceGroupManagersAPI.ListManagedInstances(ctx, listReq) for { resp, err := it.Next() if errors.Is(err, iterator.Done) { return nil } if err != nil { return err } if resp.CurrentAction == nil { return errors.New("currentAction is nil") } if *resp.CurrentAction != computepb.ManagedInstance_NONE.String() { time.Sleep(5 * time.Second) break } } } } // getInstanceIPs requests the IPs of the client's instances. func (c *Client) getInstanceIPs(ctx context.Context, groupID string, list cloudtypes.Instances) error { req := &computepb.ListInstancesRequest{ Filter: proto.String("name=" + groupID + "*"), Project: c.project, Zone: c.zone, } it := c.instanceAPI.List(ctx, req) for { resp, err := it.Next() if errors.Is(err, iterator.Done) { return nil } if err != nil { return err } if resp.Name == nil { return errors.New("instance name is nil pointer") } if len(resp.NetworkInterfaces) == 0 { return errors.New("network interface is empty") } if resp.NetworkInterfaces[0].NetworkIP == nil { return errors.New("networkIP is nil") } if len(resp.NetworkInterfaces[0].AccessConfigs) == 0 { return errors.New("access configs is empty") } if resp.NetworkInterfaces[0].AccessConfigs[0].NatIP == nil { return errors.New("natIP is nil") } instance := cloudtypes.Instance{ PrivateIP: *resp.NetworkInterfaces[0].NetworkIP, PublicIP: *resp.NetworkInterfaces[0].AccessConfigs[0].NatIP, } list[*resp.Name] = instance } } type instanceGroupManagerInput struct { Count int Name string NamedPorts []*computepb.NamedPort Template string Project string Zone string UID string } func (i *instanceGroupManagerInput) InsertInstanceGroupManagerRequest() computepb.InsertInstanceGroupManagerRequest { return computepb.InsertInstanceGroupManagerRequest{ InstanceGroupManagerResource: &computepb.InstanceGroupManager{ BaseInstanceName: proto.String(i.Name), NamedPorts: i.NamedPorts, InstanceTemplate: proto.String("projects/" + i.Project + "/global/instanceTemplates/" + i.Template), Name: proto.String(i.Name), TargetSize: proto.Int32(int32(i.Count)), }, Project: i.Project, Zone: i.Zone, } } // CreateInstancesInput is the input for a CreatInstances operation. type CreateInstancesInput struct { CountWorkers int CountControlPlanes int ImageID string InstanceType string StateDiskSizeGB int StateDiskType string KubeEnv string } type insertInstanceTemplateInput struct { Name string Network string Subnetwork string SecondarySubnetworkRangeName string ImageID string InstanceType string StateDiskSizeGB int64 StateDiskType string Role string KubeEnv string Project string Zone string Region string UID string } func (i insertInstanceTemplateInput) insertInstanceTemplateRequest() *computepb.InsertInstanceTemplateRequest { req := computepb.InsertInstanceTemplateRequest{ InstanceTemplateResource: &computepb.InstanceTemplate{ Description: proto.String("This instance belongs to a Constellation cluster."), Name: proto.String(i.Name), Properties: &computepb.InstanceProperties{ ConfidentialInstanceConfig: &computepb.ConfidentialInstanceConfig{ EnableConfidentialCompute: proto.Bool(true), }, Description: proto.String("This instance belongs to a Constellation cluster."), Disks: []*computepb.AttachedDisk{ { InitializeParams: &computepb.AttachedDiskInitializeParams{ DiskSizeGb: proto.Int64(10), SourceImage: proto.String(i.ImageID), }, AutoDelete: proto.Bool(true), Boot: proto.Bool(true), Mode: proto.String(computepb.AttachedDisk_READ_WRITE.String()), }, { InitializeParams: &computepb.AttachedDiskInitializeParams{ DiskSizeGb: proto.Int64(i.StateDiskSizeGB), DiskType: proto.String(i.StateDiskType), }, AutoDelete: proto.Bool(true), DeviceName: proto.String("state-disk"), Mode: proto.String(computepb.AttachedDisk_READ_WRITE.String()), Type: proto.String(computepb.AttachedDisk_PERSISTENT.String()), }, }, MachineType: proto.String(i.InstanceType), Metadata: &computepb.Metadata{ Items: []*computepb.Items{ { Key: proto.String("kube-env"), Value: proto.String(i.KubeEnv), }, { Key: proto.String("constellation-uid"), Value: proto.String(i.UID), }, { Key: proto.String("constellation-role"), Value: proto.String(i.Role), }, }, }, NetworkInterfaces: []*computepb.NetworkInterface{ { Network: proto.String("projects/" + i.Project + "/global/networks/" + i.Network), Subnetwork: proto.String("regions/" + i.Region + "/subnetworks/" + i.Subnetwork), AccessConfigs: []*computepb.AccessConfig{ {Type: proto.String(computepb.AccessConfig_ONE_TO_ONE_NAT.String())}, }, }, }, Scheduling: &computepb.Scheduling{ OnHostMaintenance: proto.String(computepb.Scheduling_TERMINATE.String()), }, ServiceAccounts: []*computepb.ServiceAccount{ { Scopes: []string{ "https://www.googleapis.com/auth/compute", "https://www.googleapis.com/auth/servicecontrol", "https://www.googleapis.com/auth/service.management.readonly", "https://www.googleapis.com/auth/devstorage.read_only", "https://www.googleapis.com/auth/logging.write", "https://www.googleapis.com/auth/monitoring.write", "https://www.googleapis.com/auth/trace.append", }, }, }, ShieldedInstanceConfig: &computepb.ShieldedInstanceConfig{ EnableIntegrityMonitoring: proto.Bool(true), EnableSecureBoot: proto.Bool(true), EnableVtpm: proto.Bool(true), }, Tags: &computepb.Tags{ Items: []string{"constellation-" + i.UID}, }, }, }, Project: i.Project, } // if there is an secondary IP range defined, we use it as an alias IP range if i.SecondarySubnetworkRangeName != "" { req.InstanceTemplateResource.Properties.NetworkInterfaces[0].AliasIpRanges = []*computepb.AliasIpRange{ { IpCidrRange: proto.String("/24"), SubnetworkRangeName: proto.String(i.SecondarySubnetworkRangeName), }, } } return &req }