/* Copyright (c) Edgeless Systems GmbH SPDX-License-Identifier: AGPL-3.0-only */ package client import ( "context" "errors" "fmt" "net/http" "strconv" "sync" "time" "github.com/Azure/azure-sdk-for-go/sdk/azcore" armcomputev2 "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v2" "github.com/edgelesssys/constellation/v2/cli/internal/azure" "github.com/edgelesssys/constellation/v2/cli/internal/azure/internal/poller" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudtypes" ) const ( // scaleSetCreateTimeout maximum timeout to wait for scale set creation. scaleSetCreateTimeout = 5 * time.Minute powerStateStarting = "PowerState/starting" powerStateRunning = "PowerState/running" ) func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput) error { // Create worker scale set createWorkerInput := CreateScaleSetInput{ Name: "constellation-scale-set-workers-" + c.uid, NamePrefix: c.name + "-worker-" + c.uid + "-", Count: input.CountWorkers, InstanceType: input.InstanceType, StateDiskSizeGB: int32(input.StateDiskSizeGB), StateDiskType: input.StateDiskType, Image: input.Image, UserAssingedIdentity: input.UserAssingedIdentity, LoadBalancerBackendAddressPool: azure.BackendAddressPoolWorkerName + "-" + c.uid, ConfidentialVM: input.ConfidentialVM, } // Create control plane scale set createControlPlaneInput := CreateScaleSetInput{ Name: "constellation-scale-set-controlplanes-" + c.uid, NamePrefix: c.name + "-control-plane-" + c.uid + "-", Count: input.CountControlPlanes, InstanceType: input.InstanceType, StateDiskSizeGB: int32(input.StateDiskSizeGB), StateDiskType: input.StateDiskType, Image: input.Image, UserAssingedIdentity: input.UserAssingedIdentity, LoadBalancerBackendAddressPool: azure.BackendAddressPoolControlPlaneName + "-" + c.uid, ConfidentialVM: input.ConfidentialVM, } var wg sync.WaitGroup var controlPlaneErr, workerErr error wg.Add(1) go func() { defer wg.Done() workerErr = c.createScaleSet(ctx, createWorkerInput) }() wg.Add(1) go func() { defer wg.Done() controlPlaneErr = c.createScaleSet(ctx, createControlPlaneInput) }() wg.Wait() if controlPlaneErr != nil { return fmt.Errorf("creating control-plane scaleset: %w", controlPlaneErr) } if workerErr != nil { return fmt.Errorf("creating worker scaleset: %w", workerErr) } // TODO: Remove getInstanceIPs calls after init has been refactored to not use node IPs // Get worker IPs c.workerScaleSet = createWorkerInput.Name instances, err := c.getInstanceIPs(ctx, createWorkerInput.Name, createWorkerInput.Count) if err != nil { return err } c.workers = instances // Get control plane IPs c.controlPlaneScaleSet = createControlPlaneInput.Name instances, err = c.getInstanceIPs(ctx, createControlPlaneInput.Name, createControlPlaneInput.Count) if err != nil { return err } c.controlPlanes = instances return nil } // CreateInstancesInput is the input for a CreateInstances operation. type CreateInstancesInput struct { CountWorkers int CountControlPlanes int InstanceType string StateDiskSizeGB int StateDiskType string Image string UserAssingedIdentity string ConfidentialVM bool } func (c *Client) createScaleSet(ctx context.Context, input CreateScaleSetInput) error { // TODO: Generating a random password to be able // to create the scale set. This is a temporary fix. // We need to think about azure access at some point. pw, err := azure.GeneratePassword() if err != nil { return err } scaleSet := azure.ScaleSet{ Name: input.Name, NamePrefix: input.NamePrefix, UID: c.uid, Location: c.location, InstanceType: input.InstanceType, StateDiskSizeGB: input.StateDiskSizeGB, StateDiskType: input.StateDiskType, Count: int64(input.Count), Username: "constellation", SubnetID: c.subnetID, NetworkSecurityGroup: c.networkSecurityGroup, Image: input.Image, Password: pw, UserAssignedIdentity: input.UserAssingedIdentity, Subscription: c.subscriptionID, ResourceGroup: c.resourceGroup, LoadBalancerName: c.loadBalancerName, LoadBalancerBackendAddressPool: input.LoadBalancerBackendAddressPool, ConfidentialVM: input.ConfidentialVM, }.Azure() _, err = c.scaleSetsAPI.BeginCreateOrUpdate( ctx, c.resourceGroup, input.Name, scaleSet, nil, ) if err != nil { return err } // use custom poller to wait for resource creation but skip waiting for OS provisioning. // OS provisioning does not work reliably without the azure guest agent installed. poller := poller.New[bool](&scaleSetCreationPollingHandler{ resourceGroup: c.resourceGroup, scaleSet: input.Name, scaleSetsAPI: c.scaleSetsAPI, virtualMachineScaleSetVMsAPI: c.virtualMachineScaleSetVMsAPI, }) pollCtx, cancel := context.WithTimeout(ctx, scaleSetCreateTimeout) defer cancel() _, err = poller.PollUntilDone(pollCtx, nil) return err } func (c *Client) getInstanceIPs(ctx context.Context, scaleSet string, count int) (cloudtypes.Instances, error) { instances := cloudtypes.Instances{} for i := 0; i < count; i++ { // get public ip address var publicIPAddress string pager := c.publicIPAddressesAPI.NewListVirtualMachineScaleSetVMPublicIPAddressesPager( c.resourceGroup, scaleSet, strconv.Itoa(i), scaleSet, scaleSet, nil) for pager.More() { page, err := pager.NextPage(ctx) if err != nil { return cloudtypes.Instances{}, err } for _, v := range page.Value { if v.Properties != nil && v.Properties.IPAddress != nil { publicIPAddress = *v.Properties.IPAddress break } } } // get private ip address var privateIPAddress string res, err := c.networkInterfacesAPI.GetVirtualMachineScaleSetNetworkInterface( ctx, c.resourceGroup, scaleSet, strconv.Itoa(i), scaleSet, nil) if err != nil { return nil, err } configs := res.Interface.Properties.IPConfigurations for _, config := range configs { privateIPAddress = *config.Properties.PrivateIPAddress break } instance := cloudtypes.Instance{ PrivateIP: privateIPAddress, PublicIP: publicIPAddress, } instances[strconv.Itoa(i)] = instance } return instances, nil } // CreateScaleSetInput is the input for a CreateScaleSet operation. type CreateScaleSetInput struct { Name string NamePrefix string Count int InstanceType string StateDiskSizeGB int32 StateDiskType string Image string UserAssingedIdentity string LoadBalancerBackendAddressPool string ConfidentialVM bool } // scaleSetCreationPollingHandler is a custom poller used to check if a scale set was created successfully. type scaleSetCreationPollingHandler struct { done bool instanceIDOffset int resourceGroup string scaleSet string scaleSetsAPI scaleSetsAPI virtualMachineScaleSetVMsAPI virtualMachineScaleSetVMsAPI } // Done returns true if the condition is met. func (h *scaleSetCreationPollingHandler) Done() bool { return h.done } // Poll checks if the scale set resource was created successfully and every VM is starting or running. func (h *scaleSetCreationPollingHandler) Poll(ctx context.Context) error { // check if scale set can be retrieved from API scaleSet, err := h.scaleSetsAPI.Get(ctx, h.resourceGroup, h.scaleSet, nil) if err != nil { return ignoreNotFoundError(err) } if scaleSet.SKU == nil || scaleSet.SKU.Capacity == nil { return errors.New("invalid scale set capacity") } // check if every VM in the scale set has power state starting or running for i := h.instanceIDOffset; i < int(*scaleSet.SKU.Capacity); i++ { instanceView, err := h.virtualMachineScaleSetVMsAPI.GetInstanceView(ctx, h.resourceGroup, h.scaleSet, strconv.Itoa(i), nil) if err != nil { return ignoreNotFoundError(err) } if !vmIsStartingOrRunning(instanceView.Statuses) { return nil } h.instanceIDOffset = i + 1 // skip this VM in the next Poll() invocation } h.done = true return nil } // Result returns the result of the poller if the condition is met. // If the condition is not met, an error is returned. func (h *scaleSetCreationPollingHandler) Result(ctx context.Context, out *bool) error { if !h.done { return fmt.Errorf("failed to create scale set") } *out = h.done return nil } func ignoreNotFoundError(err error) error { var respErr *azcore.ResponseError if errors.As(err, &respErr) && respErr.StatusCode == http.StatusNotFound { // resource does not exist yet - retry later return nil } return err } func vmIsStartingOrRunning(statuses []*armcomputev2.InstanceViewStatus) bool { for _, status := range statuses { if status == nil || status.Code == nil { continue } switch *status.Code { case powerStateStarting: return true case powerStateRunning: return true } } return false }