constellation/cli/internal/cloudcmd/create.go

/*
Copyright (c) Edgeless Systems GmbH

SPDX-License-Identifier: AGPL-3.0-only
*/

package cloudcmd

import (
	"context"
	"encoding/base64"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"path"
	"regexp"
	"runtime"
	"strings"

	"github.com/Azure/azure-sdk-for-go/profiles/latest/attestation/attestation"
	azpolicy "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"

	"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
	"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
	"github.com/edgelesssys/constellation/v2/internal/config"
	"github.com/edgelesssys/constellation/v2/internal/constants"
	"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
	"github.com/edgelesssys/constellation/v2/internal/variant"
)

// Creator creates cloud resources.
type Creator struct {
	out                io.Writer
	image              imageFetcher
	newTerraformClient func(ctx context.Context) (terraformClient, error)
	newLibvirtRunner   func() libvirtRunner
	newRawDownloader   func() rawDownloader
	policyPatcher      PolicyPatcher
}

// NewCreator creates a new creator.
func NewCreator(out io.Writer) *Creator {
	return &Creator{
		out:   out,
		image: imagefetcher.New(),
		newTerraformClient: func(ctx context.Context) (terraformClient, error) {
			return terraform.New(ctx, constants.TerraformWorkingDir)
		},
		newLibvirtRunner: func() libvirtRunner {
			return libvirt.New()
		},
		newRawDownloader: func() rawDownloader {
			return imagefetcher.NewDownloader()
		},
		policyPatcher: policyPatcher{},
	}
}

// CreateOptions are the options for creating a Constellation cluster.
type CreateOptions struct {
	Provider          cloudprovider.Provider
	Config            *config.Config
	InsType           string
	ControlPlaneCount int
	WorkerCount       int
	image             string
	TFLogLevel        terraform.LogLevel
}

// Create creates the handed amount of instances and all the needed resources.
func (c *Creator) Create(ctx context.Context, opts CreateOptions) (clusterid.File, error) {
	provider := opts.Config.GetProvider()
	attestationVariant := opts.Config.GetAttestationConfig().GetVariant()
	region := opts.Config.GetRegion()
	image, err := c.image.FetchReference(ctx, provider, attestationVariant, opts.Config.Image, region)
	if err != nil {
		return clusterid.File{}, fmt.Errorf("fetching image reference: %w", err)
	}
	opts.image = image

	switch opts.Provider {
	case cloudprovider.AWS:
		cl, err := c.newTerraformClient(ctx)
		if err != nil {
			return clusterid.File{}, err
		}
		defer cl.RemoveInstaller()
		return c.createAWS(ctx, cl, opts)
	case cloudprovider.GCP:
		cl, err := c.newTerraformClient(ctx)
		if err != nil {
			return clusterid.File{}, err
		}
		defer cl.RemoveInstaller()
		return c.createGCP(ctx, cl, opts)
	case cloudprovider.Azure:
		cl, err := c.newTerraformClient(ctx)
		if err != nil {
			return clusterid.File{}, err
		}
		defer cl.RemoveInstaller()
		return c.createAzure(ctx, cl, opts)
	case cloudprovider.OpenStack:
		cl, err := c.newTerraformClient(ctx)
		if err != nil {
			return clusterid.File{}, err
		}
		defer cl.RemoveInstaller()
		return c.createOpenStack(ctx, cl, opts)
	case cloudprovider.QEMU:
		if runtime.GOARCH != "amd64" || runtime.GOOS != "linux" {
			return clusterid.File{}, fmt.Errorf("creation of a QEMU based Constellation is not supported for %s/%s", runtime.GOOS, runtime.GOARCH)
		}
		cl, err := c.newTerraformClient(ctx)
		if err != nil {
			return clusterid.File{}, err
		}
		defer cl.RemoveInstaller()
		lv := c.newLibvirtRunner()
		qemuOpts := qemuCreateOptions{
			source:        image,
			CreateOptions: opts,
		}
		return c.createQEMU(ctx, cl, lv, qemuOpts)
	default:
		return clusterid.File{}, fmt.Errorf("unsupported cloud provider: %s", opts.Provider)
	}
}

func (c *Creator) createAWS(ctx context.Context, cl terraformClient, opts CreateOptions) (idFile clusterid.File, retErr error) {
	vars := terraform.AWSClusterVariables{
		CommonVariables: terraform.CommonVariables{
			Name:               opts.Config.Name,
			CountControlPlanes: opts.ControlPlaneCount,
			CountWorkers:       opts.WorkerCount,
			StateDiskSizeGB:    opts.Config.StateDiskSizeGB,
		},
		StateDiskType:          opts.Config.Provider.AWS.StateDiskType,
		Region:                 opts.Config.Provider.AWS.Region,
		Zone:                   opts.Config.Provider.AWS.Zone,
		InstanceType:           opts.InsType,
		AMIImageID:             opts.image,
		IAMProfileControlPlane: opts.Config.Provider.AWS.IAMProfileControlPlane,
		IAMProfileWorkerNodes:  opts.Config.Provider.AWS.IAMProfileWorkerNodes,
		Debug:                  opts.Config.IsDebugCluster(),
	}

	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.AWS.String())), &vars); err != nil {
		return clusterid.File{}, err
	}

	defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
	tfOutput, err := cl.CreateCluster(ctx, opts.TFLogLevel)
	if err != nil {
		return clusterid.File{}, err
	}

	return clusterid.File{
		CloudProvider: cloudprovider.AWS,
		InitSecret:    []byte(tfOutput.Secret),
		IP:            tfOutput.IP,
		UID:           tfOutput.UID,
	}, nil
}

func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts CreateOptions) (idFile clusterid.File, retErr error) {
	vars := terraform.GCPClusterVariables{
		CommonVariables: terraform.CommonVariables{
			Name:               opts.Config.Name,
			CountControlPlanes: opts.ControlPlaneCount,
			CountWorkers:       opts.WorkerCount,
			StateDiskSizeGB:    opts.Config.StateDiskSizeGB,
		},
		Project:         opts.Config.Provider.GCP.Project,
		Region:          opts.Config.Provider.GCP.Region,
		Zone:            opts.Config.Provider.GCP.Zone,
		CredentialsFile: opts.Config.Provider.GCP.ServiceAccountKeyPath,
		InstanceType:    opts.InsType,
		StateDiskType:   opts.Config.Provider.GCP.StateDiskType,
		ImageID:         opts.image,
		Debug:           opts.Config.IsDebugCluster(),
	}

	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.GCP.String())), &vars); err != nil {
		return clusterid.File{}, err
	}

	defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
	tfOutput, err := cl.CreateCluster(ctx, opts.TFLogLevel)
	if err != nil {
		return clusterid.File{}, err
	}

	return clusterid.File{
		CloudProvider: cloudprovider.GCP,
		InitSecret:    []byte(tfOutput.Secret),
		IP:            tfOutput.IP,
		UID:           tfOutput.UID,
	}, nil
}

func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (idFile clusterid.File, retErr error) {
	vars := terraform.AzureClusterVariables{
		CommonVariables: terraform.CommonVariables{
			Name:               opts.Config.Name,
			CountControlPlanes: opts.ControlPlaneCount,
			CountWorkers:       opts.WorkerCount,
			StateDiskSizeGB:    opts.Config.StateDiskSizeGB,
		},
		Location:             opts.Config.Provider.Azure.Location,
		ResourceGroup:        opts.Config.Provider.Azure.ResourceGroup,
		UserAssignedIdentity: opts.Config.Provider.Azure.UserAssignedIdentity,
		InstanceType:         opts.InsType,
		StateDiskType:        opts.Config.Provider.Azure.StateDiskType,
		ImageID:              opts.image,
		SecureBoot:           *opts.Config.Provider.Azure.SecureBoot,
		CreateMAA:            opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}),
		Debug:                opts.Config.IsDebugCluster(),
	}

	vars.ConfidentialVM = opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})

	vars = normalizeAzureURIs(vars)

	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.Azure.String())), &vars); err != nil {
		return clusterid.File{}, err
	}

	defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
	tfOutput, err := cl.CreateCluster(ctx, opts.TFLogLevel)
	if err != nil {
		return clusterid.File{}, err
	}

	if vars.CreateMAA {
		// Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
		if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
			return clusterid.File{}, err
		}
	}

	return clusterid.File{
		CloudProvider:  cloudprovider.Azure,
		IP:             tfOutput.IP,
		InitSecret:     []byte(tfOutput.Secret),
		UID:            tfOutput.UID,
		AttestationURL: tfOutput.AttestationURL,
	}, nil
}

// PolicyPatcher interacts with Azure to update the attestation policy.
type PolicyPatcher interface {
	Patch(ctx context.Context, attestationURL string) error
}

type policyPatcher struct{}

// Patch updates the attestation policy to the base64-encoded attestation policy JWT for the given attestation URL.
// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#next-steps
func (p policyPatcher) Patch(ctx context.Context, attestationURL string) error {
	// hacky way to update the MAA attestation policy. This should be changed as soon as either the Terraform provider supports it
	// or the Go SDK gets updated to a recent API version.
	// https://github.com/hashicorp/terraform-provider-azurerm/issues/20804
	cred, err := azidentity.NewDefaultAzureCredential(nil)
	if err != nil {
		return fmt.Errorf("retrieving default Azure credentials: %w", err)
	}
	token, err := cred.GetToken(ctx, azpolicy.TokenRequestOptions{
		Scopes: []string{"https://attest.azure.net/.default"},
	})
	if err != nil {
		return fmt.Errorf("retrieving token from default Azure credentials: %w", err)
	}

	client := attestation.NewPolicyClient()

	// azureGuest is the id for the "Azure VM" attestation type. Other types are documented here:
	// https://learn.microsoft.com/en-us/rest/api/attestation/policy/set
	req, err := client.SetPreparer(ctx, attestationURL, "azureGuest", p.encodeAttestationPolicy())
	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.Token))
	if err != nil {
		return fmt.Errorf("preparing request: %w", err)
	}

	resp, err := client.Send(req)
	if err != nil {
		return fmt.Errorf("sending request: %w", err)
	}
	resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return fmt.Errorf("updating attestation policy: unexpected status code: %s", resp.Status)
	}

	return nil
}

// encodeAttestationPolicy encodes the base64-encoded attestation policy in the JWS format specified here:
// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#creating-the-policy-file-in-json-web-signature-format
func (p policyPatcher) encodeAttestationPolicy() string {
	const policy = `
                version= 1.0;
                authorizationrules
                {
                    [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny();
                    [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny();
                    // The line below was edited by the Constellation CLI. Do not edit manually.
                    //[type=="secureboot", value==false] => deny();
                    [type=="x-ms-azurevm-signingdisabled", value==false] => deny();
                    [type=="x-ms-azurevm-dbvalidated", value==false] => deny();
                    [type=="x-ms-azurevm-dbxvalidated", value==false] => deny();
                    => permit();
                };
                issuancerules
                {
                };`
	encodedPolicy := base64.RawURLEncoding.EncodeToString([]byte(policy))
	const header = `{"alg":"none"}`
	payload := fmt.Sprintf(`{"AttestationPolicy":"%s"}`, encodedPolicy)

	encodedHeader := base64.RawURLEncoding.EncodeToString([]byte(header))
	encodedPayload := base64.RawURLEncoding.EncodeToString([]byte(payload))

	return fmt.Sprintf("%s.%s.", encodedHeader, encodedPayload)
}

// The azurerm Terraform provider enforces its own convention of case sensitivity for Azure URIs which Azure's API itself does not enforce or, even worse, actually returns.
// Let's go loco with case insensitive Regexp here and fix the user input here to be compliant with this arbitrary design decision.
var (
	caseInsensitiveSubscriptionsRegexp          = regexp.MustCompile(`(?i)\/subscriptions\/`)
	caseInsensitiveResourceGroupRegexp          = regexp.MustCompile(`(?i)\/resourcegroups\/`)
	caseInsensitiveProvidersRegexp              = regexp.MustCompile(`(?i)\/providers\/`)
	caseInsensitiveUserAssignedIdentitiesRegexp = regexp.MustCompile(`(?i)\/userassignedidentities\/`)
	caseInsensitiveMicrosoftManagedIdentity     = regexp.MustCompile(`(?i)\/microsoft.managedidentity\/`)
	caseInsensitiveCommunityGalleriesRegexp     = regexp.MustCompile(`(?i)\/communitygalleries\/`)
	caseInsensitiveImagesRegExp                 = regexp.MustCompile(`(?i)\/images\/`)
	caseInsensitiveVersionsRegExp               = regexp.MustCompile(`(?i)\/versions\/`)
)

func normalizeAzureURIs(vars terraform.AzureClusterVariables) terraform.AzureClusterVariables {
	vars.UserAssignedIdentity = caseInsensitiveSubscriptionsRegexp.ReplaceAllString(vars.UserAssignedIdentity, "/subscriptions/")
	vars.UserAssignedIdentity = caseInsensitiveResourceGroupRegexp.ReplaceAllString(vars.UserAssignedIdentity, "/resourceGroups/")
	vars.UserAssignedIdentity = caseInsensitiveProvidersRegexp.ReplaceAllString(vars.UserAssignedIdentity, "/providers/")
	vars.UserAssignedIdentity = caseInsensitiveUserAssignedIdentitiesRegexp.ReplaceAllString(vars.UserAssignedIdentity, "/userAssignedIdentities/")
	vars.UserAssignedIdentity = caseInsensitiveMicrosoftManagedIdentity.ReplaceAllString(vars.UserAssignedIdentity, "/Microsoft.ManagedIdentity/")
	vars.ImageID = caseInsensitiveCommunityGalleriesRegexp.ReplaceAllString(vars.ImageID, "/communityGalleries/")
	vars.ImageID = caseInsensitiveImagesRegExp.ReplaceAllString(vars.ImageID, "/images/")
	vars.ImageID = caseInsensitiveVersionsRegExp.ReplaceAllString(vars.ImageID, "/versions/")

	return vars
}

func (c *Creator) createOpenStack(ctx context.Context, cl terraformClient, opts CreateOptions) (idFile clusterid.File, retErr error) {
	// TODO: Remove this once OpenStack is supported.
	if os.Getenv("CONSTELLATION_OPENSTACK_DEV") != "1" {
		return clusterid.File{}, errors.New("OpenStack isn't supported yet")
	}
	if _, hasOSAuthURL := os.LookupEnv("OS_AUTH_URL"); !hasOSAuthURL && opts.Config.Provider.OpenStack.Cloud == "" {
		return clusterid.File{}, errors.New(
			"neither environment variable OS_AUTH_URL nor cloud name for \"clouds.yaml\" is set. OpenStack authentication requires a set of " +
				"OS_* environment variables that are typically sourced into the current shell with an openrc file " +
				"or a cloud name for \"clouds.yaml\". " +
				"See https://docs.openstack.org/openstacksdk/latest/user/config/configuration.html for more information",
		)
	}

	vars := terraform.OpenStackClusterVariables{
		CommonVariables: terraform.CommonVariables{
			Name:               opts.Config.Name,
			CountControlPlanes: opts.ControlPlaneCount,
			CountWorkers:       opts.WorkerCount,
			StateDiskSizeGB:    opts.Config.StateDiskSizeGB,
		},
		Cloud:                   opts.Config.Provider.OpenStack.Cloud,
		AvailabilityZone:        opts.Config.Provider.OpenStack.AvailabilityZone,
		FlavorID:                opts.Config.Provider.OpenStack.FlavorID,
		FloatingIPPoolID:        opts.Config.Provider.OpenStack.FloatingIPPoolID,
		StateDiskType:           opts.Config.Provider.OpenStack.StateDiskType,
		ImageURL:                opts.image,
		DirectDownload:          *opts.Config.Provider.OpenStack.DirectDownload,
		OpenstackUserDomainName: opts.Config.Provider.OpenStack.UserDomainName,
		OpenstackUsername:       opts.Config.Provider.OpenStack.Username,
		OpenstackPassword:       opts.Config.Provider.OpenStack.Password,
		Debug:                   opts.Config.IsDebugCluster(),
	}

	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.OpenStack.String())), &vars); err != nil {
		return clusterid.File{}, err
	}

	defer rollbackOnError(c.out, &retErr, &rollbackerTerraform{client: cl}, opts.TFLogLevel)
	tfOutput, err := cl.CreateCluster(ctx, opts.TFLogLevel)
	if err != nil {
		return clusterid.File{}, err
	}

	return clusterid.File{
		CloudProvider: cloudprovider.OpenStack,
		IP:            tfOutput.IP,
		InitSecret:    []byte(tfOutput.Secret),
		UID:           tfOutput.UID,
	}, nil
}

type qemuCreateOptions struct {
	source string
	CreateOptions
}

func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirtRunner, opts qemuCreateOptions) (idFile clusterid.File, retErr error) {
	qemuRollbacker := &rollbackerQEMU{client: cl, libvirt: lv, createdWorkspace: false}
	defer rollbackOnError(c.out, &retErr, qemuRollbacker, opts.TFLogLevel)

	// TODO: render progress bar
	downloader := c.newRawDownloader()
	imagePath, err := downloader.Download(ctx, c.out, false, opts.source, opts.Config.Image)
	if err != nil {
		return clusterid.File{}, fmt.Errorf("download raw image: %w", err)
	}

	libvirtURI := opts.Config.Provider.QEMU.LibvirtURI
	libvirtSocketPath := "."

	switch {
	// if no libvirt URI is specified, start a libvirt container
	case libvirtURI == "":
		if err := lv.Start(ctx, opts.Config.Name, opts.Config.Provider.QEMU.LibvirtContainerImage); err != nil {
			return clusterid.File{}, fmt.Errorf("start libvirt container: %w", err)
		}
		libvirtURI = libvirt.LibvirtTCPConnectURI

	// socket for system URI should be in /var/run/libvirt/libvirt-sock
	case libvirtURI == "qemu:///system":
		libvirtSocketPath = "/var/run/libvirt/libvirt-sock"

	// socket for session URI should be in /run/user/<uid>/libvirt/libvirt-sock
	case libvirtURI == "qemu:///session":
		libvirtSocketPath = fmt.Sprintf("/run/user/%d/libvirt/libvirt-sock", os.Getuid())

	// if a unix socket is specified we need to parse the URI to get the socket path
	case strings.HasPrefix(libvirtURI, "qemu+unix://"):
		unixURI, err := url.Parse(strings.TrimPrefix(libvirtURI, "qemu+unix://"))
		if err != nil {
			return clusterid.File{}, err
		}
		libvirtSocketPath = unixURI.Query().Get("socket")
		if libvirtSocketPath == "" {
			return clusterid.File{}, fmt.Errorf("socket path not specified in qemu+unix URI: %s", libvirtURI)
		}
	}

	metadataLibvirtURI := libvirtURI
	if libvirtSocketPath != "." {
		metadataLibvirtURI = "qemu:///system"
	}

	vars := terraform.QEMUVariables{
		CommonVariables: terraform.CommonVariables{
			Name:               opts.Config.Name,
			CountControlPlanes: opts.ControlPlaneCount,
			CountWorkers:       opts.WorkerCount,
			StateDiskSizeGB:    opts.Config.StateDiskSizeGB,
		},
		LibvirtURI:        libvirtURI,
		LibvirtSocketPath: libvirtSocketPath,
		// TODO(malt3): auto select boot mode based on attestation variant.
		// requires image info v2.
		BootMode:           "uefi",
		ImagePath:          imagePath,
		ImageFormat:        opts.Config.Provider.QEMU.ImageFormat,
		CPUCount:           opts.Config.Provider.QEMU.VCPUs,
		MemorySizeMiB:      opts.Config.Provider.QEMU.Memory,
		MetadataAPIImage:   opts.Config.Provider.QEMU.MetadataAPIImage,
		MetadataLibvirtURI: metadataLibvirtURI,
		NVRAM:              opts.Config.Provider.QEMU.NVRAM,
		Firmware:           opts.Config.Provider.QEMU.Firmware,
		// TODO(malt3) enable once we have a way to auto-select values for these
		// requires image info v2.
		// BzImagePath:        placeholder,
		// InitrdPath:         placeholder,
		// KernelCmdline:      placeholder,
	}

	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.QEMU.String())), &vars); err != nil {
		return clusterid.File{}, fmt.Errorf("prepare workspace: %w", err)
	}

	// Allow rollback of QEMU Terraform workspace from this point on
	qemuRollbacker.createdWorkspace = true

	tfOutput, err := cl.CreateCluster(ctx, opts.TFLogLevel)
	if err != nil {
		return clusterid.File{}, fmt.Errorf("create cluster: %w", err)
	}

	return clusterid.File{
		CloudProvider: cloudprovider.QEMU,
		InitSecret:    []byte(tfOutput.Secret),
		IP:            tfOutput.IP,
		UID:           tfOutput.UID,
	}, nil
}