constellation/cli/internal/terraform/terraform.go

/*
Copyright (c) Edgeless Systems GmbH

SPDX-License-Identifier: AGPL-3.0-only
*/

/*
Package terraform handles creation/destruction of a Constellation cluster using Terraform.

Since Terraform does not provide a stable Go API, we use the `terraform-exec` package to interact with Terraform.

The Terraform templates are located in the "terraform" subdirectory. The templates are embedded into the CLI binary using `go:embed`.
On use the relevant template is extracted to the working directory and the user customized variables are written to a `terraform.tfvars` file.
*/
package terraform

import (
	"context"
	"errors"
	"fmt"
	"io"
	"path/filepath"

	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
	"github.com/edgelesssys/constellation/v2/internal/constants"
	"github.com/edgelesssys/constellation/v2/internal/file"
	"github.com/hashicorp/go-version"
	install "github.com/hashicorp/hc-install"
	"github.com/hashicorp/hc-install/fs"
	"github.com/hashicorp/hc-install/product"
	"github.com/hashicorp/hc-install/releases"
	"github.com/hashicorp/hc-install/src"
	"github.com/hashicorp/terraform-exec/tfexec"
	tfjson "github.com/hashicorp/terraform-json"
	"github.com/spf13/afero"
)

const (
	tfVersion         = ">= 1.4.6"
	terraformVarsFile = "terraform.tfvars"

	// terraformUpgradePlanFile is the file name of the zipfile created by Terraform plan for Constellation upgrades.
	terraformUpgradePlanFile = "plan.zip"
)

// ErrTerraformWorkspaceExistsWithDifferentVariables is returned when existing Terraform files differ from the version the CLI wants to extract.
var ErrTerraformWorkspaceExistsWithDifferentVariables = errors.New("creating cluster: a Terraform workspace already exists with different variables")

// Client manages interaction with Terraform.
type Client struct {
	tf tfInterface

	manualStateMigrations []StateMigration
	file                  file.Handler
	workingDir            string
	remove                func()
}

// New sets up a new Client for Terraform.
func New(ctx context.Context, workingDir string) (*Client, error) {
	file := file.NewHandler(afero.NewOsFs())
	if err := file.MkdirAll(workingDir); err != nil {
		return nil, err
	}
	tf, remove, err := GetExecutable(ctx, workingDir)
	if err != nil {
		return nil, err
	}

	return &Client{
		tf:         tf,
		remove:     remove,
		file:       file,
		workingDir: workingDir,
	}, nil
}

// WithManualStateMigration adds a manual state migration to the Client.
func (c *Client) WithManualStateMigration(migration StateMigration) *Client {
	c.manualStateMigrations = append(c.manualStateMigrations, migration)
	return c
}

// ShowIAM reads the state of Constellation IAM resources from Terraform.
func (c *Client) ShowIAM(ctx context.Context, provider cloudprovider.Provider) (IAMOutput, error) {
	tfState, err := c.tf.Show(ctx)
	if err != nil {
		return IAMOutput{}, err
	}
	if tfState == nil || tfState.Values == nil {
		return IAMOutput{}, errors.New("terraform show: no values returned")
	}

	switch provider {
	case cloudprovider.GCP:
		saKeyOutputRaw, ok := tfState.Values.Outputs["sa_key"]
		if !ok {
			return IAMOutput{}, errors.New("no service account key output found")
		}
		saKeyOutput, ok := saKeyOutputRaw.Value.(string)
		if !ok {
			return IAMOutput{}, errors.New("invalid type in service account key output: not a string")
		}
		return IAMOutput{
			GCP: GCPIAMOutput{
				SaKey: saKeyOutput,
			},
		}, nil
	case cloudprovider.Azure:
		subscriptionIDRaw, ok := tfState.Values.Outputs["subscription_id"]
		if !ok {
			return IAMOutput{}, errors.New("no subscription id output found")
		}
		subscriptionIDOutput, ok := subscriptionIDRaw.Value.(string)
		if !ok {
			return IAMOutput{}, errors.New("invalid type in subscription id output: not a string")
		}
		tenantIDRaw, ok := tfState.Values.Outputs["tenant_id"]
		if !ok {
			return IAMOutput{}, errors.New("no tenant id output found")
		}
		tenantIDOutput, ok := tenantIDRaw.Value.(string)
		if !ok {
			return IAMOutput{}, errors.New("invalid type in tenant id output: not a string")
		}
		uamiIDRaw, ok := tfState.Values.Outputs["uami_id"]
		if !ok {
			return IAMOutput{}, errors.New("no UAMI id output found")
		}
		uamiIDOutput, ok := uamiIDRaw.Value.(string)
		if !ok {
			return IAMOutput{}, errors.New("invalid type in UAMI id output: not a string")
		}
		return IAMOutput{
			Azure: AzureIAMOutput{
				SubscriptionID: subscriptionIDOutput,
				TenantID:       tenantIDOutput,
				UAMIID:         uamiIDOutput,
			},
		}, nil
	case cloudprovider.AWS:
		controlPlaneProfileRaw, ok := tfState.Values.Outputs["control_plane_instance_profile"]
		if !ok {
			return IAMOutput{}, errors.New("no control plane instance profile output found")
		}
		controlPlaneProfileOutput, ok := controlPlaneProfileRaw.Value.(string)
		if !ok {
			return IAMOutput{}, errors.New("invalid type in control plane instance profile output: not a string")
		}
		workerNodeProfileRaw, ok := tfState.Values.Outputs["worker_nodes_instance_profile"]
		if !ok {
			return IAMOutput{}, errors.New("no worker node instance profile output found")
		}
		workerNodeProfileOutput, ok := workerNodeProfileRaw.Value.(string)
		if !ok {
			return IAMOutput{}, errors.New("invalid type in worker node instance profile output: not a string")
		}
		return IAMOutput{
			AWS: AWSIAMOutput{
				ControlPlaneInstanceProfile: controlPlaneProfileOutput,
				WorkerNodeInstanceProfile:   workerNodeProfileOutput,
			},
		}, nil
	default:
		return IAMOutput{}, errors.New("unsupported cloud provider")
	}
}

// ShowCluster reads the state of Constellation cluster resources from Terraform.
func (c *Client) ShowCluster(ctx context.Context, provider cloudprovider.Provider) (ApplyOutput, error) {
	tfState, err := c.tf.Show(ctx)
	if err != nil {
		return ApplyOutput{}, fmt.Errorf("terraform show: %w", err)
	}
	if tfState.Values == nil {
		return ApplyOutput{}, errors.New("terraform show: no values returned")
	}

	ipOutput, ok := tfState.Values.Outputs["ip"]
	if !ok {
		return ApplyOutput{}, errors.New("no IP output found")
	}
	ip, ok := ipOutput.Value.(string)
	if !ok {
		return ApplyOutput{}, errors.New("invalid type in IP output: not a string")
	}

	apiServerCertSANsOutput, ok := tfState.Values.Outputs["api_server_cert_sans"]
	if !ok {
		return ApplyOutput{}, errors.New("no api_server_cert_sans output found")
	}
	apiServerCertSANsUntyped, ok := apiServerCertSANsOutput.Value.([]any)
	if !ok {
		return ApplyOutput{}, fmt.Errorf("invalid type in api_server_cert_sans output: %s is not a list of elements", apiServerCertSANsOutput.Type.FriendlyName())
	}
	apiServerCertSANs, err := toStringSlice(apiServerCertSANsUntyped)
	if err != nil {
		return ApplyOutput{}, fmt.Errorf("convert api_server_cert_sans output: %w", err)
	}

	secretOutput, ok := tfState.Values.Outputs["initSecret"]
	if !ok {
		return ApplyOutput{}, errors.New("no initSecret output found")
	}
	secret, ok := secretOutput.Value.(string)
	if !ok {
		return ApplyOutput{}, errors.New("invalid type in initSecret output: not a string")
	}

	uidOutput, ok := tfState.Values.Outputs["uid"]
	if !ok {
		return ApplyOutput{}, errors.New("no uid output found")
	}
	uid, ok := uidOutput.Value.(string)
	if !ok {
		return ApplyOutput{}, errors.New("invalid type in uid output: not a string")
	}

	res := ApplyOutput{
		IP:                ip,
		APIServerCertSANs: apiServerCertSANs,
		Secret:            secret,
		UID:               uid,
	}

	switch provider {
	case cloudprovider.GCP:
		gcpProjectOutput, ok := tfState.Values.Outputs["project"]
		if !ok {
			return ApplyOutput{}, errors.New("no project output found")
		}
		gcpProject, ok := gcpProjectOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in project output: not a string")
		}

		cidrNodesOutput, ok := tfState.Values.Outputs["ip_cidr_nodes"]
		if !ok {
			return ApplyOutput{}, errors.New("no ip_cidr_nodes output found")
		}
		cidrNodes, ok := cidrNodesOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in ip_cidr_nodes output: not a string")
		}

		cidrPodsOutput, ok := tfState.Values.Outputs["ip_cidr_pods"]
		if !ok {
			return ApplyOutput{}, errors.New("no ip_cidr_pods output found")
		}
		cidrPods, ok := cidrPodsOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in ip_cidr_pods output: not a string")
		}

		res.GCP = &GCPApplyOutput{
			ProjectID:  gcpProject,
			IPCidrNode: cidrNodes,
			IPCidrPod:  cidrPods,
		}
	case cloudprovider.Azure:
		attestationURLOutput, ok := tfState.Values.Outputs["attestationURL"]
		if !ok {
			return ApplyOutput{}, errors.New("no attestationURL output found")
		}
		attestationURL, ok := attestationURLOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in attestationURL output: not a string")
		}

		azureUAMIOutput, ok := tfState.Values.Outputs["user_assigned_identity_client_id"]
		if !ok {
			return ApplyOutput{}, errors.New("no user_assigned_identity_client_id output found")
		}
		azureUAMI, ok := azureUAMIOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in user_assigned_identity_client_id output: not a string")
		}

		rgOutput, ok := tfState.Values.Outputs["resource_group"]
		if !ok {
			return ApplyOutput{}, errors.New("no resource_group output found")
		}
		rg, ok := rgOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in resource_group output: not a string")
		}

		subscriptionOutput, ok := tfState.Values.Outputs["subscription_id"]
		if !ok {
			return ApplyOutput{}, errors.New("no subscription_id output found")
		}
		subscriptionID, ok := subscriptionOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in subscription_id output: not a string")
		}

		networkSGNameOutput, ok := tfState.Values.Outputs["network_security_group_name"]
		if !ok {
			return ApplyOutput{}, errors.New("no network_security_group_name output found")
		}
		networkSGName, ok := networkSGNameOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in network_security_group_name output: not a string")
		}
		loadBalancerNameOutput, ok := tfState.Values.Outputs["loadbalancer_name"]
		if !ok {
			return ApplyOutput{}, errors.New("no loadbalancer_name output found")
		}
		loadBalancerName, ok := loadBalancerNameOutput.Value.(string)
		if !ok {
			return ApplyOutput{}, errors.New("invalid type in loadbalancer_name output: not a string")
		}
		res.Azure = &AzureApplyOutput{
			ResourceGroup:            rg,
			SubscriptionID:           subscriptionID,
			UserAssignedIdentity:     azureUAMI,
			NetworkSecurityGroupName: networkSGName,
			LoadBalancerName:         loadBalancerName,
			AttestationURL:           attestationURL,
		}
	}
	return res, nil
}

// PrepareWorkspace prepares a Terraform workspace for a Constellation cluster.
func (c *Client) PrepareWorkspace(path string, vars Variables) error {
	if err := prepareWorkspace(path, c.file, c.workingDir); err != nil {
		return fmt.Errorf("prepare workspace: %w", err)
	}

	return c.writeVars(vars)
}

// PrepareUpgradeWorkspace prepares a Terraform workspace for a Constellation version upgrade.
// It copies the Terraform state from the old working dir and the embedded Terraform files into the new working dir.
func (c *Client) PrepareUpgradeWorkspace(path, oldWorkingDir, newWorkingDir, backupDir string, vars Variables) error {
	if err := prepareUpgradeWorkspace(path, c.file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
		return fmt.Errorf("prepare upgrade workspace: %w", err)
	}

	return c.writeVars(vars)
}

// PrepareIAMUpgradeWorkspace prepares a Terraform workspace for a Constellation IAM upgrade.
func PrepareIAMUpgradeWorkspace(file file.Handler, path, oldWorkingDir, newWorkingDir, backupDir string) error {
	if err := prepareUpgradeWorkspace(path, file, oldWorkingDir, newWorkingDir, backupDir); err != nil {
		return fmt.Errorf("prepare upgrade workspace: %w", err)
	}
	// copy the vars file from the old working dir to the new working dir
	if err := file.CopyFile(filepath.Join(oldWorkingDir, terraformVarsFile), filepath.Join(newWorkingDir, terraformVarsFile)); err != nil {
		return fmt.Errorf("copying vars file: %w", err)
	}
	return nil
}

// CreateCluster creates a Constellation cluster using Terraform.
func (c *Client) CreateCluster(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (ApplyOutput, error) {
	if err := c.setLogLevel(logLevel); err != nil {
		return ApplyOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
	}

	if err := c.tf.Init(ctx); err != nil {
		return ApplyOutput{}, fmt.Errorf("terraform init: %w", err)
	}

	if err := c.applyManualStateMigrations(ctx); err != nil {
		return ApplyOutput{}, fmt.Errorf("apply manual state migrations: %w", err)
	}

	if err := c.tf.Apply(ctx); err != nil {
		return ApplyOutput{}, fmt.Errorf("terraform apply: %w", err)
	}

	return c.ShowCluster(ctx, provider)
}

// ApplyOutput contains the Terraform output values of a cluster creation
// or apply operation.
type ApplyOutput struct {
	IP                string
	APIServerCertSANs []string
	Secret            string
	UID               string
	GCP               *GCPApplyOutput
	Azure             *AzureApplyOutput
}

// AzureApplyOutput contains the Terraform output values of a terraform apply operation on Microsoft Azure.
type AzureApplyOutput struct {
	ResourceGroup            string
	SubscriptionID           string
	NetworkSecurityGroupName string
	LoadBalancerName         string
	UserAssignedIdentity     string
	// AttestationURL is the URL of the attestation provider.
	AttestationURL string
}

// GCPApplyOutput contains the Terraform output values of a terraform apply operation on GCP.
type GCPApplyOutput struct {
	ProjectID  string
	IPCidrNode string
	IPCidrPod  string
}

// IAMOutput contains the output information of the Terraform IAM operations.
type IAMOutput struct {
	GCP   GCPIAMOutput
	Azure AzureIAMOutput
	AWS   AWSIAMOutput
}

// GCPIAMOutput contains the output information of the Terraform IAM operation on GCP.
type GCPIAMOutput struct {
	SaKey string
}

// AzureIAMOutput contains the output information of the Terraform IAM operation on Microsoft Azure.
type AzureIAMOutput struct {
	SubscriptionID string
	TenantID       string
	UAMIID         string
}

// AWSIAMOutput contains the output information of the Terraform IAM operation on GCP.
type AWSIAMOutput struct {
	ControlPlaneInstanceProfile string
	WorkerNodeInstanceProfile   string
}

// ApplyIAMConfig creates an IAM configuration using Terraform.
func (c *Client) ApplyIAMConfig(ctx context.Context, provider cloudprovider.Provider, logLevel LogLevel) (IAMOutput, error) {
	if err := c.setLogLevel(logLevel); err != nil {
		return IAMOutput{}, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
	}

	if err := c.tf.Init(ctx); err != nil {
		return IAMOutput{}, err
	}

	if err := c.tf.Apply(ctx); err != nil {
		return IAMOutput{}, err
	}
	return c.ShowIAM(ctx, provider)
}

// Plan determines the diff that will be applied by Terraform.
// The plan output is written to the Terraform working directory.
// If there is a diff, the returned bool is true. Otherwise, it is false.
func (c *Client) Plan(ctx context.Context, logLevel LogLevel) (bool, error) {
	if err := c.setLogLevel(logLevel); err != nil {
		return false, fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
	}

	if err := c.tf.Init(ctx); err != nil {
		return false, fmt.Errorf("terraform init: %w", err)
	}

	if err := c.applyManualStateMigrations(ctx); err != nil {
		return false, fmt.Errorf("apply manual state migrations: %w", err)
	}

	opts := []tfexec.PlanOption{
		tfexec.Out(terraformUpgradePlanFile),
	}
	return c.tf.Plan(ctx, opts...)
}

// ShowPlan formats the diff of a plan file in the Terraform working directory,
// and writes it to the specified output.
func (c *Client) ShowPlan(ctx context.Context, logLevel LogLevel, output io.Writer) error {
	if err := c.setLogLevel(logLevel); err != nil {
		return fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
	}

	planResult, err := c.tf.ShowPlanFileRaw(ctx, terraformUpgradePlanFile)
	if err != nil {
		return fmt.Errorf("terraform show plan: %w", err)
	}

	_, err = output.Write([]byte(planResult))
	if err != nil {
		return fmt.Errorf("write plan output: %w", err)
	}

	return nil
}

// Destroy destroys Terraform-created cloud resources.
func (c *Client) Destroy(ctx context.Context, logLevel LogLevel) error {
	if err := c.setLogLevel(logLevel); err != nil {
		return fmt.Errorf("set terraform log level %s: %w", logLevel.String(), err)
	}

	if err := c.tf.Init(ctx); err != nil {
		return fmt.Errorf("terraform init: %w", err)
	}
	return c.tf.Destroy(ctx)
}

// RemoveInstaller removes the Terraform installer, if it was downloaded for this command.
func (c *Client) RemoveInstaller() {
	c.remove()
}

// CleanUpWorkspace removes terraform files from the current directory.
func (c *Client) CleanUpWorkspace() error {
	return cleanUpWorkspace(c.file, c.workingDir)
}

// GetExecutable returns a Terraform executable either from the local filesystem,
// or downloads the latest version fulfilling the version constraint.
func GetExecutable(ctx context.Context, workingDir string) (terraform *tfexec.Terraform, remove func(), err error) {
	inst := install.NewInstaller()

	version, err := version.NewConstraint(tfVersion)
	if err != nil {
		return nil, nil, err
	}

	downloadVersion := &releases.LatestVersion{
		Product:     product.Terraform,
		Constraints: version,
	}
	localVersion := &fs.Version{
		Product:     product.Terraform,
		Constraints: version,
	}

	execPath, err := inst.Ensure(ctx, []src.Source{localVersion, downloadVersion})
	if err != nil {
		return nil, nil, err
	}

	tf, err := tfexec.NewTerraform(workingDir, execPath)

	return tf, func() { _ = inst.Remove(context.Background()) }, err
}

// applyManualStateMigrations applies manual state migrations that are not handled by Terraform due to missing features.
// Each migration is expected to be idempotent.
// This is a temporary solution until we can remove the need for manual state migrations.
func (c *Client) applyManualStateMigrations(ctx context.Context) error {
	for _, migration := range c.manualStateMigrations {
		if err := migration.Hook(ctx, c.tf); err != nil {
			return fmt.Errorf("apply manual state migration %s: %w", migration.DisplayName, err)
		}
	}
	return nil
	// expects to be run on initialized workspace
	// and only works for AWS
	// if migration fails, we expect to either be on a different CSP or that the migration has already been applied
	// and we can continue
	// c.tf.StateMv(ctx, "module.control_plane.aws_iam_role.this", "module.control_plane.aws_iam_role.control_plane")
}

// writeVars tries to write the Terraform variables file or, if it exists, checks if it is the same as we are expecting.
func (c *Client) writeVars(vars Variables) error {
	if vars == nil {
		return errors.New("creating cluster: vars is nil")
	}

	pathToVarsFile := filepath.Join(c.workingDir, terraformVarsFile)
	if err := c.file.Write(pathToVarsFile, []byte(vars.String())); errors.Is(err, afero.ErrFileExists) {
		// If a variables file already exists, check if it's the same as we're expecting, so we can continue using it.
		varsContent, err := c.file.Read(pathToVarsFile)
		if err != nil {
			return fmt.Errorf("read variables file: %w", err)
		}
		if vars.String() != string(varsContent) {
			return ErrTerraformWorkspaceExistsWithDifferentVariables
		}
	} else if err != nil {
		return fmt.Errorf("write variables file: %w", err)
	}

	return nil
}

// setLogLevel sets the log level for Terraform.
func (c *Client) setLogLevel(logLevel LogLevel) error {
	if logLevel.String() != "" {
		if err := c.tf.SetLog(logLevel.String()); err != nil {
			return fmt.Errorf("set log level %s: %w", logLevel.String(), err)
		}

		// Terraform writes its log to the working directory.
		//  => Set the log path to the parent directory to have it in the user's working directory.
		if err := c.tf.SetLogPath(filepath.Join("..", constants.TerraformLogFile)); err != nil {
			return fmt.Errorf("set log path: %w", err)
		}
	}
	return nil
}

// StateMigration is a manual state migration that is not handled by Terraform due to missing features.
// TODO(AB#3248): Remove this after we can assume that all existing clusters have been migrated.
type StateMigration struct {
	DisplayName string
	Hook        func(ctx context.Context, tfClient TFMigrator) error
}

func toStringSlice(in []any) ([]string, error) {
	out := make([]string, len(in))
	for i, v := range in {
		s, ok := v.(string)
		if !ok {
			return nil, fmt.Errorf("invalid type in list: item at index %v of list is not a string", i)
		}
		out[i] = s
	}
	return out, nil
}

type tfInterface interface {
	Apply(context.Context, ...tfexec.ApplyOption) error
	Destroy(context.Context, ...tfexec.DestroyOption) error
	Init(context.Context, ...tfexec.InitOption) error
	Show(context.Context, ...tfexec.ShowOption) (*tfjson.State, error)
	Plan(ctx context.Context, opts ...tfexec.PlanOption) (bool, error)
	ShowPlanFileRaw(ctx context.Context, planPath string, opts ...tfexec.ShowOption) (string, error)
	SetLog(level string) error
	SetLogPath(path string) error
	TFMigrator
}

// TFMigrator is an interface for manual terraform state migrations (terraform state mv).
// TODO(AB#3248): Remove this after we can assume that all existing clusters have been migrated.
type TFMigrator interface {
	StateMv(ctx context.Context, src, dst string, opts ...tfexec.StateMvCmdOption) error
}