cli: support custom attestation policies for maa (#1375)

* create and update maa attestation policy

* use interface to allow unit testing

* fix test csp

* http request for policy patch

* go mod tidy

* remove hyphen

* go mod tidy

* wip: adapt to feedback

* linting fixes

* remove csp from tf call

* fix type assertion

* Add MAA URL to instance tags (#1409)

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

* conditionally create maa provider

* only set instance tag when maa is created

* fix azure unit test

* bazel tidy

* remove AzureCVM const

Co-authored-by: Thomas Tendyck <51411342+thomasten@users.noreply.github.com>

* encode policy at runtime

* remove policy arg

* fix unit test

---------

Signed-off-by: Daniel Weiße <dw@edgeless.systems>
Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>
Co-authored-by: Thomas Tendyck <51411342+thomasten@users.noreply.github.com>

cli/internal/cloudcmd/BUILD.bazel

@@ -38,6 +38,9 @@ go_library(
         "//internal/versions/components",
         "//internal/versionsapi",
         "//operators/constellation-node-operator/api/v1alpha1",
+        "@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
+        "@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
+        "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",
         "@com_github_hashicorp_terraform_json//:terraform-json",
         "@com_github_spf13_cobra//:cobra",
         "@io_k8s_api//core/v1:core",

cli/internal/cloudcmd/clients_test.go

@@ -31,6 +31,7 @@ type stubTerraformClient struct {
 	initSecret             string
 	iamOutput              terraform.IAMOutput
 	uid                    string
+	attestationURL         string
 	tfjsonState            *tfjson.State
 	cleanUpWorkspaceCalled bool
 	removeInstallerCalled  bool
@@ -46,9 +47,10 @@ type stubTerraformClient struct {
 
 func (c *stubTerraformClient) CreateCluster(ctx context.Context) (terraform.CreateOutput, error) {
 	return terraform.CreateOutput{
-		IP:     c.ip,
-		Secret: c.initSecret,
-		UID:    c.uid,
+		IP:             c.ip,
+		Secret:         c.initSecret,
+		UID:            c.uid,
+		AttestationURL: c.attestationURL,
 	}, c.createClusterErr
 }
 

cli/internal/cloudcmd/create.go

@@ -8,9 +8,11 @@ package cloudcmd
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
+	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -18,6 +20,9 @@ import (
 	"runtime"
 	"strings"
 
+	"github.com/Azure/azure-sdk-for-go/profiles/latest/attestation/attestation"
+	azpolicy "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
 	"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
 	"github.com/edgelesssys/constellation/v2/cli/internal/image"
 	"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
@@ -34,6 +39,7 @@ type Creator struct {
 	newTerraformClient func(ctx context.Context) (terraformClient, error)
 	newLibvirtRunner   func() libvirtRunner
 	newRawDownloader   func() rawDownloader
+	policyPatcher      PolicyPatcher
 }
 
 // NewCreator creates a new creator.
@@ -50,6 +56,7 @@ func NewCreator(out io.Writer) *Creator {
 		newRawDownloader: func() rawDownloader {
 			return image.NewDownloader()
 		},
+		policyPatcher: policyPatcher{},
 	}
 }
 
@@ -182,8 +189,7 @@ func (c *Creator) createGCP(ctx context.Context, cl terraformClient, config *con
 	}, nil
 }
 
-func (c *Creator) createAzure(ctx context.Context, cl terraformClient, config *config.Config,
-	insType string, controlPlaneCount, workerCount int, image string,
+func (c *Creator) createAzure(ctx context.Context, cl terraformClient, config *config.Config, insType string, controlPlaneCount, workerCount int, image string,
 ) (idFile clusterid.File, retErr error) {
 	vars := terraform.AzureClusterVariables{
 		CommonVariables: terraform.CommonVariables{
@@ -200,6 +206,7 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, config *c
 		ImageID:              image,
 		ConfidentialVM:       *config.Provider.Azure.ConfidentialVM,
 		SecureBoot:           *config.Provider.Azure.SecureBoot,
+		CreateMAA:            *config.Provider.Azure.EnforceIDKeyDigest,
 		Debug:                config.IsDebugCluster(),
 	}
 
@@ -215,14 +222,98 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, config *c
 		return clusterid.File{}, err
 	}
 
+	if vars.CreateMAA {
+		// Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
+		if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
+			return clusterid.File{}, err
+		}
+	}
+
 	return clusterid.File{
-		CloudProvider: cloudprovider.Azure,
-		IP:            tfOutput.IP,
-		InitSecret:    []byte(tfOutput.Secret),
-		UID:           tfOutput.UID,
+		CloudProvider:  cloudprovider.Azure,
+		IP:             tfOutput.IP,
+		InitSecret:     []byte(tfOutput.Secret),
+		UID:            tfOutput.UID,
+		AttestationURL: tfOutput.AttestationURL,
 	}, nil
 }
 
+// PolicyPatcher interacts with Azure to update the attestation policy.
+type PolicyPatcher interface {
+	Patch(ctx context.Context, attestationURL string) error
+}
+
+type policyPatcher struct{}
+
+// Patch updates the attestation policy to the base64-encoded attestation policy JWT for the given attestation URL.
+// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#next-steps
+func (p policyPatcher) Patch(ctx context.Context, attestationURL string) error {
+	// hacky way to update the MAA attestation policy. This should be changed as soon as either the Terraform provider supports it
+	// or the Go SDK gets updated to a recent API version.
+	// https://github.com/hashicorp/terraform-provider-azurerm/issues/20804
+	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	if err != nil {
+		return fmt.Errorf("retrieving default Azure credentials: %w", err)
+	}
+	token, err := cred.GetToken(ctx, azpolicy.TokenRequestOptions{
+		Scopes: []string{"https://attest.azure.net/.default"},
+	})
+	if err != nil {
+		return fmt.Errorf("retrieving token from default Azure credentials: %w", err)
+	}
+
+	client := attestation.NewPolicyClient()
+
+	// azureGuest is the id for the "Azure VM" attestation type. Other types are documented here:
+	// https://learn.microsoft.com/en-us/rest/api/attestation/policy/set
+	req, err := client.SetPreparer(ctx, attestationURL, "azureGuest", p.encodeAttestationPolicy())
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.Token))
+	if err != nil {
+		return fmt.Errorf("preparing request: %w", err)
+	}
+
+	resp, err := client.Send(req)
+	if err != nil {
+		return fmt.Errorf("sending request: %w", err)
+	}
+	resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("updating attestation policy: unexpected status code: %s", resp.Status)
+	}
+
+	return nil
+}
+
+// encodeAttestationPolicy encodes the base64-encoded attestation policy in the JWS format specified here:
+// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#creating-the-policy-file-in-json-web-signature-format
+func (p policyPatcher) encodeAttestationPolicy() string {
+	const policy = `
+                version= 1.0;
+                authorizationrules
+                {
+                    [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny();
+                    [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny();
+                    // The line below was edited by the Constellation CLI. Do not edit manually.
+                    //[type=="secureboot", value==false] => deny();
+                    [type=="x-ms-azurevm-signingdisabled", value==false] => deny();
+                    [type=="x-ms-azurevm-dbvalidated", value==false] => deny();
+                    [type=="x-ms-azurevm-dbxvalidated", value==false] => deny();
+                    => permit();
+                };
+                issuancerules
+                {
+                };`
+	encodedPolicy := base64.RawURLEncoding.EncodeToString([]byte(policy))
+	const header = `{"alg":"none"}`
+	payload := fmt.Sprintf(`{"AttestationPolicy":"%s"}`, encodedPolicy)
+
+	encodedHeader := base64.RawURLEncoding.EncodeToString([]byte(header))
+	encodedPayload := base64.RawURLEncoding.EncodeToString([]byte(payload))
+
+	return fmt.Sprintf("%s.%s.", encodedHeader, encodedPayload)
+}
+
 // The azurerm Terraform provider enforces its own convention of case sensitivity for Azure URIs which Azure's API itself does not enforce or, even worse, actually returns.
 // Let's go loco with case insensitive Regexp here and fix the user input here to be compliant with this arbitrary design decision.
 var (

cli/internal/cloudcmd/create_test.go

@@ -30,6 +30,7 @@ func TestCreator(t *testing.T) {
 		libvirt               *stubLibvirtRunner
 		provider              cloudprovider.Provider
 		config                *config.Config
+		policyPatcher         *stubPolicyPatcher
 		wantErr               bool
 		wantRollback          bool // Use only together with stubClients.
 		wantTerraformRollback bool // When libvirt fails, don't call into Terraform.
@@ -53,6 +54,35 @@ func TestCreator(t *testing.T) {
 			wantRollback:          true,
 			wantTerraformRollback: true,
 		},
+		"azure": {
+			tfClient:      &stubTerraformClient{ip: ip},
+			provider:      cloudprovider.Azure,
+			config:        config.Default(),
+			policyPatcher: &stubPolicyPatcher{},
+		},
+		"azure new policy patch error": {
+			tfClient:      &stubTerraformClient{ip: ip},
+			provider:      cloudprovider.Azure,
+			config:        config.Default(),
+			policyPatcher: &stubPolicyPatcher{someErr},
+			wantErr:       true,
+		},
+		"azure newTerraformClient error": {
+			newTfClientErr: someErr,
+			provider:       cloudprovider.Azure,
+			config:         config.Default(),
+			policyPatcher:  &stubPolicyPatcher{},
+			wantErr:        true,
+		},
+		"azure create cluster error": {
+			tfClient:              &stubTerraformClient{createClusterErr: someErr},
+			provider:              cloudprovider.Azure,
+			config:                config.Default(),
+			policyPatcher:         &stubPolicyPatcher{},
+			wantErr:               true,
+			wantRollback:          true,
+			wantTerraformRollback: true,
+		},
 		"qemu": {
 			tfClient: &stubTerraformClient{ip: ip},
 			libvirt:  &stubLibvirtRunner{},
@@ -112,6 +142,7 @@ func TestCreator(t *testing.T) {
 						destination: "some-destination",
 					}
 				},
+				policyPatcher: tc.policyPatcher,
 			}
 
 			idFile, err := creator.Create(context.Background(), tc.provider, tc.config, "type", 2, 3)
@@ -137,6 +168,14 @@ func TestCreator(t *testing.T) {
 	}
 }
 
+type stubPolicyPatcher struct {
+	patchErr error
+}
+
+func (s stubPolicyPatcher) Patch(ctx context.Context, attestationURL string) error {
+	return s.patchErr
+}
+
 func TestNormalizeAzureURIs(t *testing.T) {
 	testCases := map[string]struct {
 		in   terraform.AzureClusterVariables

cli/internal/clusterid/id.go

@@ -24,4 +24,7 @@ type File struct {
 	IP string `json:"ip,omitempty"`
 	// InitSecret is the secret the first Bootstrapper uses to verify the user.
 	InitSecret []byte `json:"initsecret,omitempty"`
+	// AttestationURL is the URL of the attestation service.
+	// It is only set if the cluster is created on Azure.
+	AttestationURL string `json:"attestationURL,omitempty"`
 }

cli/internal/terraform/terraform.go

@@ -128,10 +128,18 @@ func (c *Client) CreateCluster(ctx context.Context) (CreateOutput, error) {
 		return CreateOutput{}, errors.New("invalid type in uid output: not a string")
 	}
 
+	var attestationURL string
+	if attestationURLOutput, ok := tfState.Values.Outputs["attestationURL"]; ok {
+		if attestationURLString, ok := attestationURLOutput.Value.(string); ok {
+			attestationURL = attestationURLString
+		}
+	}
+
 	return CreateOutput{
-		IP:     ip,
-		Secret: secret,
-		UID:    uid,
+		IP:             ip,
+		Secret:         secret,
+		UID:            uid,
+		AttestationURL: attestationURL,
 	}, nil
 }
 
@@ -140,6 +148,9 @@ type CreateOutput struct {
 	IP     string
 	Secret string
 	UID    string
+	// AttestationURL is the URL of the attestation provider.
+	// It is only set if the cluster is created on Azure.
+	AttestationURL string
 }
 
 // IAMOutput contains the output information of the Terraform IAM operations.

cli/internal/terraform/terraform/azure/main.tf

@@ -41,6 +41,13 @@ resource "random_password" "initSecret" {
   override_special = "_%@"
 }
 
+resource "azurerm_attestation_provider" "attestation_provider" {
+  count               = var.create_maa ? 1 : 0
+  name                = format("%sap", var.name)
+  resource_group_name = var.resource_group
+  location            = var.location
+}
+
 resource "azurerm_application_insights" "insights" {
   name                = local.name
   location            = var.location
@@ -192,16 +199,21 @@ resource "azurerm_network_security_group" "security_group" {
 module "scale_set_control_plane" {
   source = "./modules/scale_set"
 
-  name                      = "${local.name}-control-plane"
-  instance_count            = var.control_plane_count
-  state_disk_size           = var.state_disk_size
-  state_disk_type           = var.state_disk_type
-  resource_group            = var.resource_group
-  location                  = var.location
-  instance_type             = var.instance_type
-  confidential_vm           = var.confidential_vm
-  secure_boot               = var.secure_boot
-  tags                      = merge(local.tags, { constellation-role = "control-plane" }, { constellation-init-secret-hash = local.initSecretHash })
+  name            = "${local.name}-control-plane"
+  instance_count  = var.control_plane_count
+  state_disk_size = var.state_disk_size
+  state_disk_type = var.state_disk_type
+  resource_group  = var.resource_group
+  location        = var.location
+  instance_type   = var.instance_type
+  confidential_vm = var.confidential_vm
+  secure_boot     = var.secure_boot
+  tags = merge(
+    local.tags,
+    { constellation-role = "control-plane" },
+    { constellation-init-secret-hash = local.initSecretHash },
+    { constellation-maa-url = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" },
+  )
   image_id                  = var.image_id
   user_assigned_identity    = var.user_assigned_identity
   network_security_group_id = azurerm_network_security_group.security_group.id
@@ -215,16 +227,21 @@ module "scale_set_control_plane" {
 module "scale_set_worker" {
   source = "./modules/scale_set"
 
-  name                      = "${local.name}-worker"
-  instance_count            = var.worker_count
-  state_disk_size           = var.state_disk_size
-  state_disk_type           = var.state_disk_type
-  resource_group            = var.resource_group
-  location                  = var.location
-  instance_type             = var.instance_type
-  confidential_vm           = var.confidential_vm
-  secure_boot               = var.secure_boot
-  tags                      = merge(local.tags, { constellation-role = "worker" }, { constellation-init-secret-hash = local.initSecretHash })
+  name            = "${local.name}-worker"
+  instance_count  = var.worker_count
+  state_disk_size = var.state_disk_size
+  state_disk_type = var.state_disk_type
+  resource_group  = var.resource_group
+  location        = var.location
+  instance_type   = var.instance_type
+  confidential_vm = var.confidential_vm
+  secure_boot     = var.secure_boot
+  tags = merge(
+    local.tags,
+    { constellation-role = "worker" },
+    { constellation-init-secret-hash = local.initSecretHash },
+    { constellation-maa-url = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" },
+  )
   image_id                  = var.image_id
   user_assigned_identity    = var.user_assigned_identity
   network_security_group_id = azurerm_network_security_group.security_group.id

cli/internal/terraform/terraform/azure/outputs.tf

@@ -10,3 +10,7 @@ output "initSecret" {
   value     = random_password.initSecret.result
   sensitive = true
 }
+
+output "attestationURL" {
+  value = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : ""
+}

cli/internal/terraform/terraform/azure/variables.tf

@@ -63,6 +63,12 @@ variable "secure_boot" {
   description = "Whether to deploy the cluster nodes with secure boot."
 }
 
+variable "create_maa" {
+  type        = bool
+  default     = false
+  description = "Whether to create a Microsoft Azure attestation provider."
+}
+
 variable "debug" {
   type        = bool
   default     = false

cli/internal/terraform/terraform_test.go

@@ -203,7 +203,7 @@ func TestPrepareIAM(t *testing.T) {
 
 func TestCreateCluster(t *testing.T) {
 	someErr := errors.New("failed")
-	newTestState := func() *tfjson.State {
+	newQEMUState := func() *tfjson.State {
 		workingState := tfjson.State{
 			Values: &tfjson.StateValues{
 				Outputs: map[string]*tfjson.StateOutput{
@@ -221,6 +221,27 @@ func TestCreateCluster(t *testing.T) {
 		}
 		return &workingState
 	}
+	newAzureState := func() *tfjson.State {
+		workingState := tfjson.State{
+			Values: &tfjson.StateValues{
+				Outputs: map[string]*tfjson.StateOutput{
+					"ip": {
+						Value: "192.0.2.100",
+					},
+					"initSecret": {
+						Value: "initSecret",
+					},
+					"uid": {
+						Value: "12345abc",
+					},
+					"attestationURL": {
+						Value: "https://12345.neu.attest.azure.net",
+					},
+				},
+			},
+		}
+		return &workingState
+	}
 	qemuVars := &QEMUVariables{
 		CommonVariables: CommonVariables{
 			Name:               "name",
@@ -241,13 +262,17 @@ func TestCreateCluster(t *testing.T) {
 		vars     Variables
 		tf       *stubTerraform
 		fs       afero.Fs
-		wantErr  bool
+		// expectedAttestationURL is the expected attestation URL to be returned by
+		// the Terraform client. It is declared in the test case because it is
+		// provider-specific.
+		expectedAttestationURL string
+		wantErr                bool
 	}{
 		"works": {
 			pathBase: "terraform",
 			provider: cloudprovider.QEMU,
 			vars:     qemuVars,
-			tf:       &stubTerraform{showState: newTestState()},
+			tf:       &stubTerraform{showState: newQEMUState()},
 			fs:       afero.NewMemMapFs(),
 		},
 		"init fails": {
@@ -330,6 +355,42 @@ func TestCreateCluster(t *testing.T) {
 			fs:      afero.NewMemMapFs(),
 			wantErr: true,
 		},
+		"working attestation url": {
+			pathBase:               "terraform",
+			provider:               cloudprovider.Azure,
+			vars:                   qemuVars, // works for mocking azure vars
+			tf:                     &stubTerraform{showState: newAzureState()},
+			fs:                     afero.NewMemMapFs(),
+			expectedAttestationURL: "https://12345.neu.attest.azure.net",
+		},
+		"no attestation url": {
+			pathBase: "terraform",
+			provider: cloudprovider.Azure,
+			vars:     qemuVars, // works for mocking azure vars
+			tf: &stubTerraform{
+				showState: &tfjson.State{
+					Values: &tfjson.StateValues{
+						Outputs: map[string]*tfjson.StateOutput{},
+					},
+				},
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
+		"attestation url has wrong type": {
+			pathBase: "terraform",
+			provider: cloudprovider.Azure,
+			vars:     qemuVars, // works for mocking azure vars
+			tf: &stubTerraform{
+				showState: &tfjson.State{
+					Values: &tfjson.StateValues{
+						Outputs: map[string]*tfjson.StateOutput{"attestationURL": {Value: 42}},
+					},
+				},
+			},
+			fs:      afero.NewMemMapFs(),
+			wantErr: true,
+		},
 	}
 
 	for name, tc := range testCases {
@@ -355,6 +416,7 @@ func TestCreateCluster(t *testing.T) {
 			assert.Equal("192.0.2.100", tfOutput.IP)
 			assert.Equal("initSecret", tfOutput.Secret)
 			assert.Equal("12345abc", tfOutput.UID)
+			assert.Equal(tc.expectedAttestationURL, tfOutput.AttestationURL)
 		})
 	}
 }

cli/internal/terraform/variables.go

@@ -175,6 +175,8 @@ type AzureClusterVariables struct {
 	ConfidentialVM bool
 	// SecureBoot sets the VM to use secure boot.
 	SecureBoot bool
+	// CreateMAA sets whether a Microsoft Azure attestation provider should be created.
+	CreateMAA bool
 	// Debug is true if debug mode is enabled.
 	Debug bool
 }
@@ -191,6 +193,7 @@ func (v *AzureClusterVariables) String() string {
 	writeLinef(b, "image_id = %q", v.ImageID)
 	writeLinef(b, "confidential_vm = %t", v.ConfidentialVM)
 	writeLinef(b, "secure_boot = %t", v.SecureBoot)
+	writeLinef(b, "create_maa = %t", v.CreateMAA)
 	writeLinef(b, "debug = %t", v.Debug)
 
 	return b.String()