Add configurable node disk type (#317)

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

cli/internal/azure/client/compute.go

@@ -19,6 +19,7 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 		Count:                          input.CountWorkers,
 		InstanceType:                   input.InstanceType,
 		StateDiskSizeGB:                int32(input.StateDiskSizeGB),
+		StateDiskType:                  input.StateDiskType,
 		Image:                          input.Image,
 		UserAssingedIdentity:           input.UserAssingedIdentity,
 		LoadBalancerBackendAddressPool: azure.BackendAddressPoolWorkerName + "-" + c.uid,
@@ -37,6 +38,7 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 		Count:                          input.CountControlPlanes,
 		InstanceType:                   input.InstanceType,
 		StateDiskSizeGB:                int32(input.StateDiskSizeGB),
+		StateDiskType:                  input.StateDiskType,
 		Image:                          input.Image,
 		UserAssingedIdentity:           input.UserAssingedIdentity,
 		LoadBalancerBackendAddressPool: azure.BackendAddressPoolControlPlaneName + "-" + c.uid,
@@ -70,6 +72,7 @@ type CreateInstancesInput struct {
 	CountControlPlanes   int
 	InstanceType         string
 	StateDiskSizeGB      int
+	StateDiskType        string
 	Image                string
 	UserAssingedIdentity string
 }
@@ -175,6 +178,7 @@ func (c *Client) createScaleSet(ctx context.Context, input CreateScaleSetInput)
 		Location:                       c.location,
 		InstanceType:                   input.InstanceType,
 		StateDiskSizeGB:                input.StateDiskSizeGB,
+		StateDiskType:                  input.StateDiskType,
 		Count:                          int64(input.Count),
 		Username:                       "constellation",
 		SubnetID:                       c.subnetID,
@@ -254,6 +258,7 @@ type CreateScaleSetInput struct {
 	Count                          int
 	InstanceType                   string
 	StateDiskSizeGB                int32
+	StateDiskType                  string
 	Image                          string
 	UserAssingedIdentity           string
 	LoadBalancerBackendAddressPool string

cli/internal/azure/scaleset.go

@@ -17,6 +17,7 @@ type ScaleSet struct {
 	Location                       string
 	InstanceType                   string
 	StateDiskSizeGB                int32
+	StateDiskType                  string
 	Count                          int64
 	Username                       string
 	SubnetID                       string
@@ -62,6 +63,9 @@ func (s ScaleSet) Azure() armcompute.VirtualMachineScaleSet {
 							CreateOption: armcompute.DiskCreateOptionTypesEmpty.ToPtr(),
 							DiskSizeGB:   to.Int32Ptr(s.StateDiskSizeGB),
 							Lun:          to.Int32Ptr(0),
+							ManagedDisk: &armcompute.VirtualMachineScaleSetManagedDiskParameters{
+								StorageAccountType: (*armcompute.StorageAccountTypes)(to.StringPtr(s.StateDiskType)),
+							},
 						},
 					},
 				},

cli/internal/cloudcmd/create.go

@@ -126,6 +126,7 @@ func (c *Creator) createGCP(ctx context.Context, cl gcpclient, config *config.Co
 		ImageID:            config.Provider.GCP.Image,
 		InstanceType:       insType,
 		StateDiskSizeGB:    config.StateDiskSizeGB,
+		StateDiskType:      config.Provider.GCP.StateDiskType,
 		KubeEnv:            gcp.KubeEnv,
 	}
 	if err := cl.CreateInstances(ctx, createInput); err != nil {
@@ -167,6 +168,7 @@ func (c *Creator) createAzure(ctx context.Context, cl azureclient, config *confi
 		CountWorkers:         workerCount,
 		InstanceType:         insType,
 		StateDiskSizeGB:      config.StateDiskSizeGB,
+		StateDiskType:        config.Provider.Azure.StateDiskType,
 		Image:                config.Provider.Azure.Image,
 		UserAssingedIdentity: config.Provider.Azure.UserAssignedIdentity,
 	}

cli/internal/gcp/client/instances.go

@@ -33,6 +33,7 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 		ImageID:                      input.ImageID,
 		InstanceType:                 input.InstanceType,
 		StateDiskSizeGB:              int64(input.StateDiskSizeGB),
+		StateDiskType:                input.StateDiskType,
 		Role:                         role.Worker.String(),
 		KubeEnv:                      input.KubeEnv,
 		Project:                      c.project,
@@ -55,6 +56,7 @@ func (c *Client) CreateInstances(ctx context.Context, input CreateInstancesInput
 		ImageID:                      input.ImageID,
 		InstanceType:                 input.InstanceType,
 		StateDiskSizeGB:              int64(input.StateDiskSizeGB),
+		StateDiskType:                input.StateDiskType,
 		Role:                         role.ControlPlane.String(),
 		KubeEnv:                      input.KubeEnv,
 		Project:                      c.project,
@@ -295,6 +297,7 @@ type CreateInstancesInput struct {
 	ImageID            string
 	InstanceType       string
 	StateDiskSizeGB    int
+	StateDiskType      string
 	KubeEnv            string
 }
 
@@ -306,6 +309,7 @@ type insertInstanceTemplateInput struct {
 	ImageID                      string
 	InstanceType                 string
 	StateDiskSizeGB              int64
+	StateDiskType                string
 	Role                         string
 	KubeEnv                      string
 	Project                      string
@@ -337,6 +341,7 @@ func (i insertInstanceTemplateInput) insertInstanceTemplateRequest() *computepb.
 					{
 						InitializeParams: &computepb.AttachedDiskInitializeParams{
 							DiskSizeGb: proto.Int64(i.StateDiskSizeGB),
+							DiskType:   proto.String(i.StateDiskType),
 						},
 						AutoDelete: proto.Bool(true),
 						DeviceName: proto.String("state-disk"),

internal/config/config.go

@@ -129,6 +129,9 @@ type AzureConfig struct {
 	//   Machine image used to create Constellation nodes.
 	Image string `yaml:"image" validate:"required"`
 	// description: |
+	//   Type of a node's state disk. The type influences boot time and I/O performance. See: https://docs.microsoft.com/en-us/azure/virtual-machines/disks-types#disk-type-comparison
+	StateDiskType string `yaml:"stateDiskType" validate:"oneof=Premium_LRS Premium_ZRS Standard_LRS StandardSSD_LRS StandardSSD_ZRS"`
+	// description: |
 	//   Expected confidential VM measurements.
 	Measurements Measurements `yaml:"measurements"`
 	// description: |
@@ -151,6 +154,9 @@ type GCPConfig struct {
 	//   Machine image used to create Constellation nodes.
 	Image string `yaml:"image" validate:"required"`
 	// description: |
+	//   Type of a node's state disk. The type influences boot time and I/O performance. See: https://cloud.google.com/compute/docs/disks#disk-types
+	StateDiskType string `yaml:"stateDiskType" validate:"oneof=pd-standard pd-balanced pd-ssd"`
+	// description: |
 	//   Roles added to service account.
 	ServiceAccountRoles []string `yaml:"serviceAccountRoles"`
 	// description: |
@@ -209,6 +215,7 @@ func Default() *Config {
 				TenantID:             "adb650a8-5da3-4b15-b4b0-3daf65ff7626",
 				Location:             "North Europe",
 				Image:                "/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/CONSTELLATION-IMAGES/providers/Microsoft.Compute/galleries/Constellation/images/constellation-coreos/versions/0.0.1658932686",
+				StateDiskType:        "StandardSSD_LRS", // TODO: Replace with Premium_LRS when we replace the default VM size (Standard_D2a_v4) since the size does not support Premium_LRS
 				Measurements:         azurePCRs,
 				UserAssignedIdentity: "/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.ManagedIdentity/userAssignedIdentities/constellation-dev-identity",
 			},
@@ -224,6 +231,7 @@ func Default() *Config {
 					"roles/storage.admin",
 					"roles/iam.serviceAccountUser",
 				},
+				StateDiskType: "pd-ssd",
 				Measurements:  gcpPCRs,
 			},
 			QEMU: &QEMUConfig{

internal/config/config_doc.go

@@ -24,7 +24,7 @@ func init() {
 	ConfigDoc.Type = "Config"
 	ConfigDoc.Comments[encoder.LineComment] = "Config defines configuration used by CLI."
 	ConfigDoc.Description = "Config defines configuration used by CLI."
-	ConfigDoc.Fields = make([]encoder.Doc, 8)
+	ConfigDoc.Fields = make([]encoder.Doc, 9)
 	ConfigDoc.Fields[0].Name = "version"
 	ConfigDoc.Fields[0].Type = "string"
 	ConfigDoc.Fields[0].Note = ""
@@ -69,6 +69,11 @@ func init() {
 	ConfigDoc.Fields[7].Comments[encoder.LineComment] = "Create SSH users on Constellation nodes."
 
 	ConfigDoc.Fields[7].AddExample("", []UserKey{{Username: "Alice", PublicKey: "ssh-rsa AAAAB3NzaC...5QXHKW1rufgtJeSeJ8= alice@domain.com"}})
+	ConfigDoc.Fields[8].Name = "kubernetesVersion"
+	ConfigDoc.Fields[8].Type = "string"
+	ConfigDoc.Fields[8].Note = ""
+	ConfigDoc.Fields[8].Description = "Kubernetes version installed in the cluster."
+	ConfigDoc.Fields[8].Comments[encoder.LineComment] = "Kubernetes version installed in the cluster."
 
 	UserKeyDoc.Type = "UserKey"
 	UserKeyDoc.Comments[encoder.LineComment] = "UserKey describes a user that should be created with corresponding public SSH key."
@@ -163,7 +168,7 @@ func init() {
 			FieldName: "azure",
 		},
 	}
-	AzureConfigDoc.Fields = make([]encoder.Doc, 6)
+	AzureConfigDoc.Fields = make([]encoder.Doc, 7)
 	AzureConfigDoc.Fields[0].Name = "subscription"
 	AzureConfigDoc.Fields[0].Type = "string"
 	AzureConfigDoc.Fields[0].Note = ""
@@ -184,16 +189,21 @@ func init() {
 	AzureConfigDoc.Fields[3].Note = ""
 	AzureConfigDoc.Fields[3].Description = "Machine image used to create Constellation nodes."
 	AzureConfigDoc.Fields[3].Comments[encoder.LineComment] = "Machine image used to create Constellation nodes."
-	AzureConfigDoc.Fields[4].Name = "measurements"
+	AzureConfigDoc.Fields[4].Name = "stateDiskType"
-	AzureConfigDoc.Fields[4].Type = "Measurements"
+	AzureConfigDoc.Fields[4].Type = "string"
 	AzureConfigDoc.Fields[4].Note = ""
-	AzureConfigDoc.Fields[4].Description = "Expected confidential VM measurements."
+	AzureConfigDoc.Fields[4].Description = "Type of a node's state disk. The type influences boot time and I/O performance. See: https://docs.microsoft.com/en-us/azure/virtual-machines/disks-types#disk-type-comparison"
-	AzureConfigDoc.Fields[4].Comments[encoder.LineComment] = "Expected confidential VM measurements."
+	AzureConfigDoc.Fields[4].Comments[encoder.LineComment] = "Type of a node's state disk. The type influences boot time and I/O performance. See: https://docs.microsoft.com/en-us/azure/virtual-machines/disks-types#disk-type-comparison"
-	AzureConfigDoc.Fields[5].Name = "userAssignedIdentity"
+	AzureConfigDoc.Fields[5].Name = "measurements"
-	AzureConfigDoc.Fields[5].Type = "string"
+	AzureConfigDoc.Fields[5].Type = "Measurements"
 	AzureConfigDoc.Fields[5].Note = ""
-	AzureConfigDoc.Fields[5].Description = "Authorize spawned VMs to access Azure API. See: https://constellation-docs.edgeless.systems/6c320851-bdd2-41d5-bf10-e27427398692/#/getting-started/install?id=azure"
+	AzureConfigDoc.Fields[5].Description = "Expected confidential VM measurements."
-	AzureConfigDoc.Fields[5].Comments[encoder.LineComment] = "Authorize spawned VMs to access Azure API. See: https://constellation-docs.edgeless.systems/6c320851-bdd2-41d5-bf10-e27427398692/#/getting-started/install?id=azure"
+	AzureConfigDoc.Fields[5].Comments[encoder.LineComment] = "Expected confidential VM measurements."
+	AzureConfigDoc.Fields[6].Name = "userAssignedIdentity"
+	AzureConfigDoc.Fields[6].Type = "string"
+	AzureConfigDoc.Fields[6].Note = ""
+	AzureConfigDoc.Fields[6].Description = "Authorize spawned VMs to access Azure API. See: https://constellation-docs.edgeless.systems/6c320851-bdd2-41d5-bf10-e27427398692/#/getting-started/install?id=azure"
+	AzureConfigDoc.Fields[6].Comments[encoder.LineComment] = "Authorize spawned VMs to access Azure API. See: https://constellation-docs.edgeless.systems/6c320851-bdd2-41d5-bf10-e27427398692/#/getting-started/install?id=azure"
 
 	GCPConfigDoc.Type = "GCPConfig"
 	GCPConfigDoc.Comments[encoder.LineComment] = "GCPConfig are GCP specific configuration values used by the CLI."
@@ -204,7 +214,7 @@ func init() {
 			FieldName: "gcp",
 		},
 	}
-	GCPConfigDoc.Fields = make([]encoder.Doc, 6)
+	GCPConfigDoc.Fields = make([]encoder.Doc, 7)
 	GCPConfigDoc.Fields[0].Name = "project"
 	GCPConfigDoc.Fields[0].Type = "string"
 	GCPConfigDoc.Fields[0].Note = ""
@@ -225,16 +235,21 @@ func init() {
 	GCPConfigDoc.Fields[3].Note = ""
 	GCPConfigDoc.Fields[3].Description = "Machine image used to create Constellation nodes."
 	GCPConfigDoc.Fields[3].Comments[encoder.LineComment] = "Machine image used to create Constellation nodes."
-	GCPConfigDoc.Fields[4].Name = "serviceAccountRoles"
+	GCPConfigDoc.Fields[4].Name = "stateDiskType"
-	GCPConfigDoc.Fields[4].Type = "[]string"
+	GCPConfigDoc.Fields[4].Type = "string"
 	GCPConfigDoc.Fields[4].Note = ""
-	GCPConfigDoc.Fields[4].Description = "Roles added to service account."
+	GCPConfigDoc.Fields[4].Description = "Type of a node's state disk. The type influences boot time and I/O performance. See: https://cloud.google.com/compute/docs/disks#disk-types"
-	GCPConfigDoc.Fields[4].Comments[encoder.LineComment] = "Roles added to service account."
+	GCPConfigDoc.Fields[4].Comments[encoder.LineComment] = "Type of a node's state disk. The type influences boot time and I/O performance. See: https://cloud.google.com/compute/docs/disks#disk-types"
-	GCPConfigDoc.Fields[5].Name = "measurements"
+	GCPConfigDoc.Fields[5].Name = "serviceAccountRoles"
-	GCPConfigDoc.Fields[5].Type = "Measurements"
+	GCPConfigDoc.Fields[5].Type = "[]string"
 	GCPConfigDoc.Fields[5].Note = ""
-	GCPConfigDoc.Fields[5].Description = "Expected confidential VM measurements."
+	GCPConfigDoc.Fields[5].Description = "Roles added to service account."
-	GCPConfigDoc.Fields[5].Comments[encoder.LineComment] = "Expected confidential VM measurements."
+	GCPConfigDoc.Fields[5].Comments[encoder.LineComment] = "Roles added to service account."
+	GCPConfigDoc.Fields[6].Name = "measurements"
+	GCPConfigDoc.Fields[6].Type = "Measurements"
+	GCPConfigDoc.Fields[6].Note = ""
+	GCPConfigDoc.Fields[6].Description = "Expected confidential VM measurements."
+	GCPConfigDoc.Fields[6].Comments[encoder.LineComment] = "Expected confidential VM measurements."
 
 	QEMUConfigDoc.Type = "QEMUConfig"
 	QEMUConfigDoc.Comments[encoder.LineComment] = ""

internal/config/config_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"reflect"
 	"testing"
 
 	"github.com/edgelesssys/constellation/internal/cloud/cloudprovider"
@@ -273,7 +274,15 @@ func TestConfigRemoveProviderExcept(t *testing.T) {
 
 func TestConfigGeneratedDocsFresh(t *testing.T) {
 	assert := assert.New(t)
-	assert.Len(ConfigDoc.Fields, 8, "remember to re-generate config docs!")
+	updateMsg := "remember to re-generate config docs! 🔨"
+
+	assert.Len(ConfigDoc.Fields, reflect.ValueOf(Config{}).NumField(), updateMsg)
+	assert.Len(UserKeyDoc.Fields, reflect.ValueOf(UserKey{}).NumField(), updateMsg)
+	assert.Len(FirewallRuleDoc.Fields, reflect.ValueOf(FirewallRule{}).NumField(), updateMsg)
+	assert.Len(ProviderConfigDoc.Fields, reflect.ValueOf(ProviderConfig{}).NumField(), updateMsg)
+	assert.Len(AzureConfigDoc.Fields, reflect.ValueOf(AzureConfig{}).NumField(), updateMsg)
+	assert.Len(GCPConfigDoc.Fields, reflect.ValueOf(GCPConfig{}).NumField(), updateMsg)
+	assert.Len(QEMUConfigDoc.Fields, reflect.ValueOf(QEMUConfig{}).NumField(), updateMsg)
 }
 
 func TestConfig_UpdateMeasurements(t *testing.T) {