terraform: azure node groups (#1955)

* init

* migration working

* make tf variables with default value optional in go through ptr type

* fix CI build

* pr feedback

* add azure targets tf

* skip migration for empty targets

* make instance_count optional

* change role naming to dashed + add validation

* make node_group.zones optional

* Update cli/internal/terraform/terraform/azure/main.tf

Co-authored-by: Malte Poll <1780588+malt3@users.noreply.github.com>

* malte feedback

---------

Co-authored-by: Malte Poll <1780588+malt3@users.noreply.github.com>

cli/internal/cloudcmd/BUILD.bazel

@@ -29,6 +29,7 @@ go_library(
         "//internal/config",
         "//internal/constants",
         "//internal/imagefetcher",
+        "//internal/role",
         "@com_github_azure_azure_sdk_for_go//profiles/latest/attestation/attestation",
         "@com_github_azure_azure_sdk_for_go_sdk_azcore//policy",
         "@com_github_azure_azure_sdk_for_go_sdk_azidentity//:azidentity",

cli/internal/cloudcmd/create.go

@@ -26,6 +26,7 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/config"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/imagefetcher"
+	"github.com/edgelesssys/constellation/v2/internal/role"
 )
 
 // Creator creates cloud resources.
@@ -214,25 +215,35 @@ func (c *Creator) createGCP(ctx context.Context, cl terraformClient, opts Create
 
 func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts CreateOptions) (idFile clusterid.File, retErr error) {
 	vars := terraform.AzureClusterVariables{
-		CommonVariables: terraform.CommonVariables{
 		Name: opts.Config.Name,
-			CountControlPlanes: opts.ControlPlaneCount,
+		NodeGroups: map[string]terraform.AzureNodeGroup{
-			CountWorkers:       opts.WorkerCount,
+			"control_plane_default": {
-			StateDiskSizeGB:    opts.Config.StateDiskSizeGB,
+				Role:          role.ControlPlane.TFString(),
+				InstanceCount: toPtr(opts.ControlPlaneCount),
+				InstanceType:  opts.InsType,
+				DiskSizeGB:    opts.Config.StateDiskSizeGB,
+				DiskType:      opts.Config.Provider.Azure.StateDiskType,
+				Zones:         nil, // TODO(elchead): support zones AB#3225
+			},
+			"worker_default": {
+				Role:          role.Worker.TFString(),
+				InstanceCount: toPtr(opts.WorkerCount),
+				InstanceType:  opts.InsType,
+				DiskSizeGB:    opts.Config.StateDiskSizeGB,
+				DiskType:      opts.Config.Provider.Azure.StateDiskType,
+				Zones:         nil,
+			},
 		},
 		Location:             opts.Config.Provider.Azure.Location,
-		ResourceGroup:        opts.Config.Provider.Azure.ResourceGroup,
-		UserAssignedIdentity: opts.Config.Provider.Azure.UserAssignedIdentity,
-		InstanceType:         opts.InsType,
-		StateDiskType:        opts.Config.Provider.Azure.StateDiskType,
 		ImageID:              opts.image,
-		SecureBoot:           *opts.Config.Provider.Azure.SecureBoot,
+		CreateMAA:            toPtr(opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
-		CreateMAA:            opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}),
+		Debug:                toPtr(opts.Config.IsDebugCluster()),
-		Debug:                opts.Config.IsDebugCluster(),
+		ConfidentialVM:       toPtr(opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
+		SecureBoot:           opts.Config.Provider.Azure.SecureBoot,
+		UserAssignedIdentity: opts.Config.Provider.Azure.UserAssignedIdentity,
+		ResourceGroup:        opts.Config.Provider.Azure.ResourceGroup,
 	}
 
-	vars.ConfidentialVM = opts.Config.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})
-
 	vars = normalizeAzureURIs(vars)
 
 	if err := cl.PrepareWorkspace(path.Join("terraform", strings.ToLower(cloudprovider.Azure.String())), &vars); err != nil {
@@ -245,7 +256,7 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea
 		return clusterid.File{}, err
 	}
 
-	if vars.CreateMAA {
+	if vars.CreateMAA != nil && *vars.CreateMAA {
 		// Patch the attestation policy to allow the cluster to boot while having secure boot disabled.
 		if err := c.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
 			return clusterid.File{}, err
@@ -442,3 +453,7 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
 		UID:           tfOutput.UID,
 	}, nil
 }
+
+func toPtr[T any](v T) *T {
+	return &v
+}

cli/internal/cmd/upgradeapply.go

@@ -156,6 +156,10 @@ func (u *upgradeApplyCmd) migrateTerraform(cmd *cobra.Command, file file.Handler
 	if err != nil {
 		return fmt.Errorf("parsing upgrade variables: %w", err)
 	}
+	if len(targets) == 0 {
+		u.log.Debugf("No targets specified. Skipping Terraform migration")
+		return nil
+	}
 	u.log.Debugf("Using migration targets:\n%v", targets)
 	u.log.Debugf("Using Terraform variables:\n%v", vars)
 
@@ -240,7 +244,7 @@ func parseTerraformUpgradeVars(cmd *cobra.Command, conf *config.Config, fetcher
 		}
 		return targets, vars, nil
 	case cloudprovider.Azure:
-		targets := []string{"azurerm_attestation_provider.attestation_provider"}
+		targets := []string{"azurerm_attestation_provider.attestation_provider", "module.scale_set_group", "module.scale_set_control_plane", "module.scale_set_worker"}
 
 		// Azure Terraform provider is very strict about it's casing
 		imageRef = strings.Replace(imageRef, "CommunityGalleries", "communityGalleries", 1)
@@ -248,16 +252,28 @@ func parseTerraformUpgradeVars(cmd *cobra.Command, conf *config.Config, fetcher
 		imageRef = strings.Replace(imageRef, "Versions", "versions", 1)
 
 		vars := &terraform.AzureClusterVariables{
-			CommonVariables:      commonVariables,
+			Name:                 conf.Name,
-			Location:             conf.Provider.Azure.Location,
 			ResourceGroup:        conf.Provider.Azure.ResourceGroup,
 			UserAssignedIdentity: conf.Provider.Azure.UserAssignedIdentity,
-			InstanceType:         conf.Provider.Azure.InstanceType,
-			StateDiskType:        conf.Provider.Azure.StateDiskType,
 			ImageID:              imageRef,
-			SecureBoot:           *conf.Provider.Azure.SecureBoot,
+			NodeGroups: map[string]terraform.AzureNodeGroup{
-			CreateMAA:            conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{}),
+				"control_plane_default": {
-			Debug:                conf.IsDebugCluster(),
+					Role:         "control-plane",
+					InstanceType: conf.Provider.Azure.InstanceType,
+					DiskSizeGB:   conf.StateDiskSizeGB,
+					DiskType:     conf.Provider.Azure.StateDiskType,
+				},
+				"worker_default": {
+					Role:         "worker",
+					InstanceType: conf.Provider.Azure.InstanceType,
+					DiskSizeGB:   conf.StateDiskSizeGB,
+					DiskType:     conf.Provider.Azure.StateDiskType,
+				},
+			},
+			Location:   conf.Provider.Azure.Location,
+			SecureBoot: conf.Provider.Azure.SecureBoot,
+			CreateMAA:  toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
+			Debug:      toPtr(conf.IsDebugCluster()),
 		}
 		return targets, vars, nil
 	case cloudprovider.GCP:
@@ -427,3 +443,7 @@ type cloudUpgrader interface {
 	CheckTerraformMigrations(fileHandler file.Handler) error
 	CleanUpTerraformMigrations(fileHandler file.Handler) error
 }
+
+func toPtr[T any](v T) *T {
+	return &v
+}

cli/internal/terraform/BUILD.bazel

@@ -103,6 +103,7 @@ go_test(
         "//internal/cloud/cloudprovider",
         "//internal/constants",
         "//internal/file",
+        "@com_github_azure_azure_sdk_for_go_sdk_azcore//to",
         "@com_github_hashicorp_terraform_exec//tfexec",
         "@com_github_hashicorp_terraform_json//:terraform-json",
         "@com_github_spf13_afero//:afero",

cli/internal/terraform/terraform/azure/main.tf

@@ -221,58 +221,46 @@ resource "azurerm_network_security_group" "security_group" {
   }
 }
 
-module "scale_set_control_plane" {
+module "scale_set_group" {
   source          = "./modules/scale_set"
-
+  for_each        = var.node_groups
-  name            = "${local.name}-control-plane"
+  base_name       = local.name
-  instance_count  = var.control_plane_count
+  node_group_name = each.key
-  state_disk_size = var.state_disk_size
+  role            = each.value.role
-  state_disk_type = var.state_disk_type
+  zones           = each.value.zones
-  resource_group  = var.resource_group
-  location        = var.location
-  instance_type   = var.instance_type
-  confidential_vm = var.confidential_vm
-  secure_boot     = var.secure_boot
   tags = merge(
     local.tags,
-    { constellation-role = "control-plane" },
     { constellation-init-secret-hash = local.initSecretHash },
     { constellation-maa-url = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" },
   )
-  image_id                  = var.image_id
+
+  instance_count            = each.value.instance_count
+  state_disk_size           = each.value.disk_size
+  state_disk_type           = each.value.disk_type
+  location                  = var.location
+  instance_type             = each.value.instance_type
+  confidential_vm           = var.confidential_vm
+  secure_boot               = var.secure_boot
+  resource_group            = var.resource_group
   user_assigned_identity    = var.user_assigned_identity
+  image_id                  = var.image_id
   network_security_group_id = azurerm_network_security_group.security_group.id
   subnet_id                 = azurerm_subnet.node_subnet.id
-  backend_address_pool_ids = [
+  backend_address_pool_ids = each.value.role == "control-plane" ? [
     azurerm_lb_backend_address_pool.all.id,
     module.loadbalancer_backend_control_plane.backendpool_id
-  ]
+    ] : [
-}
-
-module "scale_set_worker" {
-  source = "./modules/scale_set"
-
-  name            = "${local.name}-worker"
-  instance_count  = var.worker_count
-  state_disk_size = var.state_disk_size
-  state_disk_type = var.state_disk_type
-  resource_group  = var.resource_group
-  location        = var.location
-  instance_type   = var.instance_type
-  confidential_vm = var.confidential_vm
-  secure_boot     = var.secure_boot
-  tags = merge(
-    local.tags,
-    { constellation-role = "worker" },
-    { constellation-init-secret-hash = local.initSecretHash },
-    { constellation-maa-url = var.create_maa ? azurerm_attestation_provider.attestation_provider[0].attestation_uri : "" },
-  )
-  image_id                  = var.image_id
-  user_assigned_identity    = var.user_assigned_identity
-  network_security_group_id = azurerm_network_security_group.security_group.id
-  subnet_id                 = azurerm_subnet.node_subnet.id
-  backend_address_pool_ids = [
     azurerm_lb_backend_address_pool.all.id,
-    module.loadbalancer_backend_worker.backendpool_id,
+    module.loadbalancer_backend_worker.backendpool_id
   ]
 }
+
+moved {
+  from = module.scale_set_control_plane
+  to   = module.scale_set_group["control_plane_default"]
+}
+
+moved {
+  from = module.scale_set_worker
+  to   = module.scale_set_group["worker_default"]
+}

cli/internal/terraform/terraform/azure/modules/scale_set/main.tf

@@ -11,6 +11,19 @@ terraform {
   }
 }
 
+locals {
+  tags = merge(
+    var.tags,
+    { constellation-role = var.role },
+    { constellation-node-group = var.node_group_name },
+  )
+  group_uid = random_id.uid.hex
+  name      = "${var.base_name}-${var.role}${local.group_uid}"
+}
+
+resource "random_id" "uid" {
+  byte_length = 4
+}
 resource "random_password" "password" {
   length      = 16
   min_lower   = 1
@@ -20,7 +33,7 @@ resource "random_password" "password" {
 }
 
 resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
-  name                            = var.name
+  name                            = local.name
   resource_group_name             = var.resource_group
   location                        = var.location
   sku                             = var.instance_type
@@ -34,8 +47,8 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
   upgrade_mode                    = "Manual"
   secure_boot_enabled             = var.secure_boot
   source_image_id                 = var.image_id
-  tags                            = var.tags
+  tags                            = local.tags
-
+  zones                           = var.zones
   identity {
     type         = "UserAssigned"
     identity_ids = [var.user_assigned_identity]
@@ -81,6 +94,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
 
   lifecycle {
     ignore_changes = [
+      name,            # required. Allow legacy scale sets to keep their old names
       instances,       # required. autoscaling modifies the instance count externally
       source_image_id, # required. update procedure modifies the image id externally
     ]

cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf

@@ -1,7 +1,31 @@
-variable "name" {
+variable "base_name" {
   type        = string
-  default     = "constell"
+  description = "Base name of the instance group."
-  description = "Base name of the cluster."
+}
+
+variable "node_group_name" {
+  type        = string
+  description = "Constellation name for the node group (used for configuration and CSP-independent naming)."
+}
+
+variable "role" {
+  type        = string
+  description = "The role of the instance group."
+  validation {
+    condition     = contains(["control-plane", "worker"], var.role)
+    error_message = "The role has to be 'control-plane' or 'worker'."
+  }
+}
+
+variable "tags" {
+  type        = map(string)
+  description = "Tags to include in the scale_set."
+}
+
+variable "zones" {
+  type        = list(string)
+  description = "List of availability zones."
+  default     = null
 }
 
 variable "instance_count" {
@@ -61,11 +85,6 @@ variable "subnet_id" {
   description = "The ID of the subnet to use for the scale set."
 }
 
-variable "tags" {
-  type        = map(string)
-  description = "The tags to add to the scale set."
-}
-
 variable "confidential_vm" {
   type        = bool
   default     = true

cli/internal/terraform/terraform/azure/variables.tf

@@ -1,28 +1,22 @@
 variable "name" {
   type        = string
-  default     = "constell"
   description = "Base name of the cluster."
 }
 
-variable "control_plane_count" {
+variable "node_groups" {
-  type        = number
+  type = map(object({
-  description = "The number of control plane nodes to deploy."
+    role           = string
+    instance_count = optional(number)
+    instance_type  = string
+    disk_size      = number
+    disk_type      = string
+    zones          = optional(list(string))
+  }))
+  description = "A map of node group names to node group configurations."
+  validation {
+    condition     = can([for group in var.node_groups : group.role == "control-plane" || group.role == "worker"])
+    error_message = "The role has to be 'control-plane' or 'worker'."
   }
-
-variable "worker_count" {
-  type        = number
-  description = "The number of worker nodes to deploy."
-}
-
-variable "state_disk_size" {
-  type        = number
-  default     = 30
-  description = "The size of the state disk in GB."
-}
-
-variable "resource_group" {
-  type        = string
-  description = "The name of the Azure resource group to create the Constellation cluster in."
 }
 
 variable "location" {
@@ -30,27 +24,23 @@ variable "location" {
   description = "The Azure location to deploy the cluster in."
 }
 
-variable "user_assigned_identity" {
-  type        = string
-  description = "The name of the user assigned identity to attache to the nodes of the cluster."
-}
-
-variable "instance_type" {
-  type        = string
-  description = "The Azure instance type to deploy."
-}
-
-variable "state_disk_type" {
-  type        = string
-  default     = "Premium_LRS"
-  description = "The type of the state disk."
-}
-
 variable "image_id" {
   type        = string
   description = "The image to use for the cluster nodes."
 }
 
+variable "create_maa" {
+  type        = bool
+  default     = false
+  description = "Whether to create a Microsoft Azure attestation provider."
+}
+
+variable "debug" {
+  type        = bool
+  default     = false
+  description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
+}
+
 variable "confidential_vm" {
   type        = bool
   default     = true
@@ -63,14 +53,11 @@ variable "secure_boot" {
   description = "Whether to deploy the cluster nodes with secure boot."
 }
 
-variable "create_maa" {
+variable "resource_group" {
-  type        = bool
+  type        = string
-  default     = false
+  description = "The name of the Azure resource group to create the Constellation cluster in."
-  description = "Whether to create a Microsoft Azure attestation provider."
 }
-
+variable "user_assigned_identity" {
-variable "debug" {
+  type        = string
-  type        = bool
+  description = "The name of the user assigned identity to attache to the nodes of the cluster."
-  default     = false
-  description = "Enable debug mode. This opens up a debugd port that can be used to deploy a custom bootstrapper."
 }

cli/internal/terraform/variables.go

@@ -162,47 +162,45 @@ func (v *GCPIAMVariables) String() string {
 
 // AzureClusterVariables is user configuration for creating a cluster with Terraform on Azure.
 type AzureClusterVariables struct {
-	// CommonVariables contains common variables.
+	// Name of the cluster.
-	CommonVariables
+	Name string `hcl:"name" cty:"name"`
-
-	// ResourceGroup is the name of the Azure resource group to use.
-	ResourceGroup string
-	// Location is the Azure location to use.
-	Location string
-	// UserAssignedIdentity is the name of the Azure user-assigned identity to use.
-	UserAssignedIdentity string
-	// InstanceType is the Azure instance type to use.
-	InstanceType string
-	// StateDiskType is the Azure disk type to use for the state disk.
-	StateDiskType string
 	// ImageID is the ID of the Azure image to use.
-	ImageID string
+	ImageID string `hcl:"image_id" cty:"image_id"`
-	// ConfidentialVM sets the VM to be confidential.
-	ConfidentialVM bool
-	// SecureBoot sets the VM to use secure boot.
-	SecureBoot bool
 	// CreateMAA sets whether a Microsoft Azure attestation provider should be created.
-	CreateMAA bool
+	CreateMAA *bool `hcl:"create_maa" cty:"create_maa"`
 	// Debug is true if debug mode is enabled.
-	Debug bool
+	Debug *bool `hcl:"debug" cty:"debug"`
+	// ResourceGroup is the name of the Azure resource group to use.
+	ResourceGroup string `hcl:"resource_group" cty:"resource_group"`
+	// Location is the Azure location to use.
+	Location string `hcl:"location" cty:"location"`
+	// UserAssignedIdentity is the name of the Azure user-assigned identity to use.
+	UserAssignedIdentity string `hcl:"user_assigned_identity" cty:"user_assigned_identity"`
+	// ConfidentialVM sets the VM to be confidential.
+	ConfidentialVM *bool `hcl:"confidential_vm" cty:"confidential_vm"`
+	// SecureBoot sets the VM to use secure boot.
+	SecureBoot *bool `hcl:"secure_boot" cty:"secure_boot"`
+	// NodeGroups is a map of node groups to create.
+	NodeGroups map[string]AzureNodeGroup `hcl:"node_groups" cty:"node_groups"`
 }
 
 // String returns a string representation of the variables, formatted as Terraform variables.
 func (v *AzureClusterVariables) String() string {
-	b := &strings.Builder{}
+	f := hclwrite.NewEmptyFile()
-	b.WriteString(v.CommonVariables.String())
+	gohcl.EncodeIntoBody(v, f.Body())
-	writeLinef(b, "resource_group = %q", v.ResourceGroup)
+	return string(f.Bytes())
-	writeLinef(b, "location = %q", v.Location)
+}
-	writeLinef(b, "user_assigned_identity = %q", v.UserAssignedIdentity)
-	writeLinef(b, "instance_type = %q", v.InstanceType)
-	writeLinef(b, "state_disk_type = %q", v.StateDiskType)
-	writeLinef(b, "image_id = %q", v.ImageID)
-	writeLinef(b, "confidential_vm = %t", v.ConfidentialVM)
-	writeLinef(b, "secure_boot = %t", v.SecureBoot)
-	writeLinef(b, "create_maa = %t", v.CreateMAA)
-	writeLinef(b, "debug = %t", v.Debug)
 
-	return b.String()
+// AzureNodeGroup is a node group to create on Azure.
+type AzureNodeGroup struct {
+	// Role is the role of the node group.
+	Role string `hcl:"role" cty:"role"`
+	// InstanceCount is optional for upgrades.
+	InstanceCount *int      `hcl:"instance_count" cty:"instance_count"`
+	InstanceType  string    `hcl:"instance_type" cty:"instance_type"`
+	DiskSizeGB    int       `hcl:"disk_size" cty:"disk_size"`
+	DiskType      string    `hcl:"disk_type" cty:"disk_type"`
+	Zones         *[]string `hcl:"zones" cty:"zones"`
 }
 
 // AzureIAMVariables is user configuration for creating the IAM configuration with Terraform on Microsoft Azure.

cli/internal/terraform/variables_test.go

@@ -9,6 +9,7 @@ package terraform
 import (
 	"testing"
 
+	"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -142,39 +143,44 @@ service_account_id = "my-service-account"
 
 func TestAzureClusterVariables(t *testing.T) {
 	vars := AzureClusterVariables{
-		CommonVariables: CommonVariables{
 		Name: "cluster-name",
-			CountControlPlanes: 1,
+		NodeGroups: map[string]AzureNodeGroup{
-			CountWorkers:       2,
+			"control_plane_default": {
-			StateDiskSizeGB:    30,
+				Role:          "ControlPlane",
-		},
+				InstanceCount: to.Ptr(1),
-		ResourceGroup:        "my-resource-group",
-		Location:             "eu-central-1",
-		UserAssignedIdentity: "my-user-assigned-identity",
 				InstanceType:  "Standard_D2s_v3",
-		StateDiskType:        "StandardSSD_LRS",
+				DiskType:      "StandardSSD_LRS",
+				DiskSizeGB:    100,
+			},
+		},
+		ConfidentialVM:       to.Ptr(true),
+		ResourceGroup:        "my-resource-group",
+		UserAssignedIdentity: "my-user-assigned-identity",
 		ImageID:              "image-0123456789abcdef",
-		ConfidentialVM:       true,
+		CreateMAA:            to.Ptr(true),
-		SecureBoot:           false,
+		Debug:                to.Ptr(true),
-		CreateMAA:            true,
+		Location:             "eu-central-1",
-		Debug:                true,
 	}
 
 	// test that the variables are correctly rendered
 	want := `name                   = "cluster-name"
-control_plane_count = 1
+image_id               = "image-0123456789abcdef"
-worker_count = 2
+create_maa             = true
-state_disk_size = 30
+debug                  = true
 resource_group         = "my-resource-group"
 location               = "eu-central-1"
 user_assigned_identity = "my-user-assigned-identity"
-instance_type = "Standard_D2s_v3"
-state_disk_type = "StandardSSD_LRS"
-image_id = "image-0123456789abcdef"
 confidential_vm        = true
-secure_boot = false
+node_groups = {
-create_maa = true
+  control_plane_default = {
-debug = true
+    disk_size      = 100
+    disk_type      = "StandardSSD_LRS"
+    instance_count = 1
+    instance_type  = "Standard_D2s_v3"
+    role           = "ControlPlane"
+    zones          = null
+  }
+}
 `
 	got := vars.String()
 	assert.Equal(t, want, got)

internal/role/role.go

@@ -25,6 +25,18 @@ const (
 	Worker
 )
 
+// TFString returns the role as a string for Terraform.
+func (r Role) TFString() string {
+	switch r {
+	case ControlPlane:
+		return "control-plane"
+	case Worker:
+		return "worker"
+	default:
+		return "unknown"
+	}
+}
+
 // MarshalJSON marshals the Role to JSON string.
 func (r Role) MarshalJSON() ([]byte, error) {
 	return json.Marshal(r.String())