terraform: Azure Marketplace image support (#2651)

* terraform: add Azure marketplace variable

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* config: add Azure marketplace variable

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* cli: use Terraform variables from config

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* terraform: pass down marketplace variable

* image: pad Azure images to 1GiB

* terraform: add version attribute to marketplace image

* semver: allow versions to be exported without prefix

* cli: boolean var to use marketplace images

* config: remove dive key

* dev-docs: add instructions on how to use marketplace images

* terraform: fix unit test

* terraform: only fetch image for non-marketplace images

* mpimage: refactor image selection

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* [remove] increase minor version for image build

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* terraform: ignore changes to source_image_reference on upgrade

* operator: add support for parsing Azure marketplace images

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* upgrade: fix imagefetcher call

* docs: add info about azure marketplace

* image: ensure more than 1GiB in size

* image: test to pad to 2GiB

* version: change back to v2.14.0-pre

* image: GPT-conformant image size padding

* [remove] increase version

* mpimage: inline prefix func

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* ci: add marketplace image e2e test

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>

* [remove] register workflow

* ci: fix workflow name

* ci: only allow azure test

* cli: add marketplace image input to interface

* cli: fix argument passing

* version: roll back to v2.14.0

* ci: add force-flag support

* Update docs/docs/overview/license.md

* Update dev-docs/workflows/marketplace-images.md

Co-authored-by: Moritz Eckert <m1gh7ym0@gmail.com>

---------

Signed-off-by: Moritz Sanft <58110325+msanft@users.noreply.github.com>
Co-authored-by: Moritz Eckert <m1gh7ym0@gmail.com>
Co-authored-by: Thomas Tendyck <51411342+thomasten@users.noreply.github.com>
This commit is contained in:
Moritz Sanft 2023-12-08 14:40:31 +01:00 committed by GitHub
parent e113253262
commit c15e4efef6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
40 changed files with 607 additions and 34 deletions

View File

@ -53,6 +53,12 @@ inputs:
selfManagedInfra:
description: "Use self-managed infrastructure instead of infrastructure created by the Constellation CLI."
required: true
marketplaceImageVersion:
description: "Marketplace OS image version. Used instead of osImage."
required: false
force:
description: "Set the force-flag on apply to ignore version mismatches."
required: false
outputs:
kubeconfig:
@ -97,6 +103,13 @@ runs:
yq eval -i "(.image) = \"${imageInput}\"" constellation-conf.yaml
echo "image=${imageInput}" | tee -a "$GITHUB_OUTPUT"
- name: Set marketplace image flag (Azure)
if: inputs.marketplaceImageVersion != '' && inputs.cloudProvider == 'azure'
shell: bash
run: |
yq eval -i "(.provider.azure.useMarketplaceImage) = true" constellation-conf.yaml
yq eval -i "(.image) = \"${{ inputs.marketplaceImageVersion }}\"" constellation-conf.yaml
- name: Update measurements for non-stable images
if: inputs.fetchMeasurements
shell: bash
@ -163,11 +176,18 @@ runs:
kubernetesVersion: ${{ inputs.kubernetesVersion }}
selfManagedInfra: ${{ inputs.selfManagedInfra }}
- name: Set force flag
id: set-force-flag
if: inputs.force == 'true'
shell: bash
run: |
echo "flag=--force" | tee -a $GITHUB_OUTPUT
- name: Constellation init
id: constellation-init
shell: bash
run: |
constellation apply --skip-phases=infrastructure --debug
constellation apply --skip-phases=infrastructure --debug ${{ steps.set-force-flag.outputs.flag }}
echo "KUBECONFIG=$(pwd)/constellation-admin.conf" | tee -a $GITHUB_OUTPUT
- name: Wait for nodes to join and become ready

View File

@ -80,6 +80,12 @@ inputs:
description: "Access key for s3proxy"
s3SecretKey:
description: "Secret key for s3proxy"
marketplaceImageVersion:
description: "Marketplace OS image version. Used instead of osImage."
required: false
force:
description: "Set the force-flag on apply to ignore version mismatches."
required: false
outputs:
kubeconfig:
@ -266,6 +272,8 @@ runs:
internalLoadBalancer: ${{ inputs.internalLoadBalancer }}
test: ${{ inputs.test }}
selfManagedInfra: ${{ inputs.selfManagedInfra }}
marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
force: ${{ inputs.force }}
- name: Deploy log- and metrics-collection (Kubernetes)
id: deploy-logcollection

View File

@ -0,0 +1,87 @@
name: e2e test marketplace image
on:
workflow_dispatch:
inputs:
nodeCount:
description: "Number of nodes to use in the cluster. Given in format `<control-plane nodes>:<worker nodes>`."
default: "3:2"
type: string
cloudProvider:
description: "Which cloud provider to use."
type: choice
options:
- "azure"
default: "azure"
required: true
runner:
description: "Architecture of the runner that executes the CLI"
type: choice
options:
- "ubuntu-22.04"
- "macos-12"
default: "ubuntu-22.04"
test:
description: "The test to run."
type: choice
options:
- "sonobuoy quick"
- "sonobuoy full"
- "autoscaling"
- "lb"
- "perf-bench"
- "verify"
- "recover"
- "malicious join"
- "nop"
required: true
kubernetesVersion:
description: "Kubernetes version to create the cluster from."
default: "1.27"
required: true
cliVersion:
description: "Version of a released CLI to download. Leave empty to build the CLI from the checked out ref."
type: string
default: ""
required: false
marketplaceImageVersion:
description: "Marketplace image version to use in the cluster's nodes. Needs to be a release semver."
type: string
default: ""
required: true
machineType:
description: "Override VM machine type. Leave as 'default' or empty to use the default VM type for the selected cloud provider."
type: string
default: "default"
required: false
regionZone:
description: "Region or zone to create the cluster in. Leave empty for default region/zone."
type: string
git-ref:
description: "Git ref to checkout."
type: string
default: "head"
required: false
jobs:
e2e-test:
permissions:
id-token: write
checks: write
contents: read
packages: write
secrets: inherit
uses: ./.github/workflows/e2e-test.yml
with:
nodeCount: ${{ inputs.nodeCount }}
cloudProvider: ${{ inputs.cloudProvider }}
runner: ${{ inputs.runner }}
test: ${{ inputs.test }}
kubernetesVersion: ${{ inputs.kubernetesVersion }}
cliVersion: ${{ inputs.cliVersion }}
imageVersion: ${{ inputs.marketplaceImageVersion }}
machineType: ${{ inputs.machineType }}
regionZone: ${{ inputs.regionZone }}
git-ref: ${{ inputs.git-ref }}
marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
force: true

View File

@ -116,6 +116,12 @@ on:
description: "Use self-managed infrastructure."
type: boolean
default: false
marketplaceImageVersion:
description: "Marketplace image version to use."
type: string
force:
description: "Use the force-flag when applying to ignore version mismatches."
type: boolean
jobs:
split-nodeCount:
@ -238,6 +244,8 @@ jobs:
selfManagedInfra: ${{ inputs.selfManagedInfra }}
s3AccessKey: ${{ secrets.AWS_ACCESS_KEY_ID_S3PROXY }}
s3SecretKey: ${{ secrets.AWS_SECRET_ACCESS_KEY_S3PROXY }}
marketplaceImageVersion: ${{ inputs.marketplaceImageVersion }}
force: ${{ inputs.force }}
- name: Always terminate cluster
if: always()

View File

@ -30,6 +30,7 @@ go_library(
"//internal/file",
"//internal/imagefetcher",
"//internal/maa",
"//internal/mpimage",
"//internal/role",
"//internal/state",
],

View File

@ -131,7 +131,7 @@ func (a *Applier) terraformApplyVars(ctx context.Context, conf *config.Config) (
ctx,
conf.GetProvider(),
conf.GetAttestationConfig().GetVariant(),
conf.Image, conf.GetRegion(),
conf.Image, conf.GetRegion(), conf.UseMarketplaceImage(),
)
if err != nil {
return nil, fmt.Errorf("fetching image reference: %w", err)
@ -141,7 +141,7 @@ func (a *Applier) terraformApplyVars(ctx context.Context, conf *config.Config) (
case cloudprovider.AWS:
return awsTerraformVars(conf, imageRef), nil
case cloudprovider.Azure:
return azureTerraformVars(conf, imageRef), nil
return azureTerraformVars(conf, imageRef)
case cloudprovider.GCP:
return gcpTerraformVars(conf, imageRef), nil
case cloudprovider.OpenStack:

View File

@ -20,7 +20,7 @@ import (
type imageFetcher interface {
FetchReference(ctx context.Context,
provider cloudprovider.Provider, attestationVariant variant.Variant,
image, region string,
image, region string, useMarketplaceImage bool,
) (string, error)
}

View File

@ -124,7 +124,7 @@ type stubImageFetcher struct {
func (f *stubImageFetcher) FetchReference(_ context.Context,
_ cloudprovider.Provider, _ variant.Variant,
_, _ string,
_, _ string, _ bool,
) (string, error) {
return f.reference, f.fetchReferenceErr
}

View File

@ -24,6 +24,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/mpimage"
"github.com/edgelesssys/constellation/v2/internal/role"
)
@ -127,7 +128,7 @@ func normalizeAzureURIs(vars *terraform.AzureClusterVariables) *terraform.AzureC
// azureTerraformVars provides variables required to execute the Terraform scripts.
// It should be the only place to declare the Azure variables.
func azureTerraformVars(conf *config.Config, imageRef string) *terraform.AzureClusterVariables {
func azureTerraformVars(conf *config.Config, imageRef string) (*terraform.AzureClusterVariables, error) {
nodeGroups := make(map[string]terraform.AzureNodeGroup)
for groupName, group := range conf.NodeGroups {
zones := strings.Split(group.Zone, ",")
@ -147,7 +148,6 @@ func azureTerraformVars(conf *config.Config, imageRef string) *terraform.AzureCl
Name: conf.Name,
NodeGroups: nodeGroups,
Location: conf.Provider.Azure.Location,
ImageID: imageRef,
CreateMAA: toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
Debug: toPtr(conf.IsDebugCluster()),
ConfidentialVM: toPtr(conf.GetAttestationConfig().GetVariant().Equal(variant.AzureSEVSNP{})),
@ -158,8 +158,31 @@ func azureTerraformVars(conf *config.Config, imageRef string) *terraform.AzureCl
InternalLoadBalancer: conf.InternalLoadBalancer,
}
if conf.UseMarketplaceImage() {
image, err := mpimage.NewFromURI(imageRef)
if err != nil {
return nil, fmt.Errorf("parsing marketplace image URI: %w", err)
}
azureImage, ok := image.(mpimage.AzureMarketplaceImage)
if !ok {
return nil, fmt.Errorf("expected Azure marketplace image, got %T", image)
}
// If a marketplace image is used, only the marketplace reference is required.
vars.MarketplaceImage = terraform.AzureMarketplaceImageVariables{
Publisher: azureImage.Publisher,
Product: azureImage.Offer,
Name: azureImage.SKU,
Version: azureImage.Version,
}
} else {
// If not, we need to specify the exact CommunityGalleries/.. image reference.
vars.ImageID = imageRef
}
vars = normalizeAzureURIs(vars)
return vars
return vars, nil
}
func azureTerraformIAMVars(conf *config.Config, oldVars terraform.AzureIAMVariables) *terraform.AzureIAMVariables {

View File

@ -629,7 +629,7 @@ func (a *applyCmd) runNodeImageUpgrade(cmd *cobra.Command, conf *config.Config)
provider := conf.GetProvider()
attestationVariant := conf.GetAttestationConfig().GetVariant()
region := conf.GetRegion()
imageReference, err := a.imageFetcher.FetchReference(cmd.Context(), provider, attestationVariant, conf.Image, region)
imageReference, err := a.imageFetcher.FetchReference(cmd.Context(), provider, attestationVariant, conf.Image, region, conf.UseMarketplaceImage())
if err != nil {
return fmt.Errorf("fetching image reference: %w", err)
}
@ -846,6 +846,6 @@ type applier interface {
type imageFetcher interface {
FetchReference(ctx context.Context,
provider cloudprovider.Provider, attestationVariant variant.Variant,
image, region string,
image, region string, useMarketplaceImage bool,
) (string, error)
}

View File

@ -389,7 +389,7 @@ type stubImageFetcher struct {
func (f *stubImageFetcher) FetchReference(_ context.Context,
_ cloudprovider.Provider, _ variant.Variant,
_, _ string,
_, _ string, _ bool,
) (string, error) {
return f.reference, f.fetchReferenceErr
}

View File

@ -209,6 +209,8 @@ type AzureClusterVariables struct {
CustomEndpoint string `hcl:"custom_endpoint" cty:"custom_endpoint"`
// InternalLoadBalancer is true if an internal load balancer should be created.
InternalLoadBalancer bool `hcl:"internal_load_balancer" cty:"internal_load_balancer"`
// MarketplaceImage is the (optional) Azure Marketplace image to use.
MarketplaceImage AzureMarketplaceImageVariables `hcl:"marketplace_image" cty:"marketplace_image"`
}
// GetCreateMAA gets the CreateMAA variable.
@ -250,6 +252,18 @@ type AzureIAMVariables struct {
ResourceGroup string `hcl:"resource_group_name" cty:"resource_group_name"`
}
// AzureMarketplaceImageVariables is a configuration for specifying an Azure Marketplace image.
type AzureMarketplaceImageVariables struct {
// Publisher is the publisher ID of the image.
Publisher string `hcl:"publisher" cty:"publisher"`
// Product is the product ID of the image.
Product string `hcl:"product" cty:"product"`
// Name is the name of the image.
Name string `hcl:"name" cty:"name"`
// Version is the version of the image.
Version string `hcl:"version" cty:"version"`
}
// String returns a string representation of the IAM-specific variables, formatted as Terraform variables.
func (v *AzureIAMVariables) String() string {
f := hclwrite.NewEmptyFile()

View File

@ -193,6 +193,12 @@ func TestAzureClusterVariables(t *testing.T) {
Debug: to.Ptr(true),
Location: "eu-central-1",
CustomEndpoint: "example.com",
MarketplaceImage: AzureMarketplaceImageVariables{
Publisher: "edgelesssys",
Product: "constellation",
Name: "constellation",
Version: "2.13.0",
},
}
// test that the variables are correctly rendered
@ -216,6 +222,12 @@ node_groups = {
}
custom_endpoint = "example.com"
internal_load_balancer = false
marketplace_image = {
name = "constellation"
product = "constellation"
publisher = "edgelesssys"
version = "2.13.0"
}
`
got := vars.String()
assert.Equal(t, want, got)

View File

@ -0,0 +1,27 @@
# Using Marketplace Images in Constellation
This document explains the steps a user needs to take to run Constellation with dynamic billing via the cloud marketplaces.
## AWS
Marketplace Images on AWS are not available yet.
## Azure
On Azure, to use a marketplace image, ensure that the subscription has accepted the agreement to use marketplace images:
```bash
az vm image terms accept --publisher edgelesssystems --offer constellation --plan constellation
```
Then, set the VMs to use the marketplace image in the `constellation-conf.yaml` file:
```bash
yq eval -i ".provider.azure.useMarketplaceImage = true" constellation-conf.yaml
```
And ensure that the cluster uses a release image (i.e. `.image=vX.Y.Z` in the `constellation-conf.yaml` file). Afterwards, proceed with the cluster creation as usual.
## GCP
Marketplace Images on GCP are not available yet.

View File

@ -21,3 +21,7 @@ You are free to use the Constellation binaries provided by Edgeless Systems to c
Enterprise Licenses don't have the above limitations and come with support and additional features. Find out more at the [product website](https://www.edgeless.systems/products/constellation/).
Once you have received your Enterprise License file, place it in your [Constellation workspace](../architecture/orchestration.md#workspaces) in a file named `constellation.license`.
### Azure Marketplace
Constellation is available through the Azure Marketplace. This allows you to create self-managed Constellation clusters that are billed on a pay-per-use basis (hourly, per vCPU) with your Azure account. You can still get direct support by Edgeless Systems. For more information, please [contact us](https://www.edgeless.systems/enterprise-support/).

View File

@ -265,7 +265,7 @@ func writeUpgradeConfig(require *require.Assertions, image string, kubernetes st
cfg.GetProvider(),
cfg.GetAttestationConfig().GetVariant(),
image,
cfg.GetRegion(),
cfg.GetRegion(), cfg.UseMarketplaceImage(),
)
require.NoError(err)

View File

@ -53,7 +53,8 @@ func main() {
provider := conf.GetProvider()
attestationVariant := conf.GetAttestationConfig().GetVariant()
region := conf.GetRegion()
image, err := imgFetcher.FetchReference(ctx, provider, attestationVariant, conf.Image, region)
image, err := imgFetcher.FetchReference(ctx, provider, attestationVariant,
conf.Image, region, conf.UseMarketplaceImage())
if err != nil {
panic(err)
}

View File

@ -2,5 +2,5 @@
Type=esp
Format=vfat
CopyFiles=/efi:/
SizeMinBytes=512M
SizeMaxBytes=1024M
SizeMinBytes=1024M
SizeMaxBytes=2048M

View File

@ -38,6 +38,7 @@ pack() {
azure)
echo "📥 Packing Azure image..."
# Disk Images on Azure have to be a multiple of 1MiB in size.
truncate -s %1MiB "${unpacked_image_dir}/${unpacked_image_filename}"
qemu-img convert -p -f raw -O vpc -o force_size,subformat=fixed "${unpacked_image_dir}/${unpacked_image_filename}" "${packed_image}"
echo " Repacked image stored in ${packed_image}"

View File

@ -161,6 +161,9 @@ type AzureConfig struct {
// description: |
// Enable secure boot for VMs. If enabled, the OS image has to include a virtual machine guest state (VMGS) blob.
SecureBoot *bool `yaml:"secureBoot" validate:"required"`
// description: |
// Use the specified Azure Marketplace image offering.
UseMarketplaceImage *bool `yaml:"useMarketplaceImage" validate:"omitempty"`
}
// GCPConfig are GCP specific configuration values used by the CLI.
@ -693,6 +696,11 @@ func (c *Config) DeployYawolLoadBalancer() bool {
return c.Provider.OpenStack != nil && c.Provider.OpenStack.DeployYawolLoadBalancer != nil && *c.Provider.OpenStack.DeployYawolLoadBalancer
}
// UseMarketplaceImage returns whether a marketplace image should be used.
func (c *Config) UseMarketplaceImage() bool {
return c.Provider.Azure != nil && c.Provider.Azure.UseMarketplaceImage != nil && *c.Provider.Azure.UseMarketplaceImage
}
// Validate checks the config values and returns validation errors.
func (c *Config) Validate(force bool) error {
trans := ut.New(en.New()).GetFallback()

View File

@ -178,7 +178,7 @@ func init() {
FieldName: "azure",
},
}
AzureConfigDoc.Fields = make([]encoder.Doc, 7)
AzureConfigDoc.Fields = make([]encoder.Doc, 8)
AzureConfigDoc.Fields[0].Name = "subscription"
AzureConfigDoc.Fields[0].Type = "string"
AzureConfigDoc.Fields[0].Note = ""
@ -214,6 +214,11 @@ func init() {
AzureConfigDoc.Fields[6].Note = ""
AzureConfigDoc.Fields[6].Description = "Enable secure boot for VMs. If enabled, the OS image has to include a virtual machine guest state (VMGS) blob."
AzureConfigDoc.Fields[6].Comments[encoder.LineComment] = "Enable secure boot for VMs. If enabled, the OS image has to include a virtual machine guest state (VMGS) blob."
AzureConfigDoc.Fields[7].Name = "useMarketplaceImage"
AzureConfigDoc.Fields[7].Type = "bool"
AzureConfigDoc.Fields[7].Note = ""
AzureConfigDoc.Fields[7].Description = "Use the specified Azure Marketplace image offering."
AzureConfigDoc.Fields[7].Comments[encoder.LineComment] = "Use the specified Azure Marketplace image offering."
GCPConfigDoc.Type = "GCPConfig"
GCPConfigDoc.Comments[encoder.LineComment] = "GCPConfig are GCP specific configuration values used by the CLI."

View File

@ -180,6 +180,32 @@ const (
// WorkerDefault is the name of the default worker group.
WorkerDefault = "worker_default"
//
// CSP.
//
// MarketplaceImageURIScheme is the scheme used for Constellation marketplace OS images.
MarketplaceImageURIScheme = "constellation-marketplace-image"
//
// Azure.
//
// AzureMarketplaceImagePublisherKey is the URI key for the Azure Marketplace image publisher.
AzureMarketplaceImagePublisherKey = "publisher"
// AzureMarketplaceImageOfferKey is the URI key for the Azure Marketplace image offer.
AzureMarketplaceImageOfferKey = "offer"
// AzureMarketplaceImageSkuKey is the URI key for the Azure Marketplace image SKU.
AzureMarketplaceImageSkuKey = "sku"
// AzureMarketplaceImageVersionKey is the URI key for the Azure Marketplace image version.
AzureMarketplaceImageVersionKey = "version"
// AzureMarketplaceImagePublisher is the publisher of the Azure Marketplace image.
AzureMarketplaceImagePublisher = "edgelesssystems"
// AzureMarketplaceImageOffer is the offer of the Azure Marketplace image.
AzureMarketplaceImageOffer = "constellation"
// AzureMarketplaceImagePlan is the plan of the Azure Marketplace image.
AzureMarketplaceImagePlan = "constellation"
//
// Kubernetes.
//

View File

@ -14,6 +14,8 @@ go_library(
"//internal/api/versionsapi",
"//internal/attestation/variant",
"//internal/cloud/cloudprovider",
"//internal/mpimage",
"//internal/semver",
"@com_github_schollz_progressbar_v3//:progressbar",
"@com_github_spf13_afero//:afero",
],

View File

@ -23,6 +23,8 @@ import (
"github.com/edgelesssys/constellation/v2/internal/api/versionsapi"
"github.com/edgelesssys/constellation/v2/internal/attestation/variant"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/mpimage"
"github.com/edgelesssys/constellation/v2/internal/semver"
"github.com/spf13/afero"
)
@ -43,13 +45,17 @@ func New() *Fetcher {
// FetchReference fetches the image reference for a given image version uid, CSP and image variant.
func (f *Fetcher) FetchReference(ctx context.Context,
provider cloudprovider.Provider, attestationVariant variant.Variant,
image, region string,
image, region string, useMarketplaceImage bool,
) (string, error) {
ver, err := versionsapi.NewVersionFromShortPath(image, versionsapi.VersionKindImage)
if err != nil {
return "", fmt.Errorf("parsing config image short path: %w", err)
}
if useMarketplaceImage {
return buildMarketplaceImage(ver, provider)
}
imgInfoReq := versionsapi.ImageInfo{
Ref: ver.Ref(),
Stream: ver.Stream(),
@ -82,6 +88,21 @@ func (f *Fetcher) FetchReference(ctx context.Context,
return getReferenceFromImageInfo(provider, attestationVariant.String(), imgInfo, filters(provider, region)...)
}
// buildMarketplaceImage returns a marketplace image URI for the given CSP and version.
func buildMarketplaceImage(ver versionsapi.Version, provider cloudprovider.Provider) (string, error) {
sv, err := semver.New(ver.Version())
if err != nil {
return "", fmt.Errorf("parsing image version: %w", err)
}
switch provider {
case cloudprovider.Azure:
return mpimage.NewAzureMarketplaceImage(sv).URI(), nil
default:
return "", fmt.Errorf("marketplace images are not supported for csp %s", provider.String())
}
}
func filters(provider cloudprovider.Provider, region string) []filter {
var filters []filter
switch provider {

View File

@ -256,7 +256,8 @@ func TestFetchReference(t *testing.T) {
fs: af,
}
reference, err := fetcher.FetchReference(context.Background(), tc.provider, variant.Dummy{}, tc.image, "someRegion")
reference, err := fetcher.FetchReference(context.Background(), tc.provider, variant.Dummy{},
tc.image, "someRegion", false)
if tc.wantErr {
assert.Error(err)

View File

@ -0,0 +1,24 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "mpimage",
srcs = [
"mpimage.go",
"uri.go",
],
importpath = "github.com/edgelesssys/constellation/v2/internal/mpimage",
visibility = ["//:__subpackages__"],
deps = [
"//internal/cloud/cloudprovider",
"//internal/constants",
"//internal/semver",
],
)
go_test(
name = "mpimage_test",
srcs = ["uri_test.go"],
embed = [":mpimage"],
deps = ["@com_github_stretchr_testify//assert"],
)

View File

@ -0,0 +1,8 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
// The mpimage package provides utilities for handling CSP marketplace OS images.
package mpimage

80
internal/mpimage/uri.go Normal file
View File

@ -0,0 +1,80 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package mpimage
import (
"fmt"
"net/url"
"strings"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/semver"
)
// MarketplaceImage represents a CSP-agnostic marketplace image.
type MarketplaceImage interface {
URI() string
}
// NewFromURI returns a new MarketplaceImage for the given image URI.
func NewFromURI(uri string) (MarketplaceImage, error) {
u, err := url.Parse(uri)
if err != nil {
return nil, err
}
if u.Scheme != constants.MarketplaceImageURIScheme {
return nil, fmt.Errorf("invalid scheme: %s", u.Scheme)
}
switch u.Host {
case cloudprovider.Azure.String():
ver, err := semver.New(u.Query().Get(constants.AzureMarketplaceImageVersionKey))
if err != nil {
return nil, fmt.Errorf("invalid image version: %w", err)
}
return NewAzureMarketplaceImage(ver), nil
default:
return nil, fmt.Errorf("invalid host: %s", u.Host)
}
}
// AzureMarketplaceImage represents an Azure marketplace image.
type AzureMarketplaceImage struct {
Publisher string
Offer string
SKU string
Version string
}
// NewAzureMarketplaceImage returns a new Constellation marketplace image for the given version.
func NewAzureMarketplaceImage(version semver.Semver) AzureMarketplaceImage {
return AzureMarketplaceImage{
Publisher: constants.AzureMarketplaceImagePublisher,
Offer: constants.AzureMarketplaceImageOffer,
SKU: constants.AzureMarketplaceImagePlan,
Version: strings.TrimPrefix(version.String(), "v"), // Azure requires X.Y.Z format
}
}
// URI returns the URI for the image.
func (i AzureMarketplaceImage) URI() string {
u := &url.URL{
Scheme: constants.MarketplaceImageURIScheme,
Host: cloudprovider.Azure.String(),
}
q := u.Query()
q.Set(constants.AzureMarketplaceImagePublisherKey, i.Publisher)
q.Set(constants.AzureMarketplaceImageOfferKey, i.Offer)
q.Set(constants.AzureMarketplaceImageSkuKey, i.SKU)
q.Set(constants.AzureMarketplaceImageVersionKey, i.Version)
u.RawQuery = q.Encode()
return u.String()
}

View File

@ -0,0 +1,83 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package mpimage
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestNewFromURI(t *testing.T) {
testCases := map[string]struct {
uri string
want MarketplaceImage
wantErr bool
}{
"azure valid": {
uri: "constellation-marketplace-image://Azure?offer=constellation&publisher=edgelesssystems&sku=constellation&version=1.2.3",
want: AzureMarketplaceImage{
Publisher: "edgelesssystems",
Offer: "constellation",
SKU: "constellation",
Version: "1.2.3",
},
},
"azure invalid version": {
uri: "constellation-marketplace-image://Azure?offer=constellation&publisher=edgelesssystems&sku=constellation&version=asdf",
wantErr: true,
},
"invalid scheme": {
uri: "invalid://Azure?offer=constellation&publisher=edgelesssystems&sku=constellation&version=1.2.3",
wantErr: true,
},
"invalid host": {
uri: "constellation-marketplace-image://invalid?offer=constellation&publisher=edgelesssystems&sku=constellation&version=1.2.3",
wantErr: true,
},
"no uri": {
uri: "no uri",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
got, err := NewFromURI(tc.uri)
if tc.wantErr {
assert.Error(err)
} else {
assert.NoError(err)
assert.Equal(tc.want, got)
}
})
}
}
func TestAzureURI(t *testing.T) {
testCases := map[string]struct {
image AzureMarketplaceImage
want string
}{
"valid": {
image: AzureMarketplaceImage{
Publisher: "foo",
Offer: "bar",
SKU: "baz",
Version: "1.2.3",
},
want: "constellation-marketplace-image://Azure?offer=bar&publisher=foo&sku=baz&version=1.2.3",
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert.Equal(t, tc.want, tc.image.URI())
})
}
}

View File

@ -169,6 +169,7 @@ func (u *Uploader) createDisk(ctx context.Context, diskName string, diskType Dis
if diskType == DiskTypeWithVMGS && vmgs == nil {
return "", errors.New("cannot create disk with vmgs: vmgs reader is nil")
}
var createOption armcomputev5.DiskCreateOption
var requestVMGSSAS bool
switch diskType {

View File

@ -19,6 +19,7 @@ go_library(
visibility = ["//operators/constellation-node-operator:__subpackages__"],
deps = [
"//internal/constants",
"//internal/mpimage",
"//operators/constellation-node-operator/api/v1alpha1",
"//operators/constellation-node-operator/internal/cloud/api",
"//operators/constellation-node-operator/internal/poller",

View File

@ -14,6 +14,7 @@ import (
"github.com/Azure/azure-sdk-for-go/sdk/azcore/to"
"github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v5"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/mpimage"
updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1"
cspapi "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/api"
)
@ -47,11 +48,17 @@ func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, image
if err != nil {
return err
}
imageRef, err := imageReferenceFromImage(imageURI)
if err != nil {
return fmt.Errorf("parsing image reference: %w", err)
}
poller, err := c.scaleSetsAPI.BeginUpdate(ctx, resourceGroup, scaleSet, armcompute.VirtualMachineScaleSetUpdate{
Properties: &armcompute.VirtualMachineScaleSetUpdateProperties{
VirtualMachineProfile: &armcompute.VirtualMachineScaleSetUpdateVMProfile{
StorageProfile: &armcompute.VirtualMachineScaleSetUpdateStorageProfile{
ImageReference: imageReferenceFromImage(imageURI),
ImageReference: imageRef,
},
},
},
@ -141,14 +148,28 @@ func (c *Client) ListScalingGroups(ctx context.Context, uid string) ([]cspapi.Sc
return results, nil
}
func imageReferenceFromImage(img string) *armcompute.ImageReference {
func imageReferenceFromImage(img string) (*armcompute.ImageReference, error) {
ref := &armcompute.ImageReference{}
marketplaceImage, err := mpimage.NewFromURI(img)
if err == nil {
// expecting image to be an azure marketplace image
if azureMarketplaceImage, ok := marketplaceImage.(mpimage.AzureMarketplaceImage); ok {
ref.Publisher = to.Ptr(azureMarketplaceImage.Publisher)
ref.Offer = to.Ptr(azureMarketplaceImage.Offer)
ref.SKU = to.Ptr(azureMarketplaceImage.SKU)
ref.Version = to.Ptr(azureMarketplaceImage.Version)
return ref, nil
}
return nil, fmt.Errorf("marketplace image csp is unsupported: %s", img)
}
// expecting image to not be a marketplace image
if strings.HasPrefix(img, "/CommunityGalleries") {
ref.CommunityGalleryImageID = to.Ptr(img)
} else {
ref.ID = to.Ptr(img)
}
return ref
return ref, nil
}

View File

@ -289,27 +289,41 @@ func TestImageReferenceFromImage(t *testing.T) {
img string
wantID *string
wantCommunityID *string
wantPublisher *string
wantOffer *string
wantSKU *string
wantVersion *string
}{
"ID": {
img: "/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/1.5.0",
wantID: to.Ptr("/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/1.5.0"),
wantCommunityID: nil,
img: "/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/1.5.0",
wantID: to.Ptr("/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/constellation-images/providers/Microsoft.Compute/galleries/Constellation/images/constellation/versions/1.5.0"),
},
"Community": {
img: "/CommunityGalleries/ConstellationCVM-728bd310-e898-4450-a1ed-21cf2fb0d735/Images/feat-azure-cvm-sharing/Versions/2022.0826.084922",
wantID: nil,
wantCommunityID: to.Ptr("/CommunityGalleries/ConstellationCVM-728bd310-e898-4450-a1ed-21cf2fb0d735/Images/feat-azure-cvm-sharing/Versions/2022.0826.084922"),
},
"Marketplace": {
img: "constellation-marketplace-image://Azure?offer=constellation&publisher=edgelesssystems&sku=constellation&version=1.2.3",
wantPublisher: to.Ptr("edgelesssystems"),
wantOffer: to.Ptr("constellation"),
wantSKU: to.Ptr("constellation"),
wantVersion: to.Ptr("1.2.3"),
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
ref := imageReferenceFromImage(tc.img)
ref, err := imageReferenceFromImage(tc.img)
assert.NoError(err)
assert.Equal(tc.wantID, ref.ID)
assert.Equal(tc.wantCommunityID, ref.CommunityGalleryImageID)
assert.Equal(tc.wantPublisher, ref.Publisher)
assert.Equal(tc.wantOffer, ref.Offer)
assert.Equal(tc.wantSKU, ref.SKU)
assert.Equal(tc.wantVersion, ref.Version)
})
}
}

View File

@ -37,6 +37,7 @@ See the [full list of CSPs](https://docs.edgeless.systems/constellation/overview
### Optional
- `marketplace_image` (Boolean) Whether a marketplace image should be used. Currently only supported for Azure.
- `region` (String) Region to retrieve the image for. Only required for AWS.
The Constellation OS image must be [replicated to the region](https://docs.edgeless.systems/constellation/workflows/config),and the region must [support AMD SEV-SNP](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/snp-requirements.html), if it is used for Attestation.

View File

@ -38,7 +38,7 @@ type ImageDataSource struct {
type imageFetcher interface {
FetchReference(ctx context.Context,
provider cloudprovider.Provider, attestationVariant variant.Variant,
image, region string,
image, region string, useMarketplaceImage bool,
) (string, error)
}
@ -47,6 +47,7 @@ type ImageDataSourceModel struct {
AttestationVariant types.String `tfsdk:"attestation_variant"`
ImageVersion types.String `tfsdk:"image_version"`
CSP types.String `tfsdk:"csp"`
MarketplaceImage types.Bool `tfsdk:"marketplace_image"`
Region types.String `tfsdk:"region"`
Reference types.String `tfsdk:"reference"`
}
@ -69,6 +70,11 @@ func (d *ImageDataSource) Schema(_ context.Context, _ datasource.SchemaRequest,
Required: true, // TODO(msanft): Make this optional to support "lockstep" mode.
},
"csp": newCSPAttribute(),
"marketplace_image": schema.BoolAttribute{
Description: "Whether a marketplace image should be used. Currently only supported for Azure.",
MarkdownDescription: "Whether a marketplace image should be used. Currently only supported for Azure.",
Optional: true,
},
"region": schema.StringAttribute{
Description: "Region to retrieve the image for. Only required for AWS.",
MarkdownDescription: "Region to retrieve the image for. Only required for AWS.\n" +
@ -124,7 +130,8 @@ func (d *ImageDataSource) Read(ctx context.Context, req datasource.ReadRequest,
}
// Retrieve Image Reference
imageRef, err := d.imageFetcher.FetchReference(ctx, csp, attestationVariant, data.ImageVersion.ValueString(), data.Region.ValueString())
imageRef, err := d.imageFetcher.FetchReference(ctx, csp, attestationVariant,
data.ImageVersion.ValueString(), data.Region.ValueString(), data.MarketplaceImage.ValueBool())
if err != nil {
resp.Diagnostics.AddError(
"Error fetching Image Reference",

View File

@ -55,6 +55,25 @@ func TestAccImageDataSource(t *testing.T) {
},
},
},
"azure marketplace success": {
ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
PreCheck: bazelPreCheck,
Steps: []resource.TestStep{
// Read testing
{
Config: testingConfig + `
data "constellation_image" "test" {
image_version = "v2.13.0"
attestation_variant = "azure-sev-snp"
csp = "azure"
marketplace_image = true
}
`,
Check: resource.TestCheckResourceAttr("data.constellation_image.test", "reference", "constellation-marketplace-image://Azure?offer=constellation&publisher=edgelesssystems&sku=constellation&version=2.13.0"), // should be immutable
},
},
},
"gcp success": {
ProtoV6ProviderFactories: testAccProtoV6ProviderFactories,
PreCheck: bazelPreCheck,

View File

@ -268,6 +268,7 @@ module "scale_set_group" {
azurerm_lb_backend_address_pool.all.id,
module.loadbalancer_backend_worker.backendpool_id
]
marketplace_image = var.marketplace_image
}
module "jump_host" {

View File

@ -46,9 +46,10 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
disable_password_authentication = false
upgrade_mode = "Manual"
secure_boot_enabled = var.secure_boot
source_image_id = var.image_id
tags = local.tags
zones = var.zones
# specify the image id only if a non-marketplace image is used
source_image_id = var.marketplace_image != null ? null : var.image_id
tags = local.tags
zones = var.zones
identity {
type = "UserAssigned"
identity_ids = [var.user_assigned_identity]
@ -72,6 +73,26 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
}
}
# Specify marketplace plan and image if set
dynamic "plan" {
for_each = var.marketplace_image != null ? [1] : [] # if a marketplace image is set
content {
name = var.marketplace_image.name
publisher = var.marketplace_image.publisher
product = var.marketplace_image.product
}
}
dynamic "source_image_reference" {
for_each = var.marketplace_image != null ? [1] : [] # if a marketplace image is set
content {
publisher = var.marketplace_image.publisher
offer = var.marketplace_image.product
sku = var.marketplace_image.name
version = var.marketplace_image.version
}
}
data_disk {
storage_account_type = var.state_disk_type
disk_size_gb = var.state_disk_size
@ -94,9 +115,10 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
lifecycle {
ignore_changes = [
name, # required. Allow legacy scale sets to keep their old names
instances, # required. autoscaling modifies the instance count externally
source_image_id, # required. update procedure modifies the image id externally
name, # required. Allow legacy scale sets to keep their old names
instances, # required. autoscaling modifies the instance count externally
source_image_id, # required. update procedure modifies the image id externally
source_image_reference, # required. update procedure modifies the image reference externally
]
}
}

View File

@ -96,3 +96,14 @@ variable "secure_boot" {
default = false
description = "Whether to deploy the cluster nodes with secure boot."
}
variable "marketplace_image" {
type = object({
name = string
publisher = string
product = string
version = string
})
default = null
description = "Marketplace image to use for the cluster nodes."
}

View File

@ -73,3 +73,14 @@ variable "internal_load_balancer" {
default = false
description = "Whether to use an internal load balancer for the Constellation."
}
variable "marketplace_image" {
type = object({
name = string
publisher = string
product = string
version = string
})
default = null
description = "Marketplace image to use for the cluster nodes."
}