ci: add e2e-upgrade test

The test is implemented as a go test.
It can be executed as a bazel target.
The general workflow is to setup a cluster,
point the test to the workspace in which to
find the kubeconfig and the constellation config
and specify a target image, k8s and
service version. The test will succeed
if it detects all target versions in the cluster
within the configured timeout.
The CI automates the above steps.
A separate workflow is introduced as there
are multiple input fields to the test.
Adding all of these to the manual e2e test
seemed confusing.

Co-authored-by: Fabian Kammel <fk@edgeless.systems>

.github/actions/e2e_test/action.yml

@@ -23,6 +23,9 @@ inputs:
     description: "Is OS img a debug img?"
     default: "true"
     required: true
+  cliVersion:
+    description: "Version of a released CLI to download, e.g. 'v2.3.0', leave empty to build it."
+    required: false
   kubernetesVersion:
     description: "Kubernetes version to create the cluster from."
     required: false
@@ -114,6 +117,7 @@ runs:
         buildBuddyApiKey: ${{ inputs.buildBuddyApiKey }}
 
     - name: Build CLI
+      if: inputs.cliVersion == ''
       uses: ./.github/actions/build_cli
       with:
         targetOS: ${{ steps.determine-build-target.outputs.hostOS }}
@@ -121,6 +125,18 @@ runs:
         enterpriseCLI: ${{ inputs.keepMeasurements }}
         outputPath: "build/constellation"
 
+    - name: Fetch CLI
+      if: inputs.cliVersion != ''
+      shell: bash
+      run: |
+        curl -fsSL -o constellation https://github.com/edgelesssys/constellation/releases/download/${{ inputs.cliVersion }}/constellation-linux-amd64
+        chmod u+x constellation
+        echo "$(pwd)" >> $GITHUB_PATH
+        export PATH="$PATH:$(pwd)"
+        constellation version
+        # Do not spam license server from pipeline
+        sudo sh -c 'echo "127.0.0.1 license.confidential.cloud" >> /etc/hosts'
+
     - name: Build the bootstrapper
       id: build-bootstrapper
       if: inputs.isDebugImage == 'true'

.github/workflows/e2e-test-weekly.yml

@@ -188,3 +188,23 @@ jobs:
             --force-deletion-types Microsoft.Compute/virtualMachines \
             --no-wait \
             --yes
+
+  e2e-upgrade:
+    strategy:
+        fail-fast: false
+        max-parallel: 1
+        matrix:
+          fromVersion:
+            ["2.6"]
+          cloudProvider: ["gcp", "azure"]
+    name: Run upgrade tests
+    secrets: inherit
+    permissions:
+      id-token: write
+      contents: read
+    uses: ./.github/workflows/e2e-upgrade.yml
+    with:
+      fromVersion: ${{ matrix.fromVersion }}
+      cloudProvider: ${{ matrix.cloudProvider }}
+      workerNodesCount: 2
+      controlNodesCount: 3

.github/workflows/e2e-upgrade.yml

@@ -0,0 +1,224 @@
+name: e2e test upgrade
+
+on:
+  workflow_dispatch:
+    inputs:
+      cloudProvider:
+        description: "Which cloud provider to use."
+        type: choice
+        options:
+          - "gcp"
+          - "azure"
+        default: "azure"
+      workerNodesCount:
+        description: "Number of worker nodes to spawn."
+        default: "2"
+      controlNodesCount:
+        description: "Number of control-plane nodes to spawn."
+        default: "3"
+      fromVersion:
+        description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
+        type: string
+        required: true
+      toCLI:
+        description: CLI version to execute upgrade with, e.g., 'v2.1.3', or empty to build HEAD.
+        type: string
+        required: false
+      toImage:
+        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
+        type: string
+        required: false
+      toKubernetes:
+        description: Kubernetes version to target for the upgrade, empty for target's default version.
+        type: string
+        required: false
+      toMicroservices:
+        description: Microservice version to target for the upgrade, empty for target's default version.
+        type: string
+        required: false
+  workflow_call:
+    inputs:
+      cloudProvider:
+        description: "Which cloud provider to use."
+        type: string
+        required: true
+      workerNodesCount:
+        description: "Number of worker nodes to spawn."
+        type: number
+        required: true
+      controlNodesCount:
+        description: "Number of control-plane nodes to spawn."
+        type: number
+        required: true
+      fromVersion:
+        description: CLI version to create a new cluster with. This has to be a released version, e.g., 'v2.1.3'.
+        type: string
+        required: true
+      toCLI:
+        description: CLI version to execute upgrade with, e.g., 'v2.1.3', or empty to build HEAD.
+        type: string
+        required: false
+      toImage:
+        description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly.
+        type: string
+        required: false
+      toKubernetes:
+        description: Kubernetes version to target for the upgrade, empty for target's default version.
+        type: string
+        required: false
+      toMicroservices:
+        description: Kubernetes version to target for the upgrade, empty for target's default version.
+        type: string
+        required: false
+
+
+env:
+  ARM_CLIENT_ID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
+  ARM_CLIENT_SECRET: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+  ARM_SUBSCRIPTION_ID: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
+  ARM_TENANT_ID: ${{ secrets.AZURE_E2E_TENANT_ID }}
+
+jobs:
+  e2e-upgrade:
+    runs-on: ubuntu-22.04
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v3.2.0
+        with:
+          fetch-depth: 0
+          ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
+
+      - name: Setup Go environment
+        uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v3.5.0
+        with:
+          go-version: "1.20.2"
+
+      - name: Login to Azure
+        if: inputs.cloudProvider == 'azure'
+        uses: ./.github/actions/login_azure
+        with:
+          azure_credentials: ${{ secrets.AZURE_E2E_CREDENTIALS }}
+
+      - name: Login to AWS
+        uses: aws-actions/configure-aws-credentials@67fbcbb121271f7775d2e7715933280b06314838 # v1.7.0
+        with:
+          role-to-assume: arn:aws:iam::795746500882:role/GithubConstellationVersionsAPIRead
+          aws-region: eu-central-1
+
+      - name: Create Azure resource group
+        if: inputs.cloudProvider == 'azure'
+        id: az_resource_group_gen
+        shell: bash
+        run: |
+          uuid=$(cat /proc/sys/kernel/random/uuid)
+          name=e2e-test-${uuid%%-*}
+          az group create --location northeurope --name "$name" --tags e2e
+          echo "res_group_name=$name" >> "$GITHUB_OUTPUT"
+
+      - name: Find latest nightly image
+        id: find-image
+        if: inputs.toImage == ''
+        uses: ./.github/actions/versionsapi
+        with:
+          command: latest
+          ref: main
+          stream: nightly
+
+      - name: Create cluster with 'fromVersion' CLI.
+        id: e2e_test
+        uses: ./.github/actions/e2e_test
+        with:
+          workerNodesCount: ${{ inputs.workerNodesCount }}
+          controlNodesCount: ${{ inputs.controlNodesCount }}
+          cloudProvider: ${{ inputs.cloudProvider }}
+          osImage: ${{ inputs.fromVersion }}
+          cliVersion: ${{ inputs.fromVersion }}
+          isDebugImage: "false"
+          azureSubscription: ${{ secrets.AZURE_E2E_SUBSCRIPTION_ID }}
+          azureTenant: ${{ secrets.AZURE_E2E_TENANT_ID }}
+          azureClientID: ${{ secrets.AZURE_E2E_CLIENT_ID }}
+          azureClientSecret: ${{ secrets.AZURE_E2E_CLIENT_SECRET }}
+          azureUserAssignedIdentity: ${{ secrets.AZURE_E2E_USER_ASSIGNED_IDENTITY }}
+          azureResourceGroup: ${{ steps.az_resource_group_gen.outputs.res_group_name }}
+          gcpProject: ${{ secrets.GCP_E2E_PROJECT }}
+          gcp_service_account: "constellation-e2e@constellation-331613.iam.gserviceaccount.com"
+          gcpClusterServiceAccountKey: ${{ secrets.GCP_CLUSTER_SERVICE_ACCOUNT }}
+          test: "nop"
+          buildBuddyApiKey: ${{ secrets.BUILDBUDDY_ORG_API_KEY }}
+
+      - name: Run upgrade test
+        run: |
+          echo "Image target: $IMAGE"
+          echo "K8s target: $KUBERNETES"
+          echo "Microservice target: $MICROSERVICES"
+
+          if [[ -n ${MICROSERVICES} ]]; then
+            MICROSERVICES_FLAG="--target-microservices $MICROSERVICES"
+          fi
+          if [[ -n ${KUBERNETES} ]]; then
+            KUBERNETES_FLAG="--target-kubernetes $KUBERNETES"
+          fi
+
+          bazelisk run //e2e/internal/upgrade:upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --target-image "$IMAGE" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG"
+        env:
+          KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }}
+          IMAGE: ${{ inputs.toImage && inputs.toImage || steps.find-image.outputs.output }}
+          KUBERNETES: ${{ inputs.toKubernetes }}
+          MICROSERVICES: ${{ inputs.toMicroservices }}
+          WORKERNODES: ${{ inputs.workerNodesCount }}
+          CONTROLNODES: ${{ inputs.controlNodesCount }}
+
+      - name: Always fetch logs
+        if: always()
+        continue-on-error: true
+        run: |
+          kubectl logs -n kube-system -l "app.kubernetes.io/name=node-maintenance-operator" --tail=-1 > node-maintenance-operator.logs
+          kubectl get nodeversions.update.edgeless.systems constellation-version -o yaml > constellation-version.yaml
+        env:
+          KUBECONFIG: ${{ steps.e2e_test.outputs.kubeconfig }}
+
+      - name: Always upload logs
+        if: always()
+        continue-on-error: true
+        uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # tag=v3.1.1
+        with:
+          name: upgrade-logs
+          path: |
+            node-maintenance-operator.logs
+            constellation-version.yaml
+
+      - name: Always terminate cluster
+        if: always()
+        continue-on-error: true
+        uses: ./.github/actions/constellation_destroy
+        with:
+          kubeconfig: ${{ steps.e2e_test.outputs.kubeconfig }}
+
+      - name: Notify teams channel
+        if: failure() && github.ref == 'refs/heads/main'
+        continue-on-error: true
+        shell: bash
+        working-directory: .github/actions/e2e_test
+        run: |
+          sudo apt-get install gettext-base -y
+          export TEAMS_JOB_NAME="upgrade-${{ inputs.cloudProvider }}"
+          export TEAMS_RUN_ID=${{ github.run_id }}
+          envsubst < teams-payload.json > to-be-send.json
+          curl                                          \
+            -H "Content-Type: application/json"         \
+            -d @to-be-send.json                         \
+            "${{ secrets.MS_TEAMS_WEBHOOK_URI }}"
+
+      - name: Always destroy Azure resource group
+        if: always() && inputs.cloudProvider == 'azure'
+        shell: bash
+        run: |
+          az group delete \
+            --name ${{ steps.az_resource_group_gen.outputs.res_group_name }} \
+            --force-deletion-types Microsoft.Compute/virtualMachineScaleSets \
+            --force-deletion-types Microsoft.Compute/virtualMachines \
+            --no-wait \
+            --yes

.golangci.yml

@@ -2,6 +2,7 @@ run:
   timeout: 10m
   build-tags:
     - integration
+    - e2e
   modules-download-mode: readonly
   skip-dirs:
     - 3rdparty/node-maintenance-operator

bazel/ci/BUILD.bazel

@@ -3,10 +3,16 @@ load("@com_github_ash2k_bazel_tools//multirun:def.bzl", "multirun")
 load("@com_github_bazelbuild_buildtools//buildifier:def.bzl", "buildifier", "buildifier_test")
 load("//bazel/sh:def.bzl", "noop_warn", "repo_command", "sh_template")
 
-gazelle(name = "gazelle_generate")
+required_tags = ["e2e"]
+
+gazelle(
+    name = "gazelle_generate",
+    build_tags = required_tags,
+)
 
 gazelle(
     name = "gazelle_check",
+    build_tags = required_tags,
     command = "fix",
     mode = "diff",
 )

bazel/go/go_test.bzl

@@ -4,7 +4,7 @@ This module contains rules and macros for building and testing Go code.
 
 load("@io_bazel_rules_go//go:def.bzl", _go_test = "go_test")
 
-def go_test(ld = None, **kwargs):
+def go_test(ld = None, count = 3, **kwargs):
     """go_test is a wrapper for go_test that uses default settings for Constellation.
 
     It adds the following:
@@ -14,12 +14,13 @@ def go_test(ld = None, **kwargs):
 
     Args:
           ld: path to interpreter to that will be written into the elf header.
+          count: number of times each test should be executed. defaults to 3.
           **kwargs: all other arguments are passed to go_test.
     """
 
     # Sets test count to 3.
     kwargs.setdefault("args", [])
-    kwargs["args"].append("--test.count=3")
+    kwargs["args"].append("--test.count={}".format(count))
 
     # enable race detector by default
     kwargs.setdefault("race", "on")

cli/internal/cmd/configfetchmeasurements.go

@@ -12,12 +12,9 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
-	"path"
-	"strings"
 	"time"
 
 	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
-	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/config"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/edgelesssys/constellation/v2/internal/file"
@@ -181,34 +178,21 @@ func (cfm *configFetchMeasurementsCmd) parseFetchMeasurementsFlags(cmd *cobra.Co
 }
 
 func (f *fetchMeasurementsFlags) updateURLs(conf *config.Config) error {
+	ver, err := versionsapi.NewVersionFromShortPath(conf.Image, versionsapi.VersionKindImage)
+	if err != nil {
+		return fmt.Errorf("creating version from image name: %w", err)
+	}
+	measurementsURL, signatureURL, err := versionsapi.MeasurementURL(ver, conf.GetProvider())
+	if err != nil {
+		return err
+	}
+
 	if f.measurementsURL == nil {
-		url, err := measurementURL(conf.GetProvider(), conf.Image, "measurements.json")
-		if err != nil {
-			return err
-		}
-		f.measurementsURL = url
+		f.measurementsURL = measurementsURL
 	}
 
 	if f.signatureURL == nil {
-		url, err := measurementURL(conf.GetProvider(), conf.Image, "measurements.json.sig")
-		if err != nil {
-			return err
-		}
-		f.signatureURL = url
+		f.signatureURL = signatureURL
 	}
 	return nil
 }
-
-func measurementURL(provider cloudprovider.Provider, image, file string) (*url.URL, error) {
-	ver, err := versionsapi.NewVersionFromShortPath(image, versionsapi.VersionKindImage)
-	if err != nil {
-		return nil, fmt.Errorf("creating version from image name: %w", err)
-	}
-	artifactBaseURL := ver.ArtifactURL()
-	url, err := url.Parse(artifactBaseURL)
-	if err != nil {
-		return nil, fmt.Errorf("parsing artifact base URL %s: %w", artifactBaseURL, err)
-	}
-	url.Path = path.Join(url.Path, "image", "csp", strings.ToLower(provider.String()), file)
-	return url, nil
-}

cli/internal/cmd/configfetchmeasurements_test.go

@@ -119,11 +119,13 @@ func TestUpdateURLs(t *testing.T) {
 				},
 			},
 			flags:                  &fetchMeasurementsFlags{},
-			wantMeasurementsURL:    ver.ArtifactURL() + "/image/csp/gcp/measurements.json",
-			wantMeasurementsSigURL: ver.ArtifactURL() + "/image/csp/gcp/measurements.json.sig",
+			wantMeasurementsURL:    ver.ArtifactsURL() + "/image/csp/gcp/measurements.json",
+			wantMeasurementsSigURL: ver.ArtifactsURL() + "/image/csp/gcp/measurements.json.sig",
 		},
 		"both set by user": {
-			conf: &config.Config{},
+			conf: &config.Config{
+				Image: ver.ShortPath(),
+			},
 			flags: &fetchMeasurementsFlags{
 				measurementsURL: urlMustParse("get.my/measurements.json"),
 				signatureURL:    urlMustParse("get.my/measurements.json.sig"),

cli/internal/cmd/create.go

@@ -147,11 +147,11 @@ func (c *createCmd) create(cmd *cobra.Command, creator cloudCreator, fileHandler
 
 	spinner.Start("Creating", false)
 	idFile, err := creator.Create(cmd.Context(), provider, conf, instanceType, flags.controllerCount, flags.workerCount)
-	c.log.Debugf("Successfully created the cloud resources for the cluster")
 	spinner.Stop()
 	if err != nil {
 		return translateCreateErrors(cmd, err)
 	}
+	c.log.Debugf("Successfully created the cloud resources for the cluster")
 
 	if err := fileHandler.WriteJSON(constants.ClusterIDsFileName, idFile, file.OptNone); err != nil {
 		return err

cli/internal/cmd/upgradecheck.go

@@ -470,12 +470,7 @@ func getCompatibleImageMeasurements(ctx context.Context, writer io.Writer, clien
 	for _, version := range versions {
 		log.Debugf("Fetching measurements for image: %s", version)
 		shortPath := version.ShortPath()
-		measurementsURL, err := measurementURL(csp, shortPath, "measurements.json")
-		if err != nil {
-			return nil, err
-		}
-
-		signatureURL, err := measurementURL(csp, shortPath, "measurements.json.sig")
+		measurementsURL, signatureURL, err := versionsapi.MeasurementURL(version, csp)
 		if err != nil {
 			return nil, err
 		}

cli/internal/helm/charts/edgeless/constellation-services/charts/join-service/values.schema.json

@@ -10,10 +10,6 @@
             "type": "string",
             "examples": ["{'1':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA','warnOnly':true},'15':{'expected':'AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=','warnOnly':true}}"]
         },
-        "enforceIdKeyDigest": {
-            "description": "ID Key Digest enforcement policy.",
-            "enum": ["StrictChecking", "MAAFallback", "WarnOnly"]
-        },
         "idKeyConfig": {
             "description": "Configuration for validating the ID Key Digest of the SEV-SNP attestation.",
             "type": "string",

cli/internal/helm/client.go

@@ -110,7 +110,7 @@ func (c *Client) Upgrade(ctx context.Context, config *config.Config, timeout tim
 		case errors.As(err, &invalidUpgrade):
 			upgradeErrs = append(upgradeErrs, fmt.Errorf("skipping %s upgrade: %w", info.releaseName, err))
 		case err != nil:
-			return fmt.Errorf("upgrading %s: %w", info.releaseName, err)
+			return fmt.Errorf("should upgrade %s: %w", info.releaseName, err)
 		case err == nil:
 			upgradeReleases = append(upgradeReleases, chart)
 		}

cli/internal/helm/loader.go

@@ -464,10 +464,7 @@ func (i *ChartLoader) loadConstellationServicesValues() (map[string]any, error)
 }
 
 // extendConstellationServicesValues extends the given values map by some values depending on user input.
-// This extra step of separating the application of user input is necessary since service upgrades should
-// reuse user input from the init step. However, we can't rely on reuse-values, because
-// during upgrades all values need to be set locally as they might have changed.
-// Also, the charts are not rendered correctly without all of these values.
+// Values set inside this function are only applied during init, not during upgrade.
 func extendConstellationServicesValues(
 	in map[string]any, config *config.Config, masterSecret, salt []byte, maaURL string,
 ) error {

e2e/internal/upgrade/BUILD.bazel

@@ -0,0 +1,50 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("//bazel/go:go_test.bzl", "go_test")
+
+go_library(
+    name = "upgrade",
+    srcs = [
+        "helm.go",
+        "image.go",
+        "upgrade.go",
+    ],
+    importpath = "github.com/edgelesssys/constellation/v2/e2e/internal/upgrade",
+    visibility = ["//e2e:__subpackages__"],
+    deps = [
+        "//internal/attestation/measurements",
+        "//internal/cloud/cloudprovider",
+        "//internal/constants",
+        "//internal/logger",
+        "//internal/versionsapi",
+        "//internal/versionsapi/fetcher",
+        "@sh_helm_helm_v3//pkg/action",
+        "@sh_helm_helm_v3//pkg/cli",
+    ],
+)
+
+go_test(
+    name = "upgrade_test",
+    srcs = ["upgrade_test.go"],
+    count = 1,
+    data = [
+        "//cli:cli_oss_linux_amd64",
+    ],
+    embed = [":upgrade"],
+    env = {
+        "PATH_CLI": "$(location //cli:cli_oss_linux_amd64)",
+    },
+    gotags = ["e2e"],
+    tags = ["manual"],
+    deps = [
+        "//e2e/internal/kubectl",
+        "//internal/config",
+        "//internal/constants",
+        "//internal/file",
+        "//internal/semver",
+        "@com_github_spf13_afero//:afero",
+        "@com_github_stretchr_testify//require",
+        "@io_k8s_api//core/v1:core",
+        "@io_k8s_apimachinery//pkg/apis/meta/v1:meta",
+        "@io_k8s_client_go//kubernetes",
+    ],
+)

e2e/internal/upgrade/helm.go

@@ -0,0 +1,54 @@
+//go:build e2e
+
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package upgrade
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/edgelesssys/constellation/v2/internal/constants"
+	"github.com/edgelesssys/constellation/v2/internal/logger"
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/cli"
+)
+
+func servicesVersion(t *testing.T) (string, error) {
+	t.Helper()
+	log := logger.NewTest(t)
+	settings := cli.New()
+	settings.KubeConfig = "constellation-admin.conf"
+	actionConfig := &action.Configuration{}
+	if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace, "secret", log.Infof); err != nil {
+		return "", fmt.Errorf("initializing config: %w", err)
+	}
+
+	return currentVersion(actionConfig, "constellation-services")
+}
+
+func currentVersion(cfg *action.Configuration, release string) (string, error) {
+	action := action.NewList(cfg)
+	action.Filter = release
+	rel, err := action.Run()
+	if err != nil {
+		return "", err
+	}
+
+	if len(rel) == 0 {
+		return "", fmt.Errorf("release %s not found", release)
+	}
+	if len(rel) > 1 {
+		return "", fmt.Errorf("multiple releases found for %s", release)
+	}
+
+	if rel[0] == nil || rel[0].Chart == nil || rel[0].Chart.Metadata == nil {
+		return "", fmt.Errorf("received invalid release %s", release)
+	}
+
+	return rel[0].Chart.Metadata.Version, nil
+}

e2e/internal/upgrade/image.go

@@ -0,0 +1,91 @@
+//go:build e2e
+
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package upgrade
+
+import (
+	"context"
+	"errors"
+	"net/http"
+
+	"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
+	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
+	"github.com/edgelesssys/constellation/v2/internal/versionsapi"
+	"github.com/edgelesssys/constellation/v2/internal/versionsapi/fetcher"
+)
+
+type upgradeInfo struct {
+	measurements measurements.M
+	shortPath    string
+	wantImage    string
+}
+
+func fetchUpgradeInfo(ctx context.Context, csp cloudprovider.Provider, toImage string) (upgradeInfo, error) {
+	info := upgradeInfo{
+		measurements: make(measurements.M),
+		shortPath:    toImage,
+	}
+	versionsClient := fetcher.NewFetcher()
+
+	ver, err := versionsapi.NewVersionFromShortPath(toImage, versionsapi.VersionKindImage)
+	if err != nil {
+		return upgradeInfo{}, err
+	}
+
+	measurementsURL, signatureURL, err := versionsapi.MeasurementURL(ver, csp)
+	if err != nil {
+		return upgradeInfo{}, err
+	}
+
+	var fetchedMeasurements measurements.M
+	_, err = fetchedMeasurements.FetchAndVerify(
+		ctx, http.DefaultClient,
+		measurementsURL,
+		signatureURL,
+		[]byte(constants.CosignPublicKey),
+		measurements.WithMetadata{
+			CSP:   csp,
+			Image: toImage,
+		},
+	)
+	if err != nil {
+		return upgradeInfo{}, err
+	}
+	info.measurements = fetchedMeasurements
+
+	wantImage, err := fetchWantImage(ctx, versionsClient, csp, versionsapi.ImageInfo{
+		Ref:     ver.Ref,
+		Stream:  ver.Stream,
+		Version: ver.Version,
+	})
+	if err != nil {
+		return upgradeInfo{}, err
+	}
+	info.wantImage = wantImage
+
+	return info, nil
+}
+
+func fetchWantImage(ctx context.Context, client *fetcher.Fetcher, csp cloudprovider.Provider, imageInfo versionsapi.ImageInfo) (string, error) {
+	imageInfo, err := client.FetchImageInfo(ctx, imageInfo)
+	if err != nil {
+		return "", err
+	}
+
+	switch csp {
+	case cloudprovider.GCP:
+		return imageInfo.GCP["sev-es"], nil
+	case cloudprovider.Azure:
+		return imageInfo.Azure["cvm"], nil
+	case cloudprovider.AWS:
+		return imageInfo.AWS["eu-central-1"], nil
+	default:
+		return "", errors.New("finding wanted image")
+	}
+}

e2e/internal/upgrade/upgrade.go

@@ -0,0 +1,19 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+// Package upgrade tests that the CLI's upgrade apply command works as expected and
+// the operators eventually upgrade all nodes inside the cluster.
+// The test is written as a go test because:
+// 1. the helm cli does not directly provide the chart version of a release
+//
+// 2. the image patch needs to be parsed from the image-api's info.json
+//
+// 3. there is some logic required to setup the test correctly:
+//
+//   - set microservice, k8s version correctly depending on input
+//
+//   - set or fetch measurements depending on target image
+package upgrade

e2e/internal/upgrade/upgrade_test.go

@@ -0,0 +1,327 @@
+//go:build e2e
+
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+
+package upgrade
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/edgelesssys/constellation/v2/e2e/internal/kubectl"
+	"github.com/edgelesssys/constellation/v2/internal/config"
+	"github.com/edgelesssys/constellation/v2/internal/constants"
+	"github.com/edgelesssys/constellation/v2/internal/file"
+	"github.com/edgelesssys/constellation/v2/internal/semver"
+	"github.com/spf13/afero"
+	"github.com/stretchr/testify/require"
+	coreV1 "k8s.io/api/core/v1"
+	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Flags are defined globally as `go test` implicitly calls flag.Parse() before executing a testcase.
+// Thus defining and parsing flags inside a testcase would result in a panic.
+// See https://groups.google.com/g/golang-nuts/c/P6EdEdgvDuc/m/5-Dg6bPxmvQJ.
+var (
+	targetImage         = flag.String("target-image", "", "Image (shortversion) to upgrade to.")
+	targetKubernetes    = flag.String("target-kubernetes", "", "Kubernetes version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.")
+	targetMicroservices = flag.String("target-microservices", "", "Microservice version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.")
+	// When executing the test as a bazel target the workspace path is supplied through an env variable that bazel sets.
+	workspace = flag.String("workspace", "", "Constellation workspace in which to run the tests.")
+	// When executing the test as a bazel target the CLI path is supplied through an env variable that bazel sets.
+	// When executing via `go test` extra care should be taken that the supplied CLI is built on the same commit as this test.
+	cliPath     = flag.String("cli", "", "Constellation CLI to run the tests.")
+	wantWorker  = flag.Int("want-worker", 0, "Number of wanted worker nodes.")
+	wantControl = flag.Int("want-control", 0, "Number of wanted control nodes.")
+	timeout     = flag.Duration("timeout", 3*time.Hour, "Timeout after which the cluster should have converged to the target version.")
+)
+
+// setup checks that the prerequisites for the test are met:
+// - a workspace is set
+// - a CLI path is set
+// - the constellation-upgrade folder does not exist.
+func setup() error {
+	workingDir, err := workingDir(*workspace)
+	if err != nil {
+		return fmt.Errorf("getting working directory: %w", err)
+	}
+
+	if err := os.Chdir(workingDir); err != nil {
+		return fmt.Errorf("changing working directory: %w", err)
+	}
+
+	if _, err := getCLIPath(*cliPath); err != nil {
+		return fmt.Errorf("getting CLI path: %w", err)
+	}
+	if _, err := os.Stat("constellation-upgrade"); err == nil {
+		return errors.New("please remove the existing constellation-upgrade folder")
+	}
+
+	return nil
+}
+
+// TestUpgrade checks that the workspace's kubeconfig points to a healthy cluster,
+// we can write an upgrade config, we can trigger an upgrade
+// and the cluster eventually upgrades to the target version.
+func TestUpgrade(t *testing.T) {
+	require := require.New(t)
+
+	err := setup()
+	require.NoError(err)
+
+	k, err := kubectl.New()
+	require.NoError(err)
+	require.NotNil(k)
+
+	require.NotEqual(*targetImage, "", "--target-image needs to be specified")
+
+	testNodesEventuallyAvailable(t, k, *wantControl, *wantWorker)
+	testPodsEventuallyReady(t, k, "kube-system")
+
+	targetVersions := writeUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices)
+
+	cli, err := getCLIPath(*cliPath)
+	require.NoError(err)
+
+	data, err := os.ReadFile("./constellation-conf.yaml")
+	require.NoError(err)
+	log.Println(string(data))
+
+	log.Println("Triggering upgrade.")
+	cmd := exec.CommandContext(context.Background(), cli, "upgrade", "apply", "--force", "--debug")
+	msg, err := cmd.CombinedOutput()
+	require.NoErrorf(err, "%s", string(msg))
+	require.NoError(containsUnexepectedMsg(string(msg)))
+	log.Println(string(msg))
+
+	testMicroservicesEventuallyHaveVersion(t, targetVersions.microservices, *timeout)
+	testNodesEventuallyHaveVersion(t, k, targetVersions, *wantControl+*wantWorker, *timeout)
+}
+
+// workingDir returns the path to the workspace.
+func workingDir(workspace string) (string, error) {
+	workingDir := os.Getenv("BUILD_WORKING_DIRECTORY")
+	switch {
+	case workingDir != "":
+		return workingDir, nil
+	case workspace != "":
+		return workspace, nil
+	default:
+		return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set")
+	}
+}
+
+// getCLIPath returns the path to the CLI.
+func getCLIPath(cliPath string) (string, error) {
+	pathCLI := os.Getenv("PATH_CLI")
+	switch {
+	case pathCLI != "":
+		return pathCLI, nil
+	case cliPath != "":
+		return cliPath, nil
+	default:
+		return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set")
+	}
+}
+
+// containsUnexepectedMsg checks if the given input contains any unexpected messages.
+// unexepcted messages are:
+// "Skipping image & Kubernetes upgrades. Another upgrade is in progress".
+func containsUnexepectedMsg(input string) error {
+	if strings.Contains(input, "Skipping image & Kubernetes upgrades. Another upgrade is in progress") {
+		return errors.New("unexpected upgrade in progress")
+	}
+	return nil
+}
+
+func writeUpgradeConfig(require *require.Assertions, image string, kubernetes string, microservices string) versionContainer {
+	fileHandler := file.NewHandler(afero.NewOsFs())
+	cfg, err := config.New(fileHandler, constants.ConfigFilename, true)
+	require.NoError(err)
+
+	info, err := fetchUpgradeInfo(context.Background(), cfg.GetProvider(), image)
+	require.NoError(err)
+
+	log.Printf("Setting image version: %s\n", info.shortPath)
+	cfg.Image = info.shortPath
+	cfg.UpdateMeasurements(info.measurements)
+
+	defaultConfig := config.Default()
+	var kubernetesVersion semver.Semver
+	if kubernetes == "" {
+		kubernetesVersion, err = semver.New(defaultConfig.KubernetesVersion)
+		require.NoError(err)
+	} else {
+		kubernetesVersion, err = semver.New(kubernetes)
+		require.NoError(err)
+	}
+
+	var microserviceVersion string
+	if microservices == "" {
+		microserviceVersion = defaultConfig.MicroserviceVersion
+	} else {
+		microserviceVersion = microservices
+	}
+
+	log.Printf("Setting K8s version: %s\n", kubernetesVersion.String())
+	cfg.KubernetesVersion = kubernetesVersion.String()
+	log.Printf("Setting microservice version: %s\n", microserviceVersion)
+	cfg.MicroserviceVersion = microserviceVersion
+
+	err = fileHandler.WriteYAML(constants.ConfigFilename, cfg, file.OptOverwrite)
+	require.NoError(err)
+
+	return versionContainer{image: info.wantImage, kubernetes: kubernetesVersion, microservices: microserviceVersion}
+}
+
+func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion string, timeout time.Duration) {
+	require.Eventually(t, func() bool {
+		version, err := servicesVersion(t)
+		if err != nil {
+			log.Printf("Unable to fetch microservice version: %v\n", err)
+			return false
+		}
+
+		if version != wantMicroserviceVersion {
+			log.Printf("Microservices still at version: %v\n", version)
+			return false
+		}
+
+		return true
+	}, timeout, time.Minute)
+}
+
+func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions versionContainer, totalNodeCount int, timeout time.Duration) {
+	require.Eventually(t, func() bool {
+		nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{})
+		if err != nil {
+			log.Println(err)
+			return false
+		}
+		require.False(t, len(nodes.Items) < totalNodeCount, "expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items))
+
+		allUpdated := true
+		log.Printf("Node status (%v):", time.Now())
+		for _, node := range nodes.Items {
+			for key, value := range node.Annotations {
+				if key == "constellation.edgeless.systems/node-image" {
+					log.Printf("\t%s: Image %s\n", node.Name, value)
+					if value != targetVersions.image {
+						allUpdated = false
+					}
+				}
+			}
+
+			kubeletVersion := node.Status.NodeInfo.KubeletVersion
+			if kubeletVersion != targetVersions.kubernetes.String() {
+				log.Printf("\t%s: K8s (Kubelet) %s\n", node.Name, kubeletVersion)
+				allUpdated = false
+			}
+			kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion
+			if kubeProxyVersion != targetVersions.kubernetes.String() {
+				log.Printf("\t%s: K8s (Proxy) %s\n", node.Name, kubeProxyVersion)
+				allUpdated = false
+			}
+		}
+
+		return allUpdated
+	}, timeout, time.Minute)
+}
+
+// testPodsEventuallyReady checks that:
+// 1) all pods are running.
+// 2) all pods have good status conditions.
+func testPodsEventuallyReady(t *testing.T, k *kubernetes.Clientset, namespace string) {
+	require.Eventually(t, func() bool {
+		pods, err := k.CoreV1().Pods(namespace).List(context.Background(), metaV1.ListOptions{})
+		if err != nil {
+			log.Println(err)
+			return false
+		}
+
+		for _, pod := range pods.Items {
+			if pod.Status.Phase != coreV1.PodRunning {
+				log.Printf("Pod %s is not running, but %s\n", pod.Name, pod.Status.Phase)
+				return false
+			}
+
+			for _, condition := range pod.Status.Conditions {
+				switch condition.Type {
+				case coreV1.ContainersReady, coreV1.PodInitialized, coreV1.PodReady, coreV1.PodScheduled:
+					if condition.Status != coreV1.ConditionTrue {
+						log.Printf("Pod %s's status %s is false\n", pod.Name, coreV1.ContainersReady)
+						return false
+					}
+				}
+			}
+		}
+		return true
+	}, time.Minute*30, time.Minute)
+}
+
+// testNodesEventuallyAvailable checks that:
+// 1) all nodes only have good status conditions.
+// 2) the expected number of nodes have joined the cluster.
+func testNodesEventuallyAvailable(t *testing.T, k *kubernetes.Clientset, wantControlNodeCount, wantWorkerNodeCount int) {
+	require.Eventually(t, func() bool {
+		nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{})
+		if err != nil {
+			log.Println(err)
+			return false
+		}
+
+		var controlNodeCount, workerNodeCount int
+		for _, node := range nodes.Items {
+			if _, ok := node.Labels["node-role.kubernetes.io/control-plane"]; ok {
+				controlNodeCount++
+			} else {
+				workerNodeCount++
+			}
+
+			for _, condition := range node.Status.Conditions {
+				switch condition.Type {
+				case coreV1.NodeReady:
+					if condition.Status != coreV1.ConditionTrue {
+						fmt.Printf("Status %s for node %s is %s\n", condition.Type, node.Name, condition.Status)
+						return false
+					}
+				case coreV1.NodeMemoryPressure, coreV1.NodeDiskPressure, coreV1.NodePIDPressure, coreV1.NodeNetworkUnavailable:
+					if condition.Status != coreV1.ConditionFalse {
+						fmt.Printf("Status %s for node %s is %s\n", condition.Type, node.Name, condition.Status)
+						return false
+					}
+				}
+			}
+		}
+
+		if controlNodeCount != wantControlNodeCount {
+			log.Printf("Want %d control nodes but got %d\n", wantControlNodeCount, controlNodeCount)
+			return false
+		}
+		if workerNodeCount != wantWorkerNodeCount {
+			log.Printf("Want %d worker nodes but got %d\n", wantWorkerNodeCount, workerNodeCount)
+			return false
+		}
+
+		return true
+	}, time.Minute*30, time.Minute)
+}
+
+type versionContainer struct {
+	image         string
+	kubernetes    semver.Semver
+	microservices string
+}

internal/attestation/measurements/measurement-generator/generate.go

@@ -143,7 +143,7 @@ func measurementURL(provider cloudprovider.Provider, image, file string) (*url.U
 	}
 
 	return url.Parse(
-		version.ArtifactURL() + path.Join("/image", "csp", strings.ToLower(provider.String()), file),
+		version.ArtifactsURL() + path.Join("/image", "csp", strings.ToLower(provider.String()), file),
 	)
 }
 

internal/constants/constants.go

@@ -181,6 +181,10 @@ const (
 	CDNRepositoryURL = "https://cdn.confidential.cloud"
 	// CDNAPIPrefix is the prefix of the Constellation API.
 	CDNAPIPrefix = "constellation/v1"
+	// CDNMeasurementsFile is name of file containing image measurements.
+	CDNMeasurementsFile = "measurements.json"
+	// CDNMeasurementsSignature is name of file containing signature for CDNMeasurementsFile.
+	CDNMeasurementsSignature = "measurements.json.sig"
 )
 
 // VersionInfo returns the version of a binary.

internal/semver/semver.go

@@ -25,8 +25,8 @@ type Semver struct {
 	Patch int
 }
 
-// NewSemver returns a Version from a string.
-func NewSemver(version string) (Semver, error) {
+// New returns a Version from a string.
+func New(version string) (Semver, error) {
 	// ensure that semver has "v" prefix
 	if !strings.HasPrefix(version, "v") {
 		version = "v" + version
@@ -70,7 +70,7 @@ func (v Semver) IsUpgradeTo(other Semver) bool {
 // CompatibleWithBinary returns if a version is compatible version of the current built binary.
 // It checks if the version of the binary is equal or greater than the current version and allows a drift of at most one minor version.
 func (v Semver) CompatibleWithBinary() bool {
-	binaryVersion, err := NewSemver(constants.VersionInfo())
+	binaryVersion, err := New(constants.VersionInfo())
 	if err != nil {
 		return false
 	}
@@ -95,7 +95,7 @@ func (v *Semver) UnmarshalJSON(data []byte) error {
 		return err
 	}
 
-	version, err := NewSemver(s)
+	version, err := New(s)
 	if err != nil {
 		return err
 	}

internal/semver/semver_test.go

@@ -37,7 +37,7 @@ func TestNewVersion(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			assert := assert.New(t)
 
-			_, err := NewSemver(tc.version)
+			_, err := New(tc.version)
 			if tc.wantErr {
 				assert.Error(err)
 			} else {

internal/versionsapi/BUILD.bazel

@@ -14,6 +14,7 @@ go_library(
     importpath = "github.com/edgelesssys/constellation/v2/internal/versionsapi",
     visibility = ["//:__subpackages__"],
     deps = [
+        "//internal/cloud/cloudprovider",
         "//internal/constants",
         "@org_golang_x_mod//semver",
     ],
@@ -30,6 +31,7 @@ go_test(
     ],
     embed = [":versionsapi"],
     deps = [
+        "//internal/cloud/cloudprovider",
         "//internal/constants",
         "@com_github_stretchr_testify//assert",
         "@com_github_stretchr_testify//require",

internal/versionsapi/version.go

@@ -10,10 +10,12 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net/url"
 	"path"
 	"regexp"
 	"strings"
 
+	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"golang.org/x/mod/semver"
 )
@@ -142,9 +144,9 @@ func (v Version) ListPath(gran Granularity) string {
 	)
 }
 
-// ArtifactURL returns the URL to the artifacts stored for this version.
+// ArtifactsURL returns the URL to the artifacts stored for this version.
 // The URL points to a directory.
-func (v Version) ArtifactURL() string {
+func (v Version) ArtifactsURL() string {
 	return constants.CDNRepositoryURL + "/" + v.ArtifactPath()
 }
 
@@ -323,6 +325,33 @@ func ValidateStream(ref, stream string) error {
 	return fmt.Errorf("stream %q is unknown or not supported on ref %q", stream, ref)
 }
 
+// MeasurementURL builds the measurement and signature URLs for the given version and CSP.
+func MeasurementURL(version Version, csp cloudprovider.Provider) (measurementURL, signatureURL *url.URL, err error) {
+	if version.Kind != VersionKindImage {
+		return &url.URL{}, &url.URL{}, fmt.Errorf("kind %q is not supported", version.Kind)
+	}
+
+	measurementPath, err := url.JoinPath(version.ArtifactsURL(), "image", "csp", strings.ToLower(csp.String()), constants.CDNMeasurementsFile)
+	if err != nil {
+		return &url.URL{}, &url.URL{}, fmt.Errorf("joining path for measurement: %w", err)
+	}
+	signaturePath, err := url.JoinPath(version.ArtifactsURL(), "image", "csp", strings.ToLower(csp.String()), constants.CDNMeasurementsSignature)
+	if err != nil {
+		return &url.URL{}, &url.URL{}, fmt.Errorf("joining path for signature: %w", err)
+	}
+
+	measurementURL, err = url.Parse(measurementPath)
+	if err != nil {
+		return &url.URL{}, &url.URL{}, fmt.Errorf("parsing path for measurement: %w", err)
+	}
+
+	signatureURL, err = url.Parse(signaturePath)
+	if err != nil {
+		return &url.URL{}, &url.URL{}, fmt.Errorf("parsing path for signature: %w", err)
+	}
+	return measurementURL, signatureURL, nil
+}
+
 var (
 	shortPathRegex        = regexp.MustCompile(`^ref/([a-zA-Z0-9-]+)/stream/([a-zA-Z0-9-]+)/([a-zA-Z0-9.-]+)$`)
 	shortPathReleaseRegex = regexp.MustCompile(`^stream/([a-zA-Z0-9-]+)/([a-zA-Z0-9.-]+)$`)

internal/versionsapi/version_test.go

@@ -10,6 +10,7 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
 	"github.com/edgelesssys/constellation/v2/internal/constants"
 	"github.com/stretchr/testify/assert"
 )
@@ -451,6 +452,49 @@ func TestVersionListPathURL(t *testing.T) {
 	}
 }
 
+func TestVersionArtifactURL(t *testing.T) {
+	testCases := map[string]struct {
+		ver                Version
+		csp                cloudprovider.Provider
+		wantMeasurementURL string
+		wantSignatureURL   string
+		wantErr            bool
+	}{
+		"nightly-feature": {
+			ver: Version{
+				Ref:     "feat-some-feature",
+				Stream:  "nightly",
+				Version: "v2.6.0-pre.0.20230217095603-193dd48ca19f",
+				Kind:    VersionKindImage,
+			},
+			csp:                cloudprovider.GCP,
+			wantMeasurementURL: constants.CDNRepositoryURL + "/" + constants.CDNAPIPrefix + "/ref/feat-some-feature/stream/nightly/v2.6.0-pre.0.20230217095603-193dd48ca19f/image/csp/gcp/measurements.json",
+			wantSignatureURL:   constants.CDNRepositoryURL + "/" + constants.CDNAPIPrefix + "/ref/feat-some-feature/stream/nightly/v2.6.0-pre.0.20230217095603-193dd48ca19f/image/csp/gcp/measurements.json.sig",
+		},
+		"fail for wrong kind": {
+			ver: Version{
+				Kind: VersionKindCLI,
+			},
+			wantErr: true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+
+			measurementURL, signatureURL, err := MeasurementURL(tc.ver, tc.csp)
+			if tc.wantErr {
+				assert.Error(err)
+				return
+			}
+			assert.NoError(err)
+			assert.Equal(tc.wantMeasurementURL, measurementURL.String())
+			assert.Equal(tc.wantSignatureURL, signatureURL.String())
+		})
+	}
+}
+
 func TestVersionArtifactPathURL(t *testing.T) {
 	testCases := map[string]struct {
 		ver      Version
@@ -518,7 +562,7 @@ func TestVersionArtifactPathURL(t *testing.T) {
 
 			path := tc.ver.ArtifactPath()
 			assert.Equal(tc.wantPath, path)
-			url := tc.ver.ArtifactURL()
+			url := tc.ver.ArtifactsURL()
 			assert.Equal(constants.CDNRepositoryURL+"/"+tc.wantPath, url)
 		})
 	}