From 75d65e286789e5210972269892d76c6f1230ac04 Mon Sep 17 00:00:00 2001 From: Adrian Stobbe Date: Wed, 3 Jan 2024 10:00:52 +0100 Subject: [PATCH] add upgrade to test --- .../workflows/e2e-test-provider-example.yml | 151 ++++++++- e2e/internal/upgrade/BUILD.bazel | 17 +- e2e/internal/upgrade/upgrade.go | 312 ++++++++++++++++++ e2e/internal/upgrade/upgrade_test.go | 276 +--------------- e2e/provider-upgrade/BUILD.bazel | 15 + e2e/provider-upgrade/upgrade_test.go | 60 ++++ .../examples/full/aws/main.tf | 4 +- .../examples/full/azure/main.tf | 4 +- .../examples/full/gcp/main.tf | 4 +- 9 files changed, 554 insertions(+), 289 deletions(-) create mode 100644 e2e/provider-upgrade/BUILD.bazel create mode 100644 e2e/provider-upgrade/upgrade_test.go diff --git a/.github/workflows/e2e-test-provider-example.yml b/.github/workflows/e2e-test-provider-example.yml index 5987a7d6f..07ac5ff5a 100644 --- a/.github/workflows/e2e-test-provider-example.yml +++ b/.github/workflows/e2e-test-provider-example.yml @@ -23,6 +23,14 @@ on: providerVersion: description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." type: string + toImage: + description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. + type: string + required: false + toKubernetes: + description: Kubernetes version to target for the upgrade, empty for no upgrade. + type: string + required: false workflow_call: inputs: ref: @@ -41,6 +49,14 @@ on: providerVersion: description: "Constellation Terraform provider version to use (with v prefix). Empty value means build from source." type: string + toImage: + description: Image (shortpath) the cluster is upgraded to, or empty for main/nightly. + type: string + required: false + toKubernetes: + description: Kubernetes version to target for the upgrade, empty for target's default version. + type: string + required: false jobs: provider-example-test: @@ -197,15 +213,18 @@ jobs: } } } + locals { name = "${{ steps.create-prefix.outputs.prefix }}" - version = "${image_version}" + image_version = "${image_version}" microservice_version= "${prefixed_version}" kubernetes_version = "${kubernetes_version}" } + module "${{ inputs.cloudProvider }}_iam" { source = "${iam_src}" } + module "${{ inputs.cloudProvider }}_infrastructure" { source = "${infra_src}" } @@ -259,11 +278,137 @@ jobs: terraform apply -target module.azure_iam -auto-approve terraform apply -target module.azure_infrastructure -auto-approve ../build/constellation maa-patch "$(terraform output -raw maa_url)" - TF_LOG=INFO terraform apply -target constellation_cluster.azure_example -auto-approve + terraform apply -target constellation_cluster.azure_example -auto-approve else - TF_LOG=INFO terraform apply -auto-approve + terraform apply -auto-approve fi + - name: Cleanup Terraform Cluster on failure + # cleanup here already on failure, because the subsequent TF overrides might make the TF config invalid and thus the destroy would fail later + # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) + if: failure() && steps.apply_terraform.outcome != 'skipped' + working-directory: ${{ github.workspace }}/cluster + shell: bash + run: | + terraform init + terraform destroy -auto-approve + + - name: Add Provider to local Terraform registry # needed if release version was used before + if: inputs.providerVersion != '' + working-directory: ${{ github.workspace }}/build + shell: bash + run: | + bazel run //:devbuild --cli_edition=enterprise + + - name: Update cluster configuration # for duplicate variable declaration, the last one is used + working-directory: ${{ github.workspace }}/cluster + shell: bash + run: | + if [[ "${{ inputs.toImage }}" != "" ]]; then + cat >> _override.tf <> _override.tf <> _override.tf <> _override.tf < constellation-admin.conf + + if [[ -n "${MICROSERVICES}" ]]; then + MICROSERVICES_FLAG="--target-microservices=${MICROSERVICES}" + fi + if [[ -n "${KUBERNETES}" ]]; then + KUBERNETES_FLAG="--target-kubernetes=${KUBERNETES}" + fi + if [[ -n "${IMAGE}" ]]; then + IMAGE_FLAG="--target-image=${IMAGE}" + fi + + # cfg must be in same dir as KUBECONFIG + ../build/constellation config generate "${{ inputs.cloudProvider }}" + # make cfg valid with fake data + # IMPORTANT: zone needs to be correct because it is used to resolve the CSP image ref + if [[ "${{ inputs.cloudProvider }}" == "azure" ]]; then + location="${{ inputs.regionZone || 'northeurope' }}" + yq e ".provider.azure.location = \"${location}\"" -i constellation-conf.yaml + + yq e '.provider.azure.subscription = "123e4567-e89b-12d3-a456-426614174000"' -i constellation-conf.yaml + yq e '.provider.azure.tenant = "123e4567-e89b-12d3-a456-426614174001"' -i constellation-conf.yaml + yq e '.provider.azure.resourceGroup = "myResourceGroup"' -i constellation-conf.yaml + yq e '.provider.azure.userAssignedIdentity = "myIdentity"' -i constellation-conf.yaml + fi + if [[ "${{ inputs.cloudProvider }}" == "gcp" ]]; then + zone="${{ inputs.regionZone || 'europe-west3-b' }}" + region=$(echo "${zone}" | rev | cut -c 2- | rev) + yq e ".provider.gcp.region = \"${region}\"" -i constellation-conf.yaml + yq e ".provider.gcp.zone = \"${zone}\"" -i constellation-conf.yaml + + yq e '.provider.gcp.project = "demo-gcp-project"' -i constellation-conf.yaml + yq e '.nodeGroups.control_plane_default.zone = "europe-west3-b"' -i constellation-conf.yaml + # Set the zone for worker_default node group to a fictional value + yq e '.nodeGroups.worker_default.zone = "europe-west3-b"' -i constellation-conf.yaml + yq e '.provider.gcp.serviceAccountKeyPath = "/path/to/your/service-account-key.json"' -i constellation-conf.yaml + fi + if [[ "${{ inputs.cloudProvider }}" == "aws" ]]; then + zone=${{ inputs.regionZone || 'us-east-2c' }} + region=$(echo "${zone}" | rev | cut -c 2- | rev) + yq e ".provider.aws.region = \"${region}\"" -i constellation-conf.yaml + yq e ".provider.aws.zone = \"${zone}\"" -i constellation-conf.yaml + + yq e '.provider.aws.iamProfileControlPlane = "demoControlPlaneIAMProfile"' -i constellation-conf.yaml + yq e '.provider.aws.iamProfileWorkerNodes = "demoWorkerNodesIAMProfile"' -i constellation-conf.yaml + yq e '.nodeGroups.control_plane_default.zone = "eu-central-1a"' -i constellation-conf.yaml + yq e '.nodeGroups.worker_default.zone = "eu-central-1a"' -i constellation-conf.yaml + fi + + KUBECONFIG=${{ github.workspace }}/cluster/constellation-admin.conf bazel run //e2e/provider-upgrade:provider-upgrade_test -- --want-worker "$WORKERNODES" --want-control "$CONTROLNODES" --cli "${{ github.workspace }}/build/constellation" "$IMAGE_FLAG" "$KUBERNETES_FLAG" "$MICROSERVICES_FLAG" + - name: Destroy Terraform Cluster # outcome is part of the steps context (https://docs.github.com/en/actions/learn-github-actions/contexts#steps-context) if: always() && steps.apply_terraform.outcome != 'skipped' diff --git a/e2e/internal/upgrade/BUILD.bazel b/e2e/internal/upgrade/BUILD.bazel index 2d16064bd..6e368e94f 100644 --- a/e2e/internal/upgrade/BUILD.bazel +++ b/e2e/internal/upgrade/BUILD.bazel @@ -10,9 +10,19 @@ go_library( importpath = "github.com/edgelesssys/constellation/v2/e2e/internal/upgrade", visibility = ["//e2e:__subpackages__"], deps = [ + "//internal/api/attestationconfigapi", + "//internal/config", "//internal/constants", + "//internal/file", + "//internal/imagefetcher", "//internal/logger", "//internal/semver", + "//internal/versions", + "@com_github_spf13_afero//:afero", + "@com_github_stretchr_testify//require", + "@io_bazel_rules_go//go/runfiles:go_default_library", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_client_go//kubernetes", "@sh_helm_helm_v3//pkg/action", "@sh_helm_helm_v3//pkg/cli", ], @@ -35,16 +45,9 @@ go_test( tags = ["manual"], deps = [ "//e2e/internal/kubectl", - "//internal/api/attestationconfigapi", - "//internal/config", "//internal/constants", - "//internal/file", - "//internal/imagefetcher", - "//internal/semver", "//internal/versions", - "@com_github_spf13_afero//:afero", "@com_github_stretchr_testify//require", - "@io_bazel_rules_go//go/runfiles:go_default_library", "@io_k8s_api//core/v1:core", "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", "@io_k8s_client_go//kubernetes", diff --git a/e2e/internal/upgrade/upgrade.go b/e2e/internal/upgrade/upgrade.go index 6fa503655..69cf53284 100644 --- a/e2e/internal/upgrade/upgrade.go +++ b/e2e/internal/upgrade/upgrade.go @@ -1,3 +1,5 @@ +//go:build e2e + /* Copyright (c) Edgeless Systems GmbH @@ -17,3 +19,313 @@ SPDX-License-Identifier: AGPL-3.0-only // // - set or fetch measurements depending on target image package upgrade + +import ( + "bufio" + "context" + "errors" + "fmt" + "io" + "log" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/bazelbuild/rules_go/go/runfiles" + "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" + "github.com/edgelesssys/constellation/v2/internal/config" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/edgelesssys/constellation/v2/internal/file" + "github.com/edgelesssys/constellation/v2/internal/imagefetcher" + "github.com/edgelesssys/constellation/v2/internal/semver" + "github.com/edgelesssys/constellation/v2/internal/versions" + "github.com/spf13/afero" + "github.com/stretchr/testify/require" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" +) + +// tickDuration is the duration between two checks to see if the upgrade is successful. +var tickDuration = 10 * time.Second // small tick duration to speed up tests + +// VersionContainer contains the versions that the cluster should be upgraded to. +type VersionContainer struct { + ImageRef string + Kubernetes versions.ValidK8sVersion + Microservices semver.Semver +} + +// AssertUpgradeSuccessful tests that the upgrade to the target version is successful. +func AssertUpgradeSuccessful(t *testing.T, cli string, targetVersions VersionContainer, k *kubernetes.Clientset, wantControl, wantWorker int, timeout time.Duration) { + wg := queryStatusAsync(t, cli) + require.NotNil(t, k) + + testMicroservicesEventuallyHaveVersion(t, targetVersions.Microservices, timeout) + fmt.Println("Microservices are upgraded.") + + testNodesEventuallyHaveVersion(t, k, targetVersions, wantControl+wantWorker, timeout) + fmt.Println("Nodes are upgraded.") + wg.Wait() +} + +func queryStatusAsync(t *testing.T, cli string) *sync.WaitGroup { + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + // The first control plane node should finish upgrading after 20 minutes. If it does not, something is fishy. + // Nodes can upgrade in <5mins. + testStatusEventuallyWorks(t, cli, 20*time.Minute) + }() + + return &wg +} + +func testStatusEventuallyWorks(t *testing.T, cli string, timeout time.Duration) { + require.Eventually(t, func() bool { + // Show versions set in cluster. + // The string after "Cluster status:" in the output might not be updated yet. + // This is only updated after the operator finishes one reconcile loop. + cmd := exec.CommandContext(context.Background(), cli, "status") + stdout, stderr, err := runCommandWithSeparateOutputs(cmd) + if err != nil { + log.Printf("Stdout: %s\nStderr: %s", string(stdout), string(stderr)) + return false + } + + log.Println(string(stdout)) + return true + }, timeout, tickDuration) +} + +func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion semver.Semver, timeout time.Duration) { + require.Eventually(t, func() bool { + version, err := servicesVersion(t) + if err != nil { + log.Printf("Unable to fetch microservice version: %v\n", err) + return false + } + + if version != wantMicroserviceVersion { + log.Printf("Microservices still at version %v, want %v\n", version, wantMicroserviceVersion) + return false + } + + return true + }, timeout, tickDuration) +} + +func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions VersionContainer, totalNodeCount int, timeout time.Duration) { + require.Eventually(t, func() bool { + nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{}) + if err != nil { + log.Println(err) + return false + } + + // require is not printed in the logs, so we use fmt + tooSmallNodeCount := len(nodes.Items) < totalNodeCount + if tooSmallNodeCount { + log.Printf("expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items)) + return false + } + + allUpdated := true + log.Printf("Node status (%v):", time.Now()) + for _, node := range nodes.Items { + for key, value := range node.Annotations { + if targetVersions.ImageRef != "" { + if key == "constellation.edgeless.systems/node-image" { + if !strings.EqualFold(value, targetVersions.ImageRef) { + log.Printf("\t%s: Image %s, want %s\n", node.Name, value, targetVersions.ImageRef) + fmt.Printf("\tP: %s: Image %s, want %s\n", node.Name, value, targetVersions.ImageRef) + allUpdated = false + } + } + } + } + if targetVersions.Kubernetes != "" { + kubeletVersion := node.Status.NodeInfo.KubeletVersion + if kubeletVersion != string(targetVersions.Kubernetes) { + log.Printf("\t%s: K8s (Kubelet) %s, want %s\n", node.Name, kubeletVersion, targetVersions.Kubernetes) + allUpdated = false + } + kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion + if kubeProxyVersion != string(targetVersions.Kubernetes) { + log.Printf("\t%s: K8s (Proxy) %s, want %s\n", node.Name, kubeProxyVersion, targetVersions.Kubernetes) + allUpdated = false + } + } + } + return allUpdated + }, timeout, tickDuration) +} + +// runCommandWithSeparateOutputs runs the given command while separating buffers for +// stdout and stderr. +func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) { + stdout = []byte{} + stderr = []byte{} + + stdoutIn, err := cmd.StdoutPipe() + if err != nil { + err = fmt.Errorf("create stdout pipe: %w", err) + return + } + stderrIn, err := cmd.StderrPipe() + if err != nil { + err = fmt.Errorf("create stderr pipe: %w", err) + return + } + + err = cmd.Start() + if err != nil { + err = fmt.Errorf("start command: %w", err) + return + } + + continuouslyPrintOutput := func(r io.Reader, prefix string) { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + output := scanner.Text() + fmt.Printf("%s: %s\n", prefix, output) + switch prefix { + case "stdout": + stdout = append(stdout, output...) + case "stderr": + stderr = append(stderr, output...) + } + } + } + + go continuouslyPrintOutput(stdoutIn, "stdout") + go continuouslyPrintOutput(stderrIn, "stderr") + + if err = cmd.Wait(); err != nil { + err = fmt.Errorf("wait for command to finish: %w", err) + } + + return stdout, stderr, err +} + +// Setup checks that the prerequisites for the test are met: +// - a workspace is set +// - a CLI path is set +// - the constellation-upgrade folder does not exist. +func Setup(workspace, cliPath string) error { + workingDir, err := workingDir(workspace) + if err != nil { + return fmt.Errorf("getting working directory: %w", err) + } + + if err := os.Chdir(workingDir); err != nil { + return fmt.Errorf("changing working directory: %w", err) + } + + if _, err := getCLIPath(cliPath); err != nil { + return fmt.Errorf("getting CLI path: %w", err) + } + return nil +} + +// workingDir returns the path to the workspace. +func workingDir(workspace string) (string, error) { + workingDir := os.Getenv("BUILD_WORKING_DIRECTORY") + switch { + case workingDir != "": + return workingDir, nil + case workspace != "": + return workspace, nil + default: + return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set") + } +} + +// WriteUpgradeConfig writes the target versions to the config file. +func WriteUpgradeConfig(require *require.Assertions, image string, kubernetes string, microservices string, configPath string) VersionContainer { + fileHandler := file.NewHandler(afero.NewOsFs()) + attestationFetcher := attestationconfigapi.NewFetcher() + cfg, err := config.New(fileHandler, configPath, attestationFetcher, true) + var cfgErr *config.ValidationError + var longMsg string + if errors.As(err, &cfgErr) { + longMsg = cfgErr.LongMessage() + } + require.NoError(err, longMsg) + + imageFetcher := imagefetcher.New() + imageRef, err := imageFetcher.FetchReference( + context.Background(), + cfg.GetProvider(), + cfg.GetAttestationConfig().GetVariant(), + image, + cfg.GetRegion(), cfg.UseMarketplaceImage(), + ) + require.NoError(err) + + log.Printf("Setting image version: %s\n", image) + cfg.Image = image + + defaultConfig := config.Default() + var kubernetesVersion versions.ValidK8sVersion + if kubernetes == "" { + kubernetesVersion = defaultConfig.KubernetesVersion + } else { + kubernetesVersion = versions.ValidK8sVersion(kubernetes) // ignore validation because the config is only written to file + } + + var microserviceVersion semver.Semver + if microservices == "" { + microserviceVersion = defaultConfig.MicroserviceVersion + } else { + version, err := semver.New(microservices) + require.NoError(err) + microserviceVersion = version + } + + log.Printf("Setting K8s version: %s\n", kubernetesVersion) + cfg.KubernetesVersion = kubernetesVersion + log.Printf("Setting microservice version: %s\n", microserviceVersion) + cfg.MicroserviceVersion = microserviceVersion + + err = fileHandler.WriteYAML(constants.ConfigFilename, cfg, file.OptOverwrite) + require.NoError(err) + + return VersionContainer{ImageRef: imageRef, Kubernetes: kubernetesVersion, Microservices: microserviceVersion} +} + +// getCLIPath returns the path to the CLI. +func getCLIPath(cliPathFlag string) (string, error) { + pathCLI := os.Getenv("PATH_CLI") + var relCLIPath string + switch { + case pathCLI != "": + relCLIPath = pathCLI + case cliPathFlag != "": + relCLIPath = cliPathFlag + default: + return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set") + } + + // try to find the CLI in the working directory + // (e.g. when running via `go test` or when specifying a path manually) + workdir, err := os.Getwd() + if err != nil { + return "", fmt.Errorf("getting working directory: %w", err) + } + + absCLIPath := relCLIPath + if !filepath.IsAbs(relCLIPath) { + absCLIPath = filepath.Join(workdir, relCLIPath) + } + if _, err := os.Stat(absCLIPath); err == nil { + return absCLIPath, nil + } + + // fall back to runfiles (e.g. when running via bazel) + return runfiles.Rlocation(pathCLI) +} diff --git a/e2e/internal/upgrade/upgrade_test.go b/e2e/internal/upgrade/upgrade_test.go index bf9da20e2..0d50bf80f 100644 --- a/e2e/internal/upgrade/upgrade_test.go +++ b/e2e/internal/upgrade/upgrade_test.go @@ -9,31 +9,20 @@ SPDX-License-Identifier: AGPL-3.0-only package upgrade import ( - "bufio" "context" "errors" "flag" "fmt" - "io" "log" "os" "os/exec" - "path/filepath" "strings" - "sync" "testing" "time" - "github.com/bazelbuild/rules_go/go/runfiles" "github.com/edgelesssys/constellation/v2/e2e/internal/kubectl" - "github.com/edgelesssys/constellation/v2/internal/api/attestationconfigapi" - "github.com/edgelesssys/constellation/v2/internal/config" "github.com/edgelesssys/constellation/v2/internal/constants" - "github.com/edgelesssys/constellation/v2/internal/file" - "github.com/edgelesssys/constellation/v2/internal/imagefetcher" - "github.com/edgelesssys/constellation/v2/internal/semver" "github.com/edgelesssys/constellation/v2/internal/versions" - "github.com/spf13/afero" "github.com/stretchr/testify/require" coreV1 "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -63,7 +52,7 @@ var ( func TestUpgrade(t *testing.T) { require := require.New(t) - err := setup() + err := Setup(*workspace, *cliPath) require.NoError(err) k, err := kubectl.New() @@ -79,7 +68,7 @@ func TestUpgrade(t *testing.T) { cli, err := getCLIPath(*cliPath) require.NoError(err) - targetVersions := writeUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices) + targetVersions := WriteUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices, constants.ConfigFilename) log.Println("Fetching measurements for new image.") cmd := exec.CommandContext(context.Background(), cli, "config", "fetch-measurements", "--insecure", "--debug") @@ -97,77 +86,7 @@ func TestUpgrade(t *testing.T) { log.Println("Triggering upgrade.") runUpgradeApply(require, cli) - wg := queryStatusAsync(t, cli) - - testMicroservicesEventuallyHaveVersion(t, targetVersions.microservices, *timeout) - testNodesEventuallyHaveVersion(t, k, targetVersions, *wantControl+*wantWorker, *timeout) - - wg.Wait() -} - -// setup checks that the prerequisites for the test are met: -// - a workspace is set -// - a CLI path is set -// - the constellation-upgrade folder does not exist. -func setup() error { - workingDir, err := workingDir(*workspace) - if err != nil { - return fmt.Errorf("getting working directory: %w", err) - } - - if err := os.Chdir(workingDir); err != nil { - return fmt.Errorf("changing working directory: %w", err) - } - - if _, err := getCLIPath(*cliPath); err != nil { - return fmt.Errorf("getting CLI path: %w", err) - } - return nil -} - -// workingDir returns the path to the workspace. -func workingDir(workspace string) (string, error) { - workingDir := os.Getenv("BUILD_WORKING_DIRECTORY") - switch { - case workingDir != "": - return workingDir, nil - case workspace != "": - return workspace, nil - default: - return "", errors.New("neither 'BUILD_WORKING_DIRECTORY' nor 'workspace' flag set") - } -} - -// getCLIPath returns the path to the CLI. -func getCLIPath(cliPathFlag string) (string, error) { - pathCLI := os.Getenv("PATH_CLI") - var relCLIPath string - switch { - case pathCLI != "": - relCLIPath = pathCLI - case cliPathFlag != "": - relCLIPath = cliPathFlag - default: - return "", errors.New("neither 'PATH_CLI' nor 'cli' flag set") - } - - // try to find the CLI in the working directory - // (e.g. when running via `go test` or when specifying a path manually) - workdir, err := os.Getwd() - if err != nil { - return "", fmt.Errorf("getting working directory: %w", err) - } - - absCLIPath := relCLIPath - if !filepath.IsAbs(relCLIPath) { - absCLIPath = filepath.Join(workdir, relCLIPath) - } - if _, err := os.Stat(absCLIPath); err == nil { - return absCLIPath, nil - } - - // fall back to runfiles (e.g. when running via bazel) - return runfiles.Rlocation(pathCLI) + AssertUpgradeSuccessful(t, cli, targetVersions, k, *wantControl, *wantWorker, *timeout) } // testPodsEventuallyReady checks that: @@ -249,58 +168,6 @@ func testNodesEventuallyAvailable(t *testing.T, k *kubernetes.Clientset, wantCon }, time.Minute*30, time.Minute) } -func writeUpgradeConfig(require *require.Assertions, image string, kubernetes string, microservices string) versionContainer { - fileHandler := file.NewHandler(afero.NewOsFs()) - attestationFetcher := attestationconfigapi.NewFetcher() - cfg, err := config.New(fileHandler, constants.ConfigFilename, attestationFetcher, true) - var cfgErr *config.ValidationError - var longMsg string - if errors.As(err, &cfgErr) { - longMsg = cfgErr.LongMessage() - } - require.NoError(err, longMsg) - - imageFetcher := imagefetcher.New() - imageRef, err := imageFetcher.FetchReference( - context.Background(), - cfg.GetProvider(), - cfg.GetAttestationConfig().GetVariant(), - image, - cfg.GetRegion(), cfg.UseMarketplaceImage(), - ) - require.NoError(err) - - log.Printf("Setting image version: %s\n", image) - cfg.Image = image - - defaultConfig := config.Default() - var kubernetesVersion versions.ValidK8sVersion - if kubernetes == "" { - kubernetesVersion = defaultConfig.KubernetesVersion - } else { - kubernetesVersion = versions.ValidK8sVersion(kubernetes) // ignore validation because the config is only written to file - } - - var microserviceVersion semver.Semver - if microservices == "" { - microserviceVersion = defaultConfig.MicroserviceVersion - } else { - version, err := semver.New(microservices) - require.NoError(err) - microserviceVersion = version - } - - log.Printf("Setting K8s version: %s\n", kubernetesVersion) - cfg.KubernetesVersion = kubernetesVersion - log.Printf("Setting microservice version: %s\n", microserviceVersion) - cfg.MicroserviceVersion = microserviceVersion - - err = fileHandler.WriteYAML(constants.ConfigFilename, cfg, file.OptOverwrite) - require.NoError(err) - - return versionContainer{imageRef: imageRef, kubernetes: kubernetesVersion, microservices: microserviceVersion} -} - // runUpgradeCheck executes 'upgrade check' and does basic checks on the output. // We can not check images upgrades because we might use unpublished images. CLI uses public CDN to check for available images. func runUpgradeCheck(require *require.Assertions, cli, targetKubernetes string) { @@ -361,140 +228,3 @@ func containsUnexepectedMsg(input string) error { } return nil } - -func queryStatusAsync(t *testing.T, cli string) *sync.WaitGroup { - var wg sync.WaitGroup - wg.Add(1) - go func() { - defer wg.Done() - // The first control plane node should finish upgrading after 20 minutes. If it does not, something is fishy. - // Nodes can upgrade in <5mins. - testStatusEventuallyWorks(t, cli, 20*time.Minute) - }() - - return &wg -} - -func testStatusEventuallyWorks(t *testing.T, cli string, timeout time.Duration) { - require.Eventually(t, func() bool { - // Show versions set in cluster. - // The string after "Cluster status:" in the output might not be updated yet. - // This is only updated after the operator finishes one reconcile loop. - cmd := exec.CommandContext(context.Background(), cli, "status") - stdout, stderr, err := runCommandWithSeparateOutputs(cmd) - if err != nil { - log.Printf("Stdout: %s\nStderr: %s", string(stdout), string(stderr)) - return false - } - - log.Println(string(stdout)) - return true - }, timeout, time.Minute) -} - -func testMicroservicesEventuallyHaveVersion(t *testing.T, wantMicroserviceVersion semver.Semver, timeout time.Duration) { - require.Eventually(t, func() bool { - version, err := servicesVersion(t) - if err != nil { - log.Printf("Unable to fetch microservice version: %v\n", err) - return false - } - - if version != wantMicroserviceVersion { - log.Printf("Microservices still at version %v, want %v\n", version, wantMicroserviceVersion) - return false - } - - return true - }, timeout, time.Minute) -} - -func testNodesEventuallyHaveVersion(t *testing.T, k *kubernetes.Clientset, targetVersions versionContainer, totalNodeCount int, timeout time.Duration) { - require.Eventually(t, func() bool { - nodes, err := k.CoreV1().Nodes().List(context.Background(), metaV1.ListOptions{}) - if err != nil { - log.Println(err) - return false - } - require.False(t, len(nodes.Items) < totalNodeCount, "expected at least %v nodes, got %v", totalNodeCount, len(nodes.Items)) - - allUpdated := true - log.Printf("Node status (%v):", time.Now()) - for _, node := range nodes.Items { - for key, value := range node.Annotations { - if key == "constellation.edgeless.systems/node-image" { - if !strings.EqualFold(value, targetVersions.imageRef) { - log.Printf("\t%s: Image %s, want %s\n", node.Name, value, targetVersions.imageRef) - allUpdated = false - } - } - } - - kubeletVersion := node.Status.NodeInfo.KubeletVersion - if kubeletVersion != string(targetVersions.kubernetes) { - log.Printf("\t%s: K8s (Kubelet) %s, want %s\n", node.Name, kubeletVersion, targetVersions.kubernetes) - allUpdated = false - } - kubeProxyVersion := node.Status.NodeInfo.KubeProxyVersion - if kubeProxyVersion != string(targetVersions.kubernetes) { - log.Printf("\t%s: K8s (Proxy) %s, want %s\n", node.Name, kubeProxyVersion, targetVersions.kubernetes) - allUpdated = false - } - } - - return allUpdated - }, timeout, time.Minute) -} - -type versionContainer struct { - imageRef string - kubernetes versions.ValidK8sVersion - microservices semver.Semver -} - -// runCommandWithSeparateOutputs runs the given command while separating buffers for -// stdout and stderr. -func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) { - stdout = []byte{} - stderr = []byte{} - - stdoutIn, err := cmd.StdoutPipe() - if err != nil { - err = fmt.Errorf("create stdout pipe: %w", err) - return - } - stderrIn, err := cmd.StderrPipe() - if err != nil { - err = fmt.Errorf("create stderr pipe: %w", err) - return - } - - err = cmd.Start() - if err != nil { - err = fmt.Errorf("start command: %w", err) - return - } - - continuouslyPrintOutput := func(r io.Reader, prefix string) { - scanner := bufio.NewScanner(r) - for scanner.Scan() { - output := scanner.Text() - fmt.Printf("%s: %s\n", prefix, output) - switch prefix { - case "stdout": - stdout = append(stdout, output...) - case "stderr": - stderr = append(stderr, output...) - } - } - } - - go continuouslyPrintOutput(stdoutIn, "stdout") - go continuouslyPrintOutput(stderrIn, "stderr") - - if err = cmd.Wait(); err != nil { - err = fmt.Errorf("wait for command to finish: %w", err) - } - - return stdout, stderr, err -} diff --git a/e2e/provider-upgrade/BUILD.bazel b/e2e/provider-upgrade/BUILD.bazel new file mode 100644 index 000000000..62960c246 --- /dev/null +++ b/e2e/provider-upgrade/BUILD.bazel @@ -0,0 +1,15 @@ +load("//bazel/go:go_test.bzl", "go_test") + +go_test( + name = "provider-upgrade_test", + srcs = ["upgrade_test.go"], + # keep + gotags = ["e2e"], + tags = ["manual"], + deps = [ + "//e2e/internal/kubectl", + "//e2e/internal/upgrade", + "//internal/constants", + "@com_github_stretchr_testify//require", + ], +) diff --git a/e2e/provider-upgrade/upgrade_test.go b/e2e/provider-upgrade/upgrade_test.go new file mode 100644 index 000000000..78bcd65b7 --- /dev/null +++ b/e2e/provider-upgrade/upgrade_test.go @@ -0,0 +1,60 @@ +//go:build e2e + +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +// End-to-end test that is used by the e2e Terraform provider test. +package main + +import ( + "flag" + "os" + "path/filepath" + "testing" + "time" + + "github.com/edgelesssys/constellation/v2/e2e/internal/kubectl" + "github.com/edgelesssys/constellation/v2/e2e/internal/upgrade" + "github.com/edgelesssys/constellation/v2/internal/constants" + "github.com/stretchr/testify/require" +) + +var ( + targetImage = flag.String("target-image", "", "Image (shortversion) to upgrade to.") + targetKubernetes = flag.String("target-kubernetes", "", "Kubernetes version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.") + targetMicroservices = flag.String("target-microservices", "", "Microservice version (MAJOR.MINOR.PATCH) to upgrade to. Defaults to default version of target CLI.") + // When executing the test as a bazel target the CLI path is supplied through an env variable that bazel sets. + // When executing via `go test` extra care should be taken that the supplied CLI is built on the same commit as this test. + // When executing the test as a bazel target the workspace path is supplied through an env variable that bazel sets. + workspace = flag.String("workspace", "", "Constellation workspace in which to run the tests.") + cliPath = flag.String("cli", "", "Constellation CLI to run the tests.") + wantWorker = flag.Int("want-worker", 0, "Number of wanted worker nodes.") + wantControl = flag.Int("want-control", 0, "Number of wanted control nodes.") + timeout = flag.Duration("timeout", 90*time.Minute, "Timeout after which the cluster should have converged to the target version.") +) + +// TestUpgradeSuccessful tests that the upgrade to the target version is successful. +func TestUpgradeSuccessful(t *testing.T) { + require := require.New(t) + kubeconfigPath := os.Getenv("KUBECONFIG") + require.NotEmpty(kubeconfigPath, "KUBECONFIG environment variable must be set") + dir := filepath.Dir(kubeconfigPath) + configPath := filepath.Join(dir, constants.ConfigFilename) + + // only done here to construct the version struct + require.NotEqual(*targetImage, "", "--target-image needs to be specified") + v := upgrade.WriteUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices, configPath) + // ignore Kubernetes check if targetKubernetes is not set; Kubernetes is only explicitly upgraded + if *targetKubernetes == "" { + v.Kubernetes = "" + } + k, err := kubectl.New() + require.NoError(err) + + err = upgrade.Setup(*workspace, *cliPath) + require.NoError(err) + upgrade.AssertUpgradeSuccessful(t, *cliPath, v, k, *wantControl, *wantWorker, *timeout) +} diff --git a/terraform-provider-constellation/examples/full/aws/main.tf b/terraform-provider-constellation/examples/full/aws/main.tf index 9429b0f5a..9af6a2fe2 100644 --- a/terraform-provider-constellation/examples/full/aws/main.tf +++ b/terraform-provider-constellation/examples/full/aws/main.tf @@ -13,7 +13,7 @@ terraform { locals { name = "constell" - version = "vX.Y.Z" + image_version = "vX.Y.Z" kubernetes_version = "vX.Y.Z" microservice_version = "vX.Y.Z" csp = "aws" @@ -87,7 +87,7 @@ data "constellation_attestation" "foo" { data "constellation_image" "bar" { csp = local.csp attestation_variant = local.attestation_variant - version = local.version + version = local.image_version region = local.region } diff --git a/terraform-provider-constellation/examples/full/azure/main.tf b/terraform-provider-constellation/examples/full/azure/main.tf index 1b982dffa..6220f36c5 100644 --- a/terraform-provider-constellation/examples/full/azure/main.tf +++ b/terraform-provider-constellation/examples/full/azure/main.tf @@ -13,7 +13,7 @@ terraform { locals { name = "constell" - version = "vX.Y.Z" + image_version = "vX.Y.Z" kubernetes_version = "vX.Y.Z" microservice_version = "vX.Y.Z" csp = "azure" @@ -83,7 +83,7 @@ data "constellation_attestation" "foo" { data "constellation_image" "bar" { csp = local.csp attestation_variant = local.attestation_variant - version = local.version + version = local.image_version } resource "constellation_cluster" "azure_example" { diff --git a/terraform-provider-constellation/examples/full/gcp/main.tf b/terraform-provider-constellation/examples/full/gcp/main.tf index b547c69f9..552b1a823 100644 --- a/terraform-provider-constellation/examples/full/gcp/main.tf +++ b/terraform-provider-constellation/examples/full/gcp/main.tf @@ -13,7 +13,7 @@ terraform { locals { name = "constell" - version = "vX.Y.Z" + image_version = "vX.Y.Z" kubernetes_version = "vX.Y.Z" microservice_version = "vX.Y.Z" csp = "gcp" @@ -87,7 +87,7 @@ data "constellation_attestation" "foo" { data "constellation_image" "bar" { csp = local.csp attestation_variant = local.attestation_variant - version = local.version + version = local.image_version } resource "constellation_cluster" "gcp_example" {