cli: Terraform upgrades maa patching (#1821)

* patch maa after upgrade

* buildfiles

* reword comment

* remove whitespace

* temp: log measurements URL

* temp: update import

* ignore changes to attestation policies

* add issue URL

* separate output in e2e upgrade test

* use enterprise CLI for e2e test

* remove measurements print

* add license headers

cli/internal/cloudcmd/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "cloudcmd.go",
         "create.go",
         "iam.go",
+        "patch.go",
         "rollback.go",
         "terminate.go",
         "validators.go",
@@ -42,6 +43,7 @@ go_test(
         "clients_test.go",
         "create_test.go",
         "iam_test.go",
+        "patch_test.go",
         "rollback_test.go",
         "terminate_test.go",
         "validators_test.go",

cli/internal/cloudcmd/create.go

@@ -8,11 +8,9 @@ package cloudcmd
 
 import (
 	"context"
-	"encoding/base64"
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -20,10 +18,6 @@ import (
 	"runtime"
 	"strings"
 
-	"github.com/Azure/azure-sdk-for-go/profiles/latest/attestation/attestation"
-	azpolicy "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
-	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
-
 	"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
 	"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
 	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
@@ -41,7 +35,7 @@ type Creator struct {
 	newTerraformClient func(ctx context.Context) (terraformClient, error)
 	newLibvirtRunner   func() libvirtRunner
 	newRawDownloader   func() rawDownloader
-	policyPatcher      PolicyPatcher
+	policyPatcher      policyPatcher
 }
 
 // NewCreator creates a new creator.
@@ -58,7 +52,7 @@ func NewCreator(out io.Writer) *Creator {
 		newRawDownloader: func() rawDownloader {
 			return imagefetcher.NewDownloader()
 		},
-		policyPatcher: policyPatcher{},
+		policyPatcher: NewAzurePolicyPatcher(),
 	}
 }
 
@@ -254,82 +248,11 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea
 	}, nil
 }
 
-// PolicyPatcher interacts with Azure to update the attestation policy.
+// policyPatcher interacts with the CSP (currently only applies for Azure) to update the attestation policy.
-type PolicyPatcher interface {
+type policyPatcher interface {
 	Patch(ctx context.Context, attestationURL string) error
 }
 
-type policyPatcher struct{}
-
-// Patch updates the attestation policy to the base64-encoded attestation policy JWT for the given attestation URL.
-// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#next-steps
-func (p policyPatcher) Patch(ctx context.Context, attestationURL string) error {
-	// hacky way to update the MAA attestation policy. This should be changed as soon as either the Terraform provider supports it
-	// or the Go SDK gets updated to a recent API version.
-	// https://github.com/hashicorp/terraform-provider-azurerm/issues/20804
-	cred, err := azidentity.NewDefaultAzureCredential(nil)
-	if err != nil {
-		return fmt.Errorf("retrieving default Azure credentials: %w", err)
-	}
-	token, err := cred.GetToken(ctx, azpolicy.TokenRequestOptions{
-		Scopes: []string{"https://attest.azure.net/.default"},
-	})
-	if err != nil {
-		return fmt.Errorf("retrieving token from default Azure credentials: %w", err)
-	}
-
-	client := attestation.NewPolicyClient()
-
-	// azureGuest is the id for the "Azure VM" attestation type. Other types are documented here:
-	// https://learn.microsoft.com/en-us/rest/api/attestation/policy/set
-	req, err := client.SetPreparer(ctx, attestationURL, "azureGuest", p.encodeAttestationPolicy())
-	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.Token))
-	if err != nil {
-		return fmt.Errorf("preparing request: %w", err)
-	}
-
-	resp, err := client.Send(req)
-	if err != nil {
-		return fmt.Errorf("sending request: %w", err)
-	}
-	resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("updating attestation policy: unexpected status code: %s", resp.Status)
-	}
-
-	return nil
-}
-
-// encodeAttestationPolicy encodes the base64-encoded attestation policy in the JWS format specified here:
-// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#creating-the-policy-file-in-json-web-signature-format
-func (p policyPatcher) encodeAttestationPolicy() string {
-	const policy = `
-                version= 1.0;
-                authorizationrules
-                {
-                    [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny();
-                    [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny();
-                    // The line below was edited by the Constellation CLI. Do not edit manually.
-                    //[type=="secureboot", value==false] => deny();
-                    [type=="x-ms-azurevm-signingdisabled", value==false] => deny();
-                    [type=="x-ms-azurevm-dbvalidated", value==false] => deny();
-                    [type=="x-ms-azurevm-dbxvalidated", value==false] => deny();
-                    => permit();
-                };
-                issuancerules
-                {
-                };`
-	encodedPolicy := base64.RawURLEncoding.EncodeToString([]byte(policy))
-	const header = `{"alg":"none"}`
-	payload := fmt.Sprintf(`{"AttestationPolicy":"%s"}`, encodedPolicy)
-
-	encodedHeader := base64.RawURLEncoding.EncodeToString([]byte(header))
-	encodedPayload := base64.RawURLEncoding.EncodeToString([]byte(payload))
-
-	return fmt.Sprintf("%s.%s.", encodedHeader, encodedPayload)
-}
-
 // The azurerm Terraform provider enforces its own convention of case sensitivity for Azure URIs which Azure's API itself does not enforce or, even worse, actually returns.
 // Let's go loco with case insensitive Regexp here and fix the user input here to be compliant with this arbitrary design decision.
 var (

cli/internal/cloudcmd/patch.go

@@ -0,0 +1,94 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+package cloudcmd
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+
+	"github.com/Azure/azure-sdk-for-go/profiles/latest/attestation/attestation"
+	azpolicy "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
+	"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
+)
+
+// NewAzurePolicyPatcher returns a new AzurePolicyPatcher.
+func NewAzurePolicyPatcher() AzurePolicyPatcher {
+	return AzurePolicyPatcher{}
+}
+
+// AzurePolicyPatcher patches attestation policies on Azure.
+type AzurePolicyPatcher struct{}
+
+// Patch updates the attestation policy to the base64-encoded attestation policy JWT for the given attestation URL.
+// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#next-steps
+func (p AzurePolicyPatcher) Patch(ctx context.Context, attestationURL string) error {
+	// hacky way to update the MAA attestation policy. This should be changed as soon as either the Terraform provider supports it
+	// or the Go SDK gets updated to a recent API version.
+	// https://github.com/hashicorp/terraform-provider-azurerm/issues/20804
+	cred, err := azidentity.NewDefaultAzureCredential(nil)
+	if err != nil {
+		return fmt.Errorf("retrieving default Azure credentials: %w", err)
+	}
+	token, err := cred.GetToken(ctx, azpolicy.TokenRequestOptions{
+		Scopes: []string{"https://attest.azure.net/.default"},
+	})
+	if err != nil {
+		return fmt.Errorf("retrieving token from default Azure credentials: %w", err)
+	}
+
+	client := attestation.NewPolicyClient()
+
+	// azureGuest is the id for the "Azure VM" attestation type. Other types are documented here:
+	// https://learn.microsoft.com/en-us/rest/api/attestation/policy/set
+	req, err := client.SetPreparer(ctx, attestationURL, "azureGuest", p.encodeAttestationPolicy())
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.Token))
+	if err != nil {
+		return fmt.Errorf("preparing request: %w", err)
+	}
+
+	resp, err := client.Send(req)
+	if err != nil {
+		return fmt.Errorf("sending request: %w", err)
+	}
+	resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("updating attestation policy: unexpected status code: %s", resp.Status)
+	}
+
+	return nil
+}
+
+// encodeAttestationPolicy encodes the base64-encoded attestation policy in the JWS format specified here:
+// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#creating-the-policy-file-in-json-web-signature-format
+func (p AzurePolicyPatcher) encodeAttestationPolicy() string {
+	const policy = `
+                version= 1.0;
+                authorizationrules
+                {
+                    [type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny();
+                    [type=="x-ms-azurevm-debuggersdisabled", value==false] => deny();
+                    // The line below was edited by the Constellation CLI. Do not edit manually.
+                    //[type=="secureboot", value==false] => deny();
+                    [type=="x-ms-azurevm-signingdisabled", value==false] => deny();
+                    [type=="x-ms-azurevm-dbvalidated", value==false] => deny();
+                    [type=="x-ms-azurevm-dbxvalidated", value==false] => deny();
+                    => permit();
+                };
+                issuancerules
+                {
+                };`
+	encodedPolicy := base64.RawURLEncoding.EncodeToString([]byte(policy))
+	const header = `{"alg":"none"}`
+	payload := fmt.Sprintf(`{"AttestationPolicy":"%s"}`, encodedPolicy)
+
+	encodedHeader := base64.RawURLEncoding.EncodeToString([]byte(header))
+	encodedPayload := base64.RawURLEncoding.EncodeToString([]byte(payload))
+
+	return fmt.Sprintf("%s.%s.", encodedHeader, encodedPayload)
+}

cli/internal/cloudcmd/patch_test.go

@@ -0,0 +1,22 @@
+/*
+Copyright (c) Edgeless Systems GmbH
+
+SPDX-License-Identifier: AGPL-3.0-only
+*/
+package cloudcmd
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestEncodeAttestationPolicy(t *testing.T) {
+	assert := assert.New(t)
+
+	p := AzurePolicyPatcher{}
+
+	// taken from <resource group url in the azure portal>/providers/Microsoft.Attestation/attestationProviders/<attestation provider name>/mrsg_item2
+	expected := "eyJhbGciOiJub25lIn0.eyJBdHRlc3RhdGlvblBvbGljeSI6IkNpQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNCMlpYSnphVzl1UFNBeExqQTdDaUFnSUNBZ0lDQWdJQ0FnSUNBZ0lDQmhkWFJvYjNKcGVtRjBhVzl1Y25Wc1pYTUtJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lIc0tJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0JiZEhsd1pUMDlJbmd0YlhNdFlYcDFjbVYyYlMxa1pXWmhkV3gwTFhObFkzVnlaV0p2YjNSclpYbHpkbUZzYVdSaGRHVmtJaXdnZG1Gc2RXVTlQV1poYkhObFhTQTlQaUJrWlc1NUtDazdDaUFnSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnVzNSNWNHVTlQU0o0TFcxekxXRjZkWEpsZG0wdFpHVmlkV2RuWlhKelpHbHpZV0pzWldRaUxDQjJZV3gxWlQwOVptRnNjMlZkSUQwLUlHUmxibmtvS1RzS0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lDQXZMeUJVYUdVZ2JHbHVaU0JpWld4dmR5QjNZWE1nWldScGRHVmtJR0o1SUhSb1pTQkRiMjV6ZEdWc2JHRjBhVzl1SUVOTVNTNGdSRzhnYm05MElHVmthWFFnYldGdWRXRnNiSGt1Q2lBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0x5OWJkSGx3WlQwOUluTmxZM1Z5WldKdmIzUWlMQ0IyWVd4MVpUMDlabUZzYzJWZElEMC1JR1JsYm5rb0tUc0tJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0JiZEhsd1pUMDlJbmd0YlhNdFlYcDFjbVYyYlMxemFXZHVhVzVuWkdsellXSnNaV1FpTENCMllXeDFaVDA5Wm1Gc2MyVmRJRDAtSUdSbGJua29LVHNLSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNCYmRIbHdaVDA5SW5ndGJYTXRZWHAxY21WMmJTMWtZblpoYkdsa1lYUmxaQ0lzSUhaaGJIVmxQVDFtWVd4elpWMGdQVDRnWkdWdWVTZ3BPd29nSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUZ0MGVYQmxQVDBpZUMxdGN5MWhlblZ5WlhadExXUmllSFpoYkdsa1lYUmxaQ0lzSUhaaGJIVmxQVDFtWVd4elpWMGdQVDRnWkdWdWVTZ3BPd29nSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUQwLUlIQmxjbTFwZENncE93b2dJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ2ZUc0tJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lHbHpjM1ZoYm1ObGNuVnNaWE1LSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJSHNLSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJSDA3In0."
+	assert.Equal(expected, p.encodeAttestationPolicy())
+}

cli/internal/terraform/terraform/azure/main.tf

@@ -51,6 +51,15 @@ resource "azurerm_attestation_provider" "attestation_provider" {
   name                = format("constell%s", local.uid)
   resource_group_name = var.resource_group
   location            = var.location
+
+  lifecycle {
+    # Attestation policies will be set automatically upon creation, even if not specified in the resource,
+    # while they aren't being incorporated into the Terraform state correctly.
+    # To prevent them from being set to null when applying an upgrade, ignore the changes until the issue
+    # is resolved by Azure.
+    # Related issue: https://github.com/hashicorp/terraform-provider-azurerm/issues/21998
+    ignore_changes = [open_enclave_policy_base64, sgx_enclave_policy_base64, tpm_policy_base64]
+  }
 }
 
 resource "azurerm_application_insights" "insights" {

cli/internal/upgrade/BUILD.bazel

@@ -10,6 +10,7 @@ go_library(
     importpath = "github.com/edgelesssys/constellation/v2/cli/internal/upgrade",
     visibility = ["//cli:__subpackages__"],
     deps = [
+        "//cli/internal/cloudcmd",
         "//cli/internal/clusterid",
         "//cli/internal/terraform",
         "//internal/cloud/cloudprovider",

cli/internal/upgrade/terraform.go

@@ -14,6 +14,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
 	"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
 	"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
 	"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@@ -24,15 +25,17 @@ import (
 // NewTerraformUpgrader returns a new TerraformUpgrader.
 func NewTerraformUpgrader(tfClient tfClient, outWriter io.Writer) (*TerraformUpgrader, error) {
 	return &TerraformUpgrader{
-		tf:        tfClient,
+		tf:            tfClient,
-		outWriter: outWriter,
+		policyPatcher: cloudcmd.NewAzurePolicyPatcher(),
+		outWriter:     outWriter,
 	}, nil
 }
 
 // TerraformUpgrader is responsible for performing Terraform migrations on cluster upgrades.
 type TerraformUpgrader struct {
-	tf        tfClient
+	tf            tfClient
-	outWriter io.Writer
+	policyPatcher policyPatcher
+	outWriter     io.Writer
 }
 
 // TerraformUpgradeOptions are the options used for the Terraform upgrade.
@@ -139,6 +142,13 @@ func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, fileHa
 		return fmt.Errorf("terraform apply: %w", err)
 	}
 
+	// AttestationURL is only set for Azure.
+	if tfOutput.AttestationURL != "" {
+		if err := u.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
+			return fmt.Errorf("patching policies: %w", err)
+		}
+	}
+
 	outputFileContents := clusterid.File{
 		CloudProvider:  opts.CSP,
 		InitSecret:     []byte(tfOutput.Secret),
@@ -173,3 +183,8 @@ type tfClient interface {
 	Plan(ctx context.Context, logLevel terraform.LogLevel, planFile string, targets ...string) (bool, error)
 	CreateCluster(ctx context.Context, logLevel terraform.LogLevel, targets ...string) (terraform.CreateOutput, error)
 }
+
+// policyPatcher interacts with the CSP (currently only applies for Azure) to update the attestation policy.
+type policyPatcher interface {
+	Patch(ctx context.Context, attestationURL string) error
+}

cli/internal/upgrade/terraform_test.go

@@ -164,6 +164,7 @@ func TestApplyTerraformMigrations(t *testing.T) {
 
 	testCases := map[string]struct {
 		tf             tfClient
+		policyPatcher  stubPolicyPatcher
 		fs             file.Handler
 		outputFileName string
 		wantErr        bool
@@ -171,6 +172,7 @@ func TestApplyTerraformMigrations(t *testing.T) {
 		"success": {
 			tf:             &stubTerraformClient{},
 			fs:             fileHandler(),
+			policyPatcher:  stubPolicyPatcher{},
 			outputFileName: "test.json",
 		},
 		"create cluster error": {
@@ -178,18 +180,29 @@ func TestApplyTerraformMigrations(t *testing.T) {
 				CreateClusterErr: assert.AnError,
 			},
 			fs:             fileHandler(),
+			policyPatcher:  stubPolicyPatcher{},
 			outputFileName: "test.json",
 			wantErr:        true,
 		},
+		"patch error": {
+			tf: &stubTerraformClient{},
+			fs: fileHandler(),
+			policyPatcher: stubPolicyPatcher{
+				patchErr: assert.AnError,
+			},
+			wantErr: true,
+		},
 		"empty file name": {
 			tf:             &stubTerraformClient{},
 			fs:             fileHandler(),
+			policyPatcher:  stubPolicyPatcher{},
 			outputFileName: "",
 			wantErr:        true,
 		},
 		"file already exists": {
 			tf:             &stubTerraformClient{},
 			fs:             fileHandler("test.json"),
+			policyPatcher:  stubPolicyPatcher{},
 			outputFileName: "test.json",
 			wantErr:        true,
 		},
@@ -311,3 +324,11 @@ func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string,
 func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel, ...string) (terraform.CreateOutput, error) {
 	return terraform.CreateOutput{}, u.CreateClusterErr
 }
+
+type stubPolicyPatcher struct {
+	patchErr error
+}
+
+func (p *stubPolicyPatcher) PatchPolicy(context.Context, string) error {
+	return p.patchErr
+}

e2e/internal/upgrade/BUILD.bazel

@@ -29,11 +29,11 @@ go_test(
     # keep
     count = 1,
     data = [
-        "//cli:cli_oss_linux_amd64",
+        "//cli:cli_enterprise_linux_amd64",
     ],
     embed = [":upgrade"],
     env = {
-        "PATH_CLI": "$(location //cli:cli_oss_linux_amd64)",
+        "PATH_CLI": "$(location //cli:cli_enterprise_linux_amd64)",
     },
     # keep
     gotags = ["e2e"],

e2e/internal/upgrade/upgrade_test.go

@@ -13,6 +13,7 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"io"
 	"log"
 	"os"
 	"os/exec"
@@ -74,9 +75,9 @@ func TestUpgrade(t *testing.T) {
 
 	// Migrate config if necessary.
 	cmd := exec.CommandContext(context.Background(), cli, "config", "migrate", "--config", constants.ConfigFilename, "--force", "--debug")
-	msg, err := cmd.CombinedOutput()
+	stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
-	require.NoError(err, string(msg))
+	require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
-	log.Println(string(msg))
+	log.Println(string(stdout))
 
 	targetVersions := writeUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices)
 
@@ -94,9 +95,9 @@ func TestUpgrade(t *testing.T) {
 	// The string after "Cluster status:" in the output might not be updated yet.
 	// This is only updated after the operator finishes one reconcile loop.
 	cmd = exec.CommandContext(context.Background(), cli, "status")
-	msg, err = cmd.CombinedOutput()
+	stdout, stderr, err = runCommandWithSeparateOutputs(cmd)
-	require.NoError(err, string(msg))
+	require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
-	log.Println(string(msg))
+	log.Println(string(stdout))
 
 	testMicroservicesEventuallyHaveVersion(t, targetVersions.microservices, *timeout)
 	testNodesEventuallyHaveVersion(t, k, targetVersions, *wantControl+*wantWorker, *timeout)
@@ -287,27 +288,27 @@ func writeUpgradeConfig(require *require.Assertions, image string, kubernetes st
 // We can not check images upgrades because we might use unpublished images. CLI uses public CDN to check for available images.
 func runUpgradeCheck(require *require.Assertions, cli, targetKubernetes string) {
 	cmd := exec.CommandContext(context.Background(), cli, "upgrade", "check")
-	msg, err := cmd.Output()
+	stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
-	require.NoError(err, "%s", string(msg))
+	require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
 
-	require.Contains(string(msg), "The following updates are available with this CLI:")
+	require.Contains(string(stdout), "The following updates are available with this CLI:")
-	require.Contains(string(msg), "Kubernetes:")
+	require.Contains(string(stdout), "Kubernetes:")
 	log.Printf("targetKubernetes: %s\n", targetKubernetes)
 
 	if targetKubernetes == "" {
 		log.Printf("true\n")
-		require.True(containsAny(string(msg), versions.SupportedK8sVersions()))
+		require.True(containsAny(string(stdout), versions.SupportedK8sVersions()))
 	} else {
 		log.Printf("false. targetKubernetes: %s\n", targetKubernetes)
-		require.Contains(string(msg), targetKubernetes, fmt.Sprintf("Expected Kubernetes version %s in output.", targetKubernetes))
+		require.Contains(string(stdout), targetKubernetes, fmt.Sprintf("Expected Kubernetes version %s in output.", targetKubernetes))
 	}
 
 	cliVersion, err := semver.New(constants.VersionInfo())
 	require.NoError(err)
-	require.Contains(string(msg), "Services:")
+	require.Contains(string(stdout), "Services:")
-	require.Contains(string(msg), fmt.Sprintf("--> %s", cliVersion.String()))
+	require.Contains(string(stdout), fmt.Sprintf("--> %s", cliVersion.String()))
 
-	log.Println(string(msg))
+	log.Println(string(stdout))
 }
 
 func containsAny(text string, substrs []string) bool {
@@ -322,17 +323,17 @@ func containsAny(text string, substrs []string) bool {
 func runUpgradeApply(require *require.Assertions, cli string) {
 	tfLogFlag := ""
 	cmd := exec.CommandContext(context.Background(), cli, "--help")
-	msg, err := cmd.CombinedOutput()
+	stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
-	require.NoErrorf(err, "%s", string(msg))
+	require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
-	if strings.Contains(string(msg), "--tf-log") {
+	if strings.Contains(string(stdout), "--tf-log") {
 		tfLogFlag = "--tf-log=DEBUG"
 	}
 
 	cmd = exec.CommandContext(context.Background(), cli, "upgrade", "apply", "--force", "--debug", "--yes", tfLogFlag)
-	msg, err = cmd.CombinedOutput()
+	stdout, stderr, err = runCommandWithSeparateOutputs(cmd)
-	require.NoErrorf(err, "%s", string(msg))
+	require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
-	require.NoError(containsUnexepectedMsg(string(msg)))
+	require.NoError(containsUnexepectedMsg(string(stdout)))
-	log.Println(string(msg))
+	log.Println(string(stdout))
 }
 
 // containsUnexepectedMsg checks if the given input contains any unexpected messages.
@@ -404,3 +405,42 @@ type versionContainer struct {
 	kubernetes    semver.Semver
 	microservices string
 }
+
+// runCommandWithSeparateOutputs runs the given command while separating buffers for
+// stdout and stderr.
+func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) {
+	stdoutIn, err := cmd.StdoutPipe()
+	if err != nil {
+		err = fmt.Errorf("create stdout pipe: %w", err)
+		return
+	}
+	stderrIn, err := cmd.StderrPipe()
+	if err != nil {
+		err = fmt.Errorf("create stderr pipe: %w", err)
+		return
+	}
+
+	err = cmd.Start()
+	if err != nil {
+		err = fmt.Errorf("start command: %w", err)
+		return
+	}
+
+	stdout, err = io.ReadAll(stdoutIn)
+	if err != nil {
+		err = fmt.Errorf("start command: %w", err)
+		return
+	}
+
+	stderr, err = io.ReadAll(stderrIn)
+	if err != nil {
+		err = fmt.Errorf("start command: %w", err)
+		return
+	}
+
+	if err = cmd.Wait(); err != nil {
+		err = fmt.Errorf("wait for command to finish: %w", err)
+	}
+
+	return
+}