mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-10-01 01:36:09 -04:00
cli: Terraform upgrades maa patching (#1821)
* patch maa after upgrade * buildfiles * reword comment * remove whitespace * temp: log measurements URL * temp: update import * ignore changes to attestation policies * add issue URL * separate output in e2e upgrade test * use enterprise CLI for e2e test * remove measurements print * add license headers
This commit is contained in:
parent
7ef7f09dda
commit
8c3b963a3f
@ -8,6 +8,7 @@ go_library(
|
||||
"cloudcmd.go",
|
||||
"create.go",
|
||||
"iam.go",
|
||||
"patch.go",
|
||||
"rollback.go",
|
||||
"terminate.go",
|
||||
"validators.go",
|
||||
@ -42,6 +43,7 @@ go_test(
|
||||
"clients_test.go",
|
||||
"create_test.go",
|
||||
"iam_test.go",
|
||||
"patch_test.go",
|
||||
"rollback_test.go",
|
||||
"terminate_test.go",
|
||||
"validators_test.go",
|
||||
|
@ -8,11 +8,9 @@ package cloudcmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
@ -20,10 +18,6 @@ import (
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/profiles/latest/attestation/attestation"
|
||||
azpolicy "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
|
||||
"github.com/edgelesssys/constellation/v2/cli/internal/libvirt"
|
||||
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
|
||||
@ -41,7 +35,7 @@ type Creator struct {
|
||||
newTerraformClient func(ctx context.Context) (terraformClient, error)
|
||||
newLibvirtRunner func() libvirtRunner
|
||||
newRawDownloader func() rawDownloader
|
||||
policyPatcher PolicyPatcher
|
||||
policyPatcher policyPatcher
|
||||
}
|
||||
|
||||
// NewCreator creates a new creator.
|
||||
@ -58,7 +52,7 @@ func NewCreator(out io.Writer) *Creator {
|
||||
newRawDownloader: func() rawDownloader {
|
||||
return imagefetcher.NewDownloader()
|
||||
},
|
||||
policyPatcher: policyPatcher{},
|
||||
policyPatcher: NewAzurePolicyPatcher(),
|
||||
}
|
||||
}
|
||||
|
||||
@ -254,82 +248,11 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, opts Crea
|
||||
}, nil
|
||||
}
|
||||
|
||||
// PolicyPatcher interacts with Azure to update the attestation policy.
|
||||
type PolicyPatcher interface {
|
||||
// policyPatcher interacts with the CSP (currently only applies for Azure) to update the attestation policy.
|
||||
type policyPatcher interface {
|
||||
Patch(ctx context.Context, attestationURL string) error
|
||||
}
|
||||
|
||||
type policyPatcher struct{}
|
||||
|
||||
// Patch updates the attestation policy to the base64-encoded attestation policy JWT for the given attestation URL.
|
||||
// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#next-steps
|
||||
func (p policyPatcher) Patch(ctx context.Context, attestationURL string) error {
|
||||
// hacky way to update the MAA attestation policy. This should be changed as soon as either the Terraform provider supports it
|
||||
// or the Go SDK gets updated to a recent API version.
|
||||
// https://github.com/hashicorp/terraform-provider-azurerm/issues/20804
|
||||
cred, err := azidentity.NewDefaultAzureCredential(nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("retrieving default Azure credentials: %w", err)
|
||||
}
|
||||
token, err := cred.GetToken(ctx, azpolicy.TokenRequestOptions{
|
||||
Scopes: []string{"https://attest.azure.net/.default"},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("retrieving token from default Azure credentials: %w", err)
|
||||
}
|
||||
|
||||
client := attestation.NewPolicyClient()
|
||||
|
||||
// azureGuest is the id for the "Azure VM" attestation type. Other types are documented here:
|
||||
// https://learn.microsoft.com/en-us/rest/api/attestation/policy/set
|
||||
req, err := client.SetPreparer(ctx, attestationURL, "azureGuest", p.encodeAttestationPolicy())
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.Token))
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := client.Send(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sending request: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("updating attestation policy: unexpected status code: %s", resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// encodeAttestationPolicy encodes the base64-encoded attestation policy in the JWS format specified here:
|
||||
// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#creating-the-policy-file-in-json-web-signature-format
|
||||
func (p policyPatcher) encodeAttestationPolicy() string {
|
||||
const policy = `
|
||||
version= 1.0;
|
||||
authorizationrules
|
||||
{
|
||||
[type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny();
|
||||
[type=="x-ms-azurevm-debuggersdisabled", value==false] => deny();
|
||||
// The line below was edited by the Constellation CLI. Do not edit manually.
|
||||
//[type=="secureboot", value==false] => deny();
|
||||
[type=="x-ms-azurevm-signingdisabled", value==false] => deny();
|
||||
[type=="x-ms-azurevm-dbvalidated", value==false] => deny();
|
||||
[type=="x-ms-azurevm-dbxvalidated", value==false] => deny();
|
||||
=> permit();
|
||||
};
|
||||
issuancerules
|
||||
{
|
||||
};`
|
||||
encodedPolicy := base64.RawURLEncoding.EncodeToString([]byte(policy))
|
||||
const header = `{"alg":"none"}`
|
||||
payload := fmt.Sprintf(`{"AttestationPolicy":"%s"}`, encodedPolicy)
|
||||
|
||||
encodedHeader := base64.RawURLEncoding.EncodeToString([]byte(header))
|
||||
encodedPayload := base64.RawURLEncoding.EncodeToString([]byte(payload))
|
||||
|
||||
return fmt.Sprintf("%s.%s.", encodedHeader, encodedPayload)
|
||||
}
|
||||
|
||||
// The azurerm Terraform provider enforces its own convention of case sensitivity for Azure URIs which Azure's API itself does not enforce or, even worse, actually returns.
|
||||
// Let's go loco with case insensitive Regexp here and fix the user input here to be compliant with this arbitrary design decision.
|
||||
var (
|
||||
|
94
cli/internal/cloudcmd/patch.go
Normal file
94
cli/internal/cloudcmd/patch.go
Normal file
@ -0,0 +1,94 @@
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
package cloudcmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/Azure/azure-sdk-for-go/profiles/latest/attestation/attestation"
|
||||
azpolicy "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy"
|
||||
"github.com/Azure/azure-sdk-for-go/sdk/azidentity"
|
||||
)
|
||||
|
||||
// NewAzurePolicyPatcher returns a new AzurePolicyPatcher.
|
||||
func NewAzurePolicyPatcher() AzurePolicyPatcher {
|
||||
return AzurePolicyPatcher{}
|
||||
}
|
||||
|
||||
// AzurePolicyPatcher patches attestation policies on Azure.
|
||||
type AzurePolicyPatcher struct{}
|
||||
|
||||
// Patch updates the attestation policy to the base64-encoded attestation policy JWT for the given attestation URL.
|
||||
// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#next-steps
|
||||
func (p AzurePolicyPatcher) Patch(ctx context.Context, attestationURL string) error {
|
||||
// hacky way to update the MAA attestation policy. This should be changed as soon as either the Terraform provider supports it
|
||||
// or the Go SDK gets updated to a recent API version.
|
||||
// https://github.com/hashicorp/terraform-provider-azurerm/issues/20804
|
||||
cred, err := azidentity.NewDefaultAzureCredential(nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("retrieving default Azure credentials: %w", err)
|
||||
}
|
||||
token, err := cred.GetToken(ctx, azpolicy.TokenRequestOptions{
|
||||
Scopes: []string{"https://attest.azure.net/.default"},
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("retrieving token from default Azure credentials: %w", err)
|
||||
}
|
||||
|
||||
client := attestation.NewPolicyClient()
|
||||
|
||||
// azureGuest is the id for the "Azure VM" attestation type. Other types are documented here:
|
||||
// https://learn.microsoft.com/en-us/rest/api/attestation/policy/set
|
||||
req, err := client.SetPreparer(ctx, attestationURL, "azureGuest", p.encodeAttestationPolicy())
|
||||
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.Token))
|
||||
if err != nil {
|
||||
return fmt.Errorf("preparing request: %w", err)
|
||||
}
|
||||
|
||||
resp, err := client.Send(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sending request: %w", err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf("updating attestation policy: unexpected status code: %s", resp.Status)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// encodeAttestationPolicy encodes the base64-encoded attestation policy in the JWS format specified here:
|
||||
// https://learn.microsoft.com/en-us/azure/attestation/author-sign-policy#creating-the-policy-file-in-json-web-signature-format
|
||||
func (p AzurePolicyPatcher) encodeAttestationPolicy() string {
|
||||
const policy = `
|
||||
version= 1.0;
|
||||
authorizationrules
|
||||
{
|
||||
[type=="x-ms-azurevm-default-securebootkeysvalidated", value==false] => deny();
|
||||
[type=="x-ms-azurevm-debuggersdisabled", value==false] => deny();
|
||||
// The line below was edited by the Constellation CLI. Do not edit manually.
|
||||
//[type=="secureboot", value==false] => deny();
|
||||
[type=="x-ms-azurevm-signingdisabled", value==false] => deny();
|
||||
[type=="x-ms-azurevm-dbvalidated", value==false] => deny();
|
||||
[type=="x-ms-azurevm-dbxvalidated", value==false] => deny();
|
||||
=> permit();
|
||||
};
|
||||
issuancerules
|
||||
{
|
||||
};`
|
||||
encodedPolicy := base64.RawURLEncoding.EncodeToString([]byte(policy))
|
||||
const header = `{"alg":"none"}`
|
||||
payload := fmt.Sprintf(`{"AttestationPolicy":"%s"}`, encodedPolicy)
|
||||
|
||||
encodedHeader := base64.RawURLEncoding.EncodeToString([]byte(header))
|
||||
encodedPayload := base64.RawURLEncoding.EncodeToString([]byte(payload))
|
||||
|
||||
return fmt.Sprintf("%s.%s.", encodedHeader, encodedPayload)
|
||||
}
|
22
cli/internal/cloudcmd/patch_test.go
Normal file
22
cli/internal/cloudcmd/patch_test.go
Normal file
@ -0,0 +1,22 @@
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
package cloudcmd
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestEncodeAttestationPolicy(t *testing.T) {
|
||||
assert := assert.New(t)
|
||||
|
||||
p := AzurePolicyPatcher{}
|
||||
|
||||
// taken from <resource group url in the azure portal>/providers/Microsoft.Attestation/attestationProviders/<attestation provider name>/mrsg_item2
|
||||
expected := "eyJhbGciOiJub25lIn0.eyJBdHRlc3RhdGlvblBvbGljeSI6IkNpQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNCMlpYSnphVzl1UFNBeExqQTdDaUFnSUNBZ0lDQWdJQ0FnSUNBZ0lDQmhkWFJvYjNKcGVtRjBhVzl1Y25Wc1pYTUtJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lIc0tJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0JiZEhsd1pUMDlJbmd0YlhNdFlYcDFjbVYyYlMxa1pXWmhkV3gwTFhObFkzVnlaV0p2YjNSclpYbHpkbUZzYVdSaGRHVmtJaXdnZG1Gc2RXVTlQV1poYkhObFhTQTlQaUJrWlc1NUtDazdDaUFnSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnVzNSNWNHVTlQU0o0TFcxekxXRjZkWEpsZG0wdFpHVmlkV2RuWlhKelpHbHpZV0pzWldRaUxDQjJZV3gxWlQwOVptRnNjMlZkSUQwLUlHUmxibmtvS1RzS0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lDQXZMeUJVYUdVZ2JHbHVaU0JpWld4dmR5QjNZWE1nWldScGRHVmtJR0o1SUhSb1pTQkRiMjV6ZEdWc2JHRjBhVzl1SUVOTVNTNGdSRzhnYm05MElHVmthWFFnYldGdWRXRnNiSGt1Q2lBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0x5OWJkSGx3WlQwOUluTmxZM1Z5WldKdmIzUWlMQ0IyWVd4MVpUMDlabUZzYzJWZElEMC1JR1JsYm5rb0tUc0tJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0JiZEhsd1pUMDlJbmd0YlhNdFlYcDFjbVYyYlMxemFXZHVhVzVuWkdsellXSnNaV1FpTENCMllXeDFaVDA5Wm1Gc2MyVmRJRDAtSUdSbGJua29LVHNLSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUNCYmRIbHdaVDA5SW5ndGJYTXRZWHAxY21WMmJTMWtZblpoYkdsa1lYUmxaQ0lzSUhaaGJIVmxQVDFtWVd4elpWMGdQVDRnWkdWdWVTZ3BPd29nSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUZ0MGVYQmxQVDBpZUMxdGN5MWhlblZ5WlhadExXUmllSFpoYkdsa1lYUmxaQ0lzSUhaaGJIVmxQVDFtWVd4elpWMGdQVDRnWkdWdWVTZ3BPd29nSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJQ0FnSUQwLUlIQmxjbTFwZENncE93b2dJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ2ZUc0tJQ0FnSUNBZ0lDQWdJQ0FnSUNBZ0lHbHpjM1ZoYm1ObGNuVnNaWE1LSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJSHNLSUNBZ0lDQWdJQ0FnSUNBZ0lDQWdJSDA3In0."
|
||||
assert.Equal(expected, p.encodeAttestationPolicy())
|
||||
}
|
@ -51,6 +51,15 @@ resource "azurerm_attestation_provider" "attestation_provider" {
|
||||
name = format("constell%s", local.uid)
|
||||
resource_group_name = var.resource_group
|
||||
location = var.location
|
||||
|
||||
lifecycle {
|
||||
# Attestation policies will be set automatically upon creation, even if not specified in the resource,
|
||||
# while they aren't being incorporated into the Terraform state correctly.
|
||||
# To prevent them from being set to null when applying an upgrade, ignore the changes until the issue
|
||||
# is resolved by Azure.
|
||||
# Related issue: https://github.com/hashicorp/terraform-provider-azurerm/issues/21998
|
||||
ignore_changes = [open_enclave_policy_base64, sgx_enclave_policy_base64, tpm_policy_base64]
|
||||
}
|
||||
}
|
||||
|
||||
resource "azurerm_application_insights" "insights" {
|
||||
|
@ -10,6 +10,7 @@ go_library(
|
||||
importpath = "github.com/edgelesssys/constellation/v2/cli/internal/upgrade",
|
||||
visibility = ["//cli:__subpackages__"],
|
||||
deps = [
|
||||
"//cli/internal/cloudcmd",
|
||||
"//cli/internal/clusterid",
|
||||
"//cli/internal/terraform",
|
||||
"//internal/cloud/cloudprovider",
|
||||
|
@ -14,6 +14,7 @@ import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/cli/internal/cloudcmd"
|
||||
"github.com/edgelesssys/constellation/v2/cli/internal/clusterid"
|
||||
"github.com/edgelesssys/constellation/v2/cli/internal/terraform"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
|
||||
@ -24,15 +25,17 @@ import (
|
||||
// NewTerraformUpgrader returns a new TerraformUpgrader.
|
||||
func NewTerraformUpgrader(tfClient tfClient, outWriter io.Writer) (*TerraformUpgrader, error) {
|
||||
return &TerraformUpgrader{
|
||||
tf: tfClient,
|
||||
outWriter: outWriter,
|
||||
tf: tfClient,
|
||||
policyPatcher: cloudcmd.NewAzurePolicyPatcher(),
|
||||
outWriter: outWriter,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// TerraformUpgrader is responsible for performing Terraform migrations on cluster upgrades.
|
||||
type TerraformUpgrader struct {
|
||||
tf tfClient
|
||||
outWriter io.Writer
|
||||
tf tfClient
|
||||
policyPatcher policyPatcher
|
||||
outWriter io.Writer
|
||||
}
|
||||
|
||||
// TerraformUpgradeOptions are the options used for the Terraform upgrade.
|
||||
@ -139,6 +142,13 @@ func (u *TerraformUpgrader) ApplyTerraformMigrations(ctx context.Context, fileHa
|
||||
return fmt.Errorf("terraform apply: %w", err)
|
||||
}
|
||||
|
||||
// AttestationURL is only set for Azure.
|
||||
if tfOutput.AttestationURL != "" {
|
||||
if err := u.policyPatcher.Patch(ctx, tfOutput.AttestationURL); err != nil {
|
||||
return fmt.Errorf("patching policies: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
outputFileContents := clusterid.File{
|
||||
CloudProvider: opts.CSP,
|
||||
InitSecret: []byte(tfOutput.Secret),
|
||||
@ -173,3 +183,8 @@ type tfClient interface {
|
||||
Plan(ctx context.Context, logLevel terraform.LogLevel, planFile string, targets ...string) (bool, error)
|
||||
CreateCluster(ctx context.Context, logLevel terraform.LogLevel, targets ...string) (terraform.CreateOutput, error)
|
||||
}
|
||||
|
||||
// policyPatcher interacts with the CSP (currently only applies for Azure) to update the attestation policy.
|
||||
type policyPatcher interface {
|
||||
Patch(ctx context.Context, attestationURL string) error
|
||||
}
|
||||
|
@ -164,6 +164,7 @@ func TestApplyTerraformMigrations(t *testing.T) {
|
||||
|
||||
testCases := map[string]struct {
|
||||
tf tfClient
|
||||
policyPatcher stubPolicyPatcher
|
||||
fs file.Handler
|
||||
outputFileName string
|
||||
wantErr bool
|
||||
@ -171,6 +172,7 @@ func TestApplyTerraformMigrations(t *testing.T) {
|
||||
"success": {
|
||||
tf: &stubTerraformClient{},
|
||||
fs: fileHandler(),
|
||||
policyPatcher: stubPolicyPatcher{},
|
||||
outputFileName: "test.json",
|
||||
},
|
||||
"create cluster error": {
|
||||
@ -178,18 +180,29 @@ func TestApplyTerraformMigrations(t *testing.T) {
|
||||
CreateClusterErr: assert.AnError,
|
||||
},
|
||||
fs: fileHandler(),
|
||||
policyPatcher: stubPolicyPatcher{},
|
||||
outputFileName: "test.json",
|
||||
wantErr: true,
|
||||
},
|
||||
"patch error": {
|
||||
tf: &stubTerraformClient{},
|
||||
fs: fileHandler(),
|
||||
policyPatcher: stubPolicyPatcher{
|
||||
patchErr: assert.AnError,
|
||||
},
|
||||
wantErr: true,
|
||||
},
|
||||
"empty file name": {
|
||||
tf: &stubTerraformClient{},
|
||||
fs: fileHandler(),
|
||||
policyPatcher: stubPolicyPatcher{},
|
||||
outputFileName: "",
|
||||
wantErr: true,
|
||||
},
|
||||
"file already exists": {
|
||||
tf: &stubTerraformClient{},
|
||||
fs: fileHandler("test.json"),
|
||||
policyPatcher: stubPolicyPatcher{},
|
||||
outputFileName: "test.json",
|
||||
wantErr: true,
|
||||
},
|
||||
@ -311,3 +324,11 @@ func (u *stubTerraformClient) Plan(context.Context, terraform.LogLevel, string,
|
||||
func (u *stubTerraformClient) CreateCluster(context.Context, terraform.LogLevel, ...string) (terraform.CreateOutput, error) {
|
||||
return terraform.CreateOutput{}, u.CreateClusterErr
|
||||
}
|
||||
|
||||
type stubPolicyPatcher struct {
|
||||
patchErr error
|
||||
}
|
||||
|
||||
func (p *stubPolicyPatcher) PatchPolicy(context.Context, string) error {
|
||||
return p.patchErr
|
||||
}
|
||||
|
@ -29,11 +29,11 @@ go_test(
|
||||
# keep
|
||||
count = 1,
|
||||
data = [
|
||||
"//cli:cli_oss_linux_amd64",
|
||||
"//cli:cli_enterprise_linux_amd64",
|
||||
],
|
||||
embed = [":upgrade"],
|
||||
env = {
|
||||
"PATH_CLI": "$(location //cli:cli_oss_linux_amd64)",
|
||||
"PATH_CLI": "$(location //cli:cli_enterprise_linux_amd64)",
|
||||
},
|
||||
# keep
|
||||
gotags = ["e2e"],
|
||||
|
@ -13,6 +13,7 @@ import (
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"os/exec"
|
||||
@ -74,9 +75,9 @@ func TestUpgrade(t *testing.T) {
|
||||
|
||||
// Migrate config if necessary.
|
||||
cmd := exec.CommandContext(context.Background(), cli, "config", "migrate", "--config", constants.ConfigFilename, "--force", "--debug")
|
||||
msg, err := cmd.CombinedOutput()
|
||||
require.NoError(err, string(msg))
|
||||
log.Println(string(msg))
|
||||
stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
|
||||
require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
|
||||
log.Println(string(stdout))
|
||||
|
||||
targetVersions := writeUpgradeConfig(require, *targetImage, *targetKubernetes, *targetMicroservices)
|
||||
|
||||
@ -94,9 +95,9 @@ func TestUpgrade(t *testing.T) {
|
||||
// The string after "Cluster status:" in the output might not be updated yet.
|
||||
// This is only updated after the operator finishes one reconcile loop.
|
||||
cmd = exec.CommandContext(context.Background(), cli, "status")
|
||||
msg, err = cmd.CombinedOutput()
|
||||
require.NoError(err, string(msg))
|
||||
log.Println(string(msg))
|
||||
stdout, stderr, err = runCommandWithSeparateOutputs(cmd)
|
||||
require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
|
||||
log.Println(string(stdout))
|
||||
|
||||
testMicroservicesEventuallyHaveVersion(t, targetVersions.microservices, *timeout)
|
||||
testNodesEventuallyHaveVersion(t, k, targetVersions, *wantControl+*wantWorker, *timeout)
|
||||
@ -287,27 +288,27 @@ func writeUpgradeConfig(require *require.Assertions, image string, kubernetes st
|
||||
// We can not check images upgrades because we might use unpublished images. CLI uses public CDN to check for available images.
|
||||
func runUpgradeCheck(require *require.Assertions, cli, targetKubernetes string) {
|
||||
cmd := exec.CommandContext(context.Background(), cli, "upgrade", "check")
|
||||
msg, err := cmd.Output()
|
||||
require.NoError(err, "%s", string(msg))
|
||||
stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
|
||||
require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
|
||||
|
||||
require.Contains(string(msg), "The following updates are available with this CLI:")
|
||||
require.Contains(string(msg), "Kubernetes:")
|
||||
require.Contains(string(stdout), "The following updates are available with this CLI:")
|
||||
require.Contains(string(stdout), "Kubernetes:")
|
||||
log.Printf("targetKubernetes: %s\n", targetKubernetes)
|
||||
|
||||
if targetKubernetes == "" {
|
||||
log.Printf("true\n")
|
||||
require.True(containsAny(string(msg), versions.SupportedK8sVersions()))
|
||||
require.True(containsAny(string(stdout), versions.SupportedK8sVersions()))
|
||||
} else {
|
||||
log.Printf("false. targetKubernetes: %s\n", targetKubernetes)
|
||||
require.Contains(string(msg), targetKubernetes, fmt.Sprintf("Expected Kubernetes version %s in output.", targetKubernetes))
|
||||
require.Contains(string(stdout), targetKubernetes, fmt.Sprintf("Expected Kubernetes version %s in output.", targetKubernetes))
|
||||
}
|
||||
|
||||
cliVersion, err := semver.New(constants.VersionInfo())
|
||||
require.NoError(err)
|
||||
require.Contains(string(msg), "Services:")
|
||||
require.Contains(string(msg), fmt.Sprintf("--> %s", cliVersion.String()))
|
||||
require.Contains(string(stdout), "Services:")
|
||||
require.Contains(string(stdout), fmt.Sprintf("--> %s", cliVersion.String()))
|
||||
|
||||
log.Println(string(msg))
|
||||
log.Println(string(stdout))
|
||||
}
|
||||
|
||||
func containsAny(text string, substrs []string) bool {
|
||||
@ -322,17 +323,17 @@ func containsAny(text string, substrs []string) bool {
|
||||
func runUpgradeApply(require *require.Assertions, cli string) {
|
||||
tfLogFlag := ""
|
||||
cmd := exec.CommandContext(context.Background(), cli, "--help")
|
||||
msg, err := cmd.CombinedOutput()
|
||||
require.NoErrorf(err, "%s", string(msg))
|
||||
if strings.Contains(string(msg), "--tf-log") {
|
||||
stdout, stderr, err := runCommandWithSeparateOutputs(cmd)
|
||||
require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
|
||||
if strings.Contains(string(stdout), "--tf-log") {
|
||||
tfLogFlag = "--tf-log=DEBUG"
|
||||
}
|
||||
|
||||
cmd = exec.CommandContext(context.Background(), cli, "upgrade", "apply", "--force", "--debug", "--yes", tfLogFlag)
|
||||
msg, err = cmd.CombinedOutput()
|
||||
require.NoErrorf(err, "%s", string(msg))
|
||||
require.NoError(containsUnexepectedMsg(string(msg)))
|
||||
log.Println(string(msg))
|
||||
stdout, stderr, err = runCommandWithSeparateOutputs(cmd)
|
||||
require.NoError(err, "Stdout: %s\nStderr: %s", string(stdout), string(stderr))
|
||||
require.NoError(containsUnexepectedMsg(string(stdout)))
|
||||
log.Println(string(stdout))
|
||||
}
|
||||
|
||||
// containsUnexepectedMsg checks if the given input contains any unexpected messages.
|
||||
@ -404,3 +405,42 @@ type versionContainer struct {
|
||||
kubernetes semver.Semver
|
||||
microservices string
|
||||
}
|
||||
|
||||
// runCommandWithSeparateOutputs runs the given command while separating buffers for
|
||||
// stdout and stderr.
|
||||
func runCommandWithSeparateOutputs(cmd *exec.Cmd) (stdout, stderr []byte, err error) {
|
||||
stdoutIn, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("create stdout pipe: %w", err)
|
||||
return
|
||||
}
|
||||
stderrIn, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("create stderr pipe: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
err = cmd.Start()
|
||||
if err != nil {
|
||||
err = fmt.Errorf("start command: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
stdout, err = io.ReadAll(stdoutIn)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("start command: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
stderr, err = io.ReadAll(stderrIn)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("start command: %w", err)
|
||||
return
|
||||
}
|
||||
|
||||
if err = cmd.Wait(); err != nil {
|
||||
err = fmt.Errorf("wait for command to finish: %w", err)
|
||||
}
|
||||
|
||||
return
|
||||
}
|
||||
|
Loading…
Reference in New Issue
Block a user