diff --git a/.github/workflows/e2e-test-release.yml b/.github/workflows/e2e-test-release.yml index aa39c8c0a..25e914a74 100644 --- a/.github/workflows/e2e-test-release.yml +++ b/.github/workflows/e2e-test-release.yml @@ -127,6 +127,10 @@ jobs: provider: "azure" kubernetes-version: "v1.27" runner: "ubuntu-22.04" + - test: "autoscaling" + provider: "aws" + kubernetes-version: "v1.27" + runner: "ubuntu-22.04" # perf-bench test on latest k8s version, not supported on AWS - test: "perf-bench" @@ -238,8 +242,8 @@ jobs: fail-fast: false max-parallel: 1 matrix: - fromVersion: ["v2.6.0"] - cloudProvider: ["gcp", "azure"] + fromVersion: ["v2.7.1"] + cloudProvider: ["gcp", "azure", "aws"] name: Run upgrade tests secrets: inherit permissions: diff --git a/.github/workflows/e2e-test-weekly.yml b/.github/workflows/e2e-test-weekly.yml index 891065fe2..db44896b6 100644 --- a/.github/workflows/e2e-test-weekly.yml +++ b/.github/workflows/e2e-test-weekly.yml @@ -253,8 +253,8 @@ jobs: fail-fast: false max-parallel: 1 matrix: - fromVersion: ["v2.6.0"] - cloudProvider: ["gcp", "azure"] + fromVersion: ["v2.7.1"] + cloudProvider: ["gcp", "azure", "aws"] name: Run upgrade tests secrets: inherit permissions: diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml index 98bb04baa..586449701 100644 --- a/.github/workflows/e2e-upgrade.yml +++ b/.github/workflows/e2e-upgrade.yml @@ -9,6 +9,7 @@ on: options: - "gcp" - "azure" + - "aws" default: "azure" workerNodesCount: description: "Number of worker nodes to spawn." diff --git a/bazel/toolchains/go_module_deps.bzl b/bazel/toolchains/go_module_deps.bzl index e96efe67e..987149534 100644 --- a/bazel/toolchains/go_module_deps.bzl +++ b/bazel/toolchains/go_module_deps.bzl @@ -363,8 +363,8 @@ def go_dependencies(): build_file_generation = "on", build_file_proto_mode = "disable_global", importpath = "github.com/aws/aws-sdk-go-v2", - sum = "h1:GMupCNNI7FARX27L7GjCJM8NgivWbRgpjNI/hOQjFS8=", - version = "v1.17.8", + sum = "h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY=", + version = "v1.18.0", ) go_repository( name = "com_github_aws_aws_sdk_go_v2_aws_protocol_eventstream", @@ -390,6 +390,7 @@ def go_dependencies(): sum = "h1:oZCEFcrMppP/CNiS8myzv9JgOzq2s0d3v3MXYil/mxQ=", version = "v1.13.20", ) + go_repository( name = "com_github_aws_aws_sdk_go_v2_feature_ec2_imds", build_file_generation = "on", @@ -411,16 +412,16 @@ def go_dependencies(): build_file_generation = "on", build_file_proto_mode = "disable_global", importpath = "github.com/aws/aws-sdk-go-v2/internal/configsources", - sum = "h1:dpbVNUjczQ8Ae3QKHbpHBpfvaVkRdesxpTOe9pTouhU=", - version = "v1.1.32", + sum = "h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4=", + version = "v1.1.33", ) go_repository( name = "com_github_aws_aws_sdk_go_v2_internal_endpoints_v2", build_file_generation = "on", build_file_proto_mode = "disable_global", importpath = "github.com/aws/aws-sdk-go-v2/internal/endpoints/v2", - sum = "h1:QH2kOS3Ht7x+u0gHCh06CXL/h6G8LQJFpZfFBYBNboo=", - version = "v2.4.26", + sum = "h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk=", + version = "v2.4.27", ) go_repository( name = "com_github_aws_aws_sdk_go_v2_internal_ini", @@ -438,6 +439,14 @@ def go_dependencies(): sum = "h1:DWYZIsyqagnWL00f8M/SOr9fN063OEQWn9LLTbdYXsk=", version = "v1.0.23", ) + go_repository( + name = "com_github_aws_aws_sdk_go_v2_service_autoscaling", + build_file_generation = "on", + build_file_proto_mode = "disable_global", + importpath = "github.com/aws/aws-sdk-go-v2/service/autoscaling", + sum = "h1:OpzahvFZn/B+TNWLZf0ARovZoQB0Q2MvM+y13gdL+WY=", + version = "v1.28.6", + ) go_repository( name = "com_github_aws_aws_sdk_go_v2_service_cloudfront", @@ -576,6 +585,7 @@ def go_dependencies(): sum = "h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8=", version = "v1.13.5", ) + go_repository( name = "com_github_aybabtme_rgbterm", build_file_generation = "on", @@ -5824,8 +5834,8 @@ def go_dependencies(): build_file_generation = "on", build_file_proto_mode = "disable_global", importpath = "github.com/rogpeppe/go-internal", - sum = "h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=", - version = "v1.9.0", + sum = "h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=", + version = "v1.10.0", ) go_repository( name = "com_github_rs_cors", @@ -9165,8 +9175,8 @@ def go_dependencies(): build_file_generation = "on", build_file_proto_mode = "disable_global", importpath = "golang.org/x/vuln", - sum = "h1:SJ0lK20LZB3cfTHvYOXH2m7DCIEaFdSlXtICBRv5bYU=", - version = "v0.0.0-20230411201117-aaaefcd264f6", + sum = "h1:9GRdj6wAIkDrsMevuolY+SXERPjQPp2P1ysYA0jpZe0=", + version = "v0.1.0", ) go_repository( diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index fcafe20b8..c6e441c0d 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -189,7 +189,7 @@ func (k *KubeWrapper) InitCluster( log.Infof("Waiting for Cilium to become healthy") timeToStartWaiting := time.Now() - // TODO(Nirusu): Reduce the timeout when we switched the package repository - this is only this high because I once + // TODO(3u13r): Reduce the timeout when we switched the package repository - this is only this high because we once // saw polling times of ~16 minutes when hitting a slow PoP from Fastly (GitHub's / ghcr.io CDN). waitCtx, cancel = context.WithTimeout(ctx, 20*time.Minute) defer cancel() diff --git a/cli/internal/cmd/upgradeapply.go b/cli/internal/cmd/upgradeapply.go index 3df5952c4..28157172d 100644 --- a/cli/internal/cmd/upgradeapply.go +++ b/cli/internal/cmd/upgradeapply.go @@ -94,7 +94,7 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand return fmt.Errorf("upgrading measurements: %w", err) } - if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP { + if conf.GetProvider() == cloudprovider.Azure || conf.GetProvider() == cloudprovider.GCP || conf.GetProvider() == cloudprovider.AWS { err = u.handleServiceUpgrade(cmd, conf, flags) upgradeErr := &compatibility.InvalidUpgradeError{} switch { @@ -114,7 +114,7 @@ func (u *upgradeApplyCmd) upgradeApply(cmd *cobra.Command, fileHandler file.Hand return fmt.Errorf("upgrading NodeVersion: %w", err) } } else { - cmd.PrintErrln("WARNING: Skipping service and image upgrades, which are currently only supported for Azure and GCP.") + cmd.PrintErrln("WARNING: Skipping service and image upgrades, which are currently only supported for AWS, Azure, and GCP.") } return nil diff --git a/cli/internal/terraform/terraform/iam/aws/main.tf b/cli/internal/terraform/terraform/iam/aws/main.tf index fafb1bbbb..3457fd6ff 100644 --- a/cli/internal/terraform/terraform/iam/aws/main.tf +++ b/cli/internal/terraform/terraform/iam/aws/main.tf @@ -112,7 +112,13 @@ resource "aws_iam_policy" "control_plane_policy" { "logs:DescribeLogGroups", "logs:ListTagsLogGroup", "logs:PutLogEvents", - "tag:GetResources" + "tag:GetResources", + "ec2:DescribeLaunchTemplateVersions", + "autoscaling:SetDesiredCapacity", + "autoscaling:TerminateInstanceInAutoScalingGroup", + "ec2:DescribeInstanceStatus", + "ec2:CreateLaunchTemplateVersion", + "ec2:ModifyLaunchTemplate" ], "Resource": [ "*" diff --git a/docs/docs/workflows/scale.md b/docs/docs/workflows/scale.md index 3b7c0d479..9531e90c9 100644 --- a/docs/docs/workflows/scale.md +++ b/docs/docs/workflows/scale.md @@ -66,11 +66,9 @@ Alternatively, you can manually scale your cluster up or down: -:::caution - -Scaling isn't yet implemented for AWS. If you require this feature, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md)! - -::: +1. Go to Auto Scaling Groups and select the worker ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. @@ -100,11 +98,9 @@ To increase the number of control-plane nodes, follow these steps: -:::caution - -Scaling isn't yet implemented for AWS. If you require this feature, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md)! - -::: +1. Go to Auto Scaling Groups and select the control-plane ASG to scale up. +2. Click **Edit** +3. Set the new (increased) **Desired capacity** and **Update**. diff --git a/docs/docs/workflows/upgrade.md b/docs/docs/workflows/upgrade.md index cbd58b09f..497b336bf 100644 --- a/docs/docs/workflows/upgrade.md +++ b/docs/docs/workflows/upgrade.md @@ -6,12 +6,6 @@ You configure the desired versions in your local Constellation configuration and To learn about available versions you use the `upgrade check` command. Which versions are available depends on the CLI version you are using. -:::caution - -Upgrades aren't yet implemented for AWS. If you require this feature, [let us know](https://github.com/edgelesssys/constellation/issues/new?assignees=&labels=&template=feature_request.md)! - -::: - ## Update the CLI Each CLI comes with a set of supported microservice and Kubernetes versions. diff --git a/hack/tools/go.mod b/hack/tools/go.mod index 84dd34606..f9db58a5b 100644 --- a/hack/tools/go.mod +++ b/hack/tools/go.mod @@ -5,8 +5,8 @@ go 1.20 require ( github.com/google/go-licenses v1.6.0 github.com/katexochen/sh/v3 v3.6.0 - golang.org/x/tools v0.9.1 - golang.org/x/vuln v0.0.0-20230411201117-aaaefcd264f6 + golang.org/x/tools v0.8.1-0.20230421161920-b9619ee54b47 + golang.org/x/vuln v0.1.0 ) require ( diff --git a/hack/tools/go.sum b/hack/tools/go.sum index 65c75d335..0b19fcc31 100644 --- a/hack/tools/go.sum +++ b/hack/tools/go.sum @@ -572,10 +572,10 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.5.0/go.mod h1:N+Kgy78s5I24c24dU8OfWNEotWjutIs8SnJvn5IDq+k= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= -golang.org/x/vuln v0.0.0-20230411201117-aaaefcd264f6 h1:SJ0lK20LZB3cfTHvYOXH2m7DCIEaFdSlXtICBRv5bYU= -golang.org/x/vuln v0.0.0-20230411201117-aaaefcd264f6/go.mod h1:64LpnL2PuSMzFYeCmJjYiRbroOUG9aCZYznINnF5PHE= +golang.org/x/tools v0.8.1-0.20230421161920-b9619ee54b47 h1:fQlOhMJ24apqitZX8S4hbCbHU1Z9AvyWkN3BYI55Le4= +golang.org/x/tools v0.8.1-0.20230421161920-b9619ee54b47/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= +golang.org/x/vuln v0.1.0 h1:9GRdj6wAIkDrsMevuolY+SXERPjQPp2P1ysYA0jpZe0= +golang.org/x/vuln v0.1.0/go.mod h1:/YuzZYjGbwB8y19CisAppfyw3uTZnuCz3r+qgx/QRzU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/operators/constellation-node-operator/BUILD.bazel b/operators/constellation-node-operator/BUILD.bazel index 36dff6e1b..cabfdb200 100644 --- a/operators/constellation-node-operator/BUILD.bazel +++ b/operators/constellation-node-operator/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "//3rdparty/node-maintenance-operator/api/v1beta1", "//operators/constellation-node-operator/api/v1alpha1", "//operators/constellation-node-operator/controllers", + "//operators/constellation-node-operator/internal/cloud/aws/client", "//operators/constellation-node-operator/internal/cloud/azure/client", "//operators/constellation-node-operator/internal/cloud/fake/client", "//operators/constellation-node-operator/internal/cloud/gcp/client", diff --git a/operators/constellation-node-operator/controllers/nodeversion_controller.go b/operators/constellation-node-operator/controllers/nodeversion_controller.go index c2e0693c6..d37e569a7 100644 --- a/operators/constellation-node-operator/controllers/nodeversion_controller.go +++ b/operators/constellation-node-operator/controllers/nodeversion_controller.go @@ -217,6 +217,7 @@ func (r *NodeVersionReconciler) Reconcile(ctx context.Context, req ctrl.Request) newNodeConfig := newNodeConfig{desiredNodeVersion, groups.Outdated, pendingNodeList.Items, scalingGroupByID, newNodesBudget} if err := r.createNewNodes(ctx, newNodeConfig); err != nil { + logr.Error(err, "Creating new nodes") return ctrl.Result{Requeue: shouldRequeue}, nil } // cleanup obsolete nodes diff --git a/operators/constellation-node-operator/controllers/pendingnode_controller.go b/operators/constellation-node-operator/controllers/pendingnode_controller.go index b9ed266d6..b8756828d 100644 --- a/operators/constellation-node-operator/controllers/pendingnode_controller.go +++ b/operators/constellation-node-operator/controllers/pendingnode_controller.go @@ -63,6 +63,7 @@ func NewPendingNodeReconciler(nodeStateGetter nodeStateGetter, client client.Cli // If the node is trying to join the cluster and fails to join within the deadline referenced in the PendingNode spec, the node is deleted. func (r *PendingNodeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { logr := log.FromContext(ctx) + logr.Info("Reconciling PendingNode", "pendingNode", req.NamespacedName) var pendingNode updatev1alpha1.PendingNode if err := r.Get(ctx, req.NamespacedName, &pendingNode); err != nil { diff --git a/operators/constellation-node-operator/go.mod b/operators/constellation-node-operator/go.mod index 2ae5d5730..e6017c65b 100644 --- a/operators/constellation-node-operator/go.mod +++ b/operators/constellation-node-operator/go.mod @@ -13,6 +13,9 @@ require ( github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.0 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.2.2 github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/compute/armcompute/v4 v4.2.1 + github.com/aws/aws-sdk-go-v2/config v1.18.21 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 + github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1 github.com/edgelesssys/constellation/v2 v2.6.0 github.com/edgelesssys/constellation/v2/3rdparty/node-maintenance-operator v0.0.0 github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api v0.0.0 @@ -35,14 +38,26 @@ require ( ) require ( + github.com/aws/aws-sdk-go-v2 v1.18.0 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.13.20 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.12.8 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/google/s2a-go v0.1.2 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect ) require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0 // indirect + github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.6 github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/coreos/go-semver v0.3.1 // indirect diff --git a/operators/constellation-node-operator/go.sum b/operators/constellation-node-operator/go.sum index 0d9e9fbaf..824aad05d 100644 --- a/operators/constellation-node-operator/go.sum +++ b/operators/constellation-node-operator/go.sum @@ -59,6 +59,41 @@ github.com/AzureAD/microsoft-authentication-library-for-go v0.9.0/go.mod h1:kgDm github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/aws/aws-sdk-go-v2 v1.17.7/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.17.8/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2 v1.18.0 h1:882kkTpSFhdgYRKVZ/VCgf7sd0ru57p2JCxz4/oN5RY= +github.com/aws/aws-sdk-go-v2 v1.18.0/go.mod h1:uzbQtefpm44goOPmdKyAlXSNcwlRgF3ePWVW6EtJvvw= +github.com/aws/aws-sdk-go-v2/config v1.18.21 h1:ENTXWKwE8b9YXgQCsruGLhvA9bhg+RqAsL9XEMEsa2c= +github.com/aws/aws-sdk-go-v2/config v1.18.21/go.mod h1:+jPQiVPz1diRnjj6VGqWcLK6EzNmQ42l7J3OqGTLsSY= +github.com/aws/aws-sdk-go-v2/credentials v1.13.20 h1:oZCEFcrMppP/CNiS8myzv9JgOzq2s0d3v3MXYil/mxQ= +github.com/aws/aws-sdk-go-v2/credentials v1.13.20/go.mod h1:xtZnXErtbZ8YGXC3+8WfajpMBn5Ga/3ojZdxHq6iI8o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2 h1:jOzQAesnBFDmz93feqKnsTHsXrlwWORNZMFHMV+WLFU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.2/go.mod h1:cDh1p6XkSGSwSRIArWRc6+UqAQ7x4alQ0QfpVR6f+co= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.31/go.mod h1:QT0BqUvX1Bh2ABdTGnjqEjvjzrCfIniM9Sc8zn9Yndo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.32/go.mod h1:RudqOgadTWdcS3t/erPQo24pcVEoYyqj/kKW5Vya21I= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33 h1:kG5eQilShqmJbv11XL1VpyDbaEJzWxd4zRiCG30GSn4= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.33/go.mod h1:7i0PF1ME/2eUPFcjkVIwq+DOygHEoK92t5cDqNgYbIw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.25/go.mod h1:zBHOPwhBc3FlQjQJE/D3IfPWiWaQmT06Vq9aNukDo0k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.26/go.mod h1:vq86l7956VgFr0/FWQ2BWnK07QC3WYsepKzy33qqY5U= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27 h1:vFQlirhuM8lLlpI7imKOMsjdQLuN9CPi+k44F/OFVsk= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.27/go.mod h1:UrHnn3QV/d0pBZ6QBAEQcqFLf8FAzLmoUfPVIueOvoM= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33 h1:HbH1VjUgrCdLJ+4lnnuLI4iVNRvBbBELGaJ5f69ClA8= +github.com/aws/aws-sdk-go-v2/internal/ini v1.3.33/go.mod h1:zG2FcwjQarWaqXSCGpgcr3RSjZ6dHGguZSppUL0XR7Q= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.6 h1:OpzahvFZn/B+TNWLZf0ARovZoQB0Q2MvM+y13gdL+WY= +github.com/aws/aws-sdk-go-v2/service/autoscaling v1.28.6/go.mod h1:cQ05ETcKMluA1/g1/jMQTD/qv9E1WeYCyHmqErEoHBk= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1 h1:xn5CI639mnWvdiweqoRx/H221Ia9Asx9XxfIRhe0MPo= +github.com/aws/aws-sdk-go-v2/service/ec2 v1.92.1/go.mod h1:ZZLfkd1Y7fjXujjMg1CFqNmaTl314eCbShlHQO7VTWo= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.25/go.mod h1:/95IA+0lMnzW6XzqYJRpjjsAbKEORVeO0anQqjd2CNU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26 h1:uUt4XctZLhl9wBE1L8lobU3bVN8SNUP7T+olb0bWBO4= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.26/go.mod h1:Bd4C/4PkVGubtNe5iMXu5BNnaBi/9t/UsFspPt4ram8= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.8 h1:5cb3D6xb006bPTqEfCNaEA6PPEfBXxxy4NNeX/44kGk= +github.com/aws/aws-sdk-go-v2/service/sso v1.12.8/go.mod h1:GNIveDnP+aE3jujyUSH5aZ/rktsTM5EvtKnCqBZawdw= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8 h1:NZaj0ngZMzsubWZbrEFSB4rgSQRbFq38Sd6KBxHuOIU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.8/go.mod h1:44qFP1g7pfd+U+sQHLPalAPKnyfTZjJsYR4xIwsJy5o= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.9 h1:Qf1aWwnsNkyAoqDqmdM3nHwN78XQjec27LjM6b9vyfI= +github.com/aws/aws-sdk-go-v2/service/sts v1.18.9/go.mod h1:yyW88BEPXA2fGFyI2KCcZC3dNpiT0CZAHaF+i656/tQ= +github.com/aws/smithy-go v1.13.5 h1:hgz0X/DX0dGqTYpGALqXJoRKRj5oQ7150i5FdTePzO8= +github.com/aws/smithy-go v1.13.5/go.mod h1:Tg+OJXh4MB2R/uN61Ko2f6hTZwB/ZYGOtib8J3gBHzA= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -173,6 +208,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -213,6 +249,10 @@ github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1: github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -275,8 +315,8 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/spf13/afero v1.9.5 h1:stMpOSZFs//0Lv29HduCmli3GUfpFoF3Y1Q/aXj/wVM= github.com/spf13/afero v1.9.5/go.mod h1:UBogFpq8E9Hx+xc5CNTTEpTnuHVmXDwZcZcE1eb/UhQ= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/BUILD.bazel b/operators/constellation-node-operator/internal/cloud/aws/client/BUILD.bazel new file mode 100644 index 000000000..948105967 --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/BUILD.bazel @@ -0,0 +1,46 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library") +load("//bazel/go:go_test.bzl", "go_test") + +go_library( + name = "client", + srcs = [ + "api.go", + "autoscaler.go", + "client.go", + "nodeimage.go", + "pendingnode.go", + "scalinggroup.go", + ], + importpath = "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/aws/client", + visibility = ["//operators/constellation-node-operator:__subpackages__"], + deps = [ + "//operators/constellation-node-operator/api/v1alpha1", + "@com_github_aws_aws_sdk_go_v2_config//:config", + "@com_github_aws_aws_sdk_go_v2_feature_ec2_imds//:imds", + "@com_github_aws_aws_sdk_go_v2_service_autoscaling//:autoscaling", + "@com_github_aws_aws_sdk_go_v2_service_autoscaling//types", + "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2", + "@com_github_aws_aws_sdk_go_v2_service_ec2//types", + "@io_k8s_sigs_controller_runtime//pkg/log", + ], +) + +go_test( + name = "client_test", + srcs = [ + "client_test.go", + "nodeimage_test.go", + "pendingnode_test.go", + "scalinggroup_test.go", + ], + embed = [":client"], + deps = [ + "//operators/constellation-node-operator/api/v1alpha1", + "@com_github_aws_aws_sdk_go_v2_service_autoscaling//:autoscaling", + "@com_github_aws_aws_sdk_go_v2_service_autoscaling//types", + "@com_github_aws_aws_sdk_go_v2_service_ec2//:ec2", + "@com_github_aws_aws_sdk_go_v2_service_ec2//types", + "@com_github_stretchr_testify//assert", + "@com_github_stretchr_testify//require", + ], +) diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/api.go b/operators/constellation-node-operator/internal/cloud/aws/client/api.go new file mode 100644 index 000000000..53316583b --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/api.go @@ -0,0 +1,28 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "context" + + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + "github.com/aws/aws-sdk-go-v2/service/ec2" +) + +type ec2API interface { + DescribeInstances(ctx context.Context, params *ec2.DescribeInstancesInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) + DescribeInstanceStatus(ctx context.Context, params *ec2.DescribeInstanceStatusInput, optFns ...func(*ec2.Options)) (*ec2.DescribeInstanceStatusOutput, error) + CreateLaunchTemplateVersion(ctx context.Context, params *ec2.CreateLaunchTemplateVersionInput, optFns ...func(*ec2.Options)) (*ec2.CreateLaunchTemplateVersionOutput, error) + ModifyLaunchTemplate(ctx context.Context, params *ec2.ModifyLaunchTemplateInput, optFns ...func(*ec2.Options)) (*ec2.ModifyLaunchTemplateOutput, error) + DescribeLaunchTemplateVersions(ctx context.Context, params *ec2.DescribeLaunchTemplateVersionsInput, optFns ...func(*ec2.Options)) (*ec2.DescribeLaunchTemplateVersionsOutput, error) +} + +type scalingAPI interface { + DescribeAutoScalingGroups(ctx context.Context, params *autoscaling.DescribeAutoScalingGroupsInput, optFns ...func(*autoscaling.Options)) (*autoscaling.DescribeAutoScalingGroupsOutput, error) + SetDesiredCapacity(ctx context.Context, params *autoscaling.SetDesiredCapacityInput, optFns ...func(*autoscaling.Options)) (*autoscaling.SetDesiredCapacityOutput, error) + TerminateInstanceInAutoScalingGroup(ctx context.Context, params *autoscaling.TerminateInstanceInAutoScalingGroupInput, optFns ...func(*autoscaling.Options)) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/autoscaler.go b/operators/constellation-node-operator/internal/cloud/aws/client/autoscaler.go new file mode 100644 index 000000000..e74ef3b9b --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/autoscaler.go @@ -0,0 +1,12 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +// AutoscalingCloudProvider returns the cloud-provider name as used by k8s cluster-autoscaler. +func (c *Client) AutoscalingCloudProvider() string { + return "aws" +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/client.go b/operators/constellation-node-operator/internal/cloud/aws/client/client.go new file mode 100644 index 000000000..b477e76ef --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/client.go @@ -0,0 +1,65 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds" + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + "github.com/aws/aws-sdk-go-v2/service/ec2" +) + +// Client is a client for the AWS Cloud. +type Client struct { + ec2Client ec2API + scalingClient scalingAPI +} + +// New creates a client with initialized clients. +func New(ctx context.Context) (*Client, error) { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load aws config: %w", err) + } + // get region from ec2metadata + imdsClient := imds.NewFromConfig(cfg) + regionOut, err := imdsClient.GetRegion(ctx, &imds.GetRegionInput{}) + if err != nil { + return nil, fmt.Errorf("failed to get region from ec2metadata: %w", err) + } + return NewWithRegion(ctx, regionOut.Region) +} + +// NewWithRegion creates a client with initialized clients and a given region. +func NewWithRegion(ctx context.Context, region string) (*Client, error) { + cfg, err := config.LoadDefaultConfig(ctx) + if err != nil { + return nil, fmt.Errorf("failed to load aws config: %w", err) + } + + cfg.Region = region + + ec2Client := ec2.NewFromConfig(cfg) + scalingClient := autoscaling.NewFromConfig(cfg) + return &Client{ + ec2Client: ec2Client, + scalingClient: scalingClient, + }, nil +} + +func getInstanceNameFromProviderID(providerID string) (string, error) { + // aws:///us-east-2a/i-06888991e7138ed4e + providerIDParts := strings.Split(providerID, "/") + if len(providerIDParts) != 5 { + return "", fmt.Errorf("invalid providerID: %s", providerID) + } + return providerIDParts[4], nil +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/client_test.go b/operators/constellation-node-operator/internal/cloud/aws/client/client_test.go new file mode 100644 index 000000000..a2e81cc28 --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/client_test.go @@ -0,0 +1,50 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetInstanceNameFromProviderID(t *testing.T) { + testCases := map[string]struct { + providerID string + want string + wantErr bool + }{ + "valid": { + providerID: "aws:///us-east-2a/i-06888991e7138ed4e", + want: "i-06888991e7138ed4e", + }, + "too many parts": { + providerID: "aws:///us-east-2a/i-06888991e7138ed4e/invalid", + wantErr: true, + }, + "too few parts": { + providerID: "aws:///us-east-2a", + wantErr: true, + }, + } + + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + got, err := getInstanceNameFromProviderID(tc.providerID) + if tc.wantErr { + require.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.want, got) + }) + } +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage.go b/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage.go new file mode 100644 index 000000000..61d6026ee --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage.go @@ -0,0 +1,219 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "context" + "fmt" + "time" + + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" +) + +// GetNodeImage returns the image name of the node. +func (c *Client) GetNodeImage(ctx context.Context, providerID string) (string, error) { + instanceName, err := getInstanceNameFromProviderID(providerID) + if err != nil { + return "", fmt.Errorf("failed to get instance name from providerID: %w", err) + } + + params := &ec2.DescribeInstancesInput{ + InstanceIds: []string{ + instanceName, + }, + } + + resp, err := c.ec2Client.DescribeInstances(ctx, params) + if err != nil { + return "", fmt.Errorf("failed to describe instances: %w", err) + } + + if len(resp.Reservations) == 0 { + return "", fmt.Errorf("no reservations for instance %q", instanceName) + } + + if len(resp.Reservations[0].Instances) == 0 { + return "", fmt.Errorf("no instances for instance %q", instanceName) + } + + if resp.Reservations[0].Instances[0].ImageId == nil { + return "", fmt.Errorf("no image for instance %q", instanceName) + } + + return *resp.Reservations[0].Instances[0].ImageId, nil +} + +// GetScalingGroupID returns the scaling group ID of the node. +func (c *Client) GetScalingGroupID(ctx context.Context, providerID string) (string, error) { + instanceName, err := getInstanceNameFromProviderID(providerID) + if err != nil { + return "", fmt.Errorf("failed to get instance name from providerID: %w", err) + } + params := &ec2.DescribeInstancesInput{ + InstanceIds: []string{ + instanceName, + }, + } + + resp, err := c.ec2Client.DescribeInstances(ctx, params) + if err != nil { + return "", fmt.Errorf("failed to describe instances: %w", err) + } + + if len(resp.Reservations) == 0 { + return "", fmt.Errorf("no reservations for instance %q", instanceName) + } + + if len(resp.Reservations[0].Instances) == 0 { + return "", fmt.Errorf("no instances for instance %q", instanceName) + } + + if resp.Reservations[0].Instances[0].Tags == nil { + return "", fmt.Errorf("no tags for instance %q", instanceName) + } + + for _, tag := range resp.Reservations[0].Instances[0].Tags { + if tag.Key == nil || tag.Value == nil { + continue + } + if *tag.Key == "aws:autoscaling:groupName" { + return *tag.Value, nil + } + } + + return "", fmt.Errorf("node %q does not have valid tags", providerID) +} + +// CreateNode creates a node in the specified scaling group. +func (c *Client) CreateNode(ctx context.Context, scalingGroupID string) (nodeName, providerID string, err error) { + containsInstance := func(instances []types.Instance, target types.Instance) bool { + for _, i := range instances { + if i.InstanceId == nil || target.InstanceId == nil { + continue + } + if *i.InstanceId == *target.InstanceId { + return true + } + } + return false + } + + // get current capacity + groups, err := c.scalingClient.DescribeAutoScalingGroups( + ctx, + &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: []string{scalingGroupID}, + }, + ) + if err != nil { + return "", "", fmt.Errorf("failed to describe autoscaling group: %w", err) + } + + if len(groups.AutoScalingGroups) != 1 { + return "", "", fmt.Errorf("expected exactly one autoscaling group, got %d", len(groups.AutoScalingGroups)) + } + + if groups.AutoScalingGroups[0].DesiredCapacity == nil { + return "", "", fmt.Errorf("desired capacity is nil") + } + currentCapacity := int(*groups.AutoScalingGroups[0].DesiredCapacity) + + // check for int32 overflow + if currentCapacity >= int(^uint32(0)>>1) { + return "", "", fmt.Errorf("current capacity is at maximum") + } + + // get current list of instances + previousInstances := groups.AutoScalingGroups[0].Instances + + // create new instance by increasing capacity by 1 + _, err = c.scalingClient.SetDesiredCapacity( + ctx, + &autoscaling.SetDesiredCapacityInput{ + AutoScalingGroupName: &scalingGroupID, + DesiredCapacity: toPtr(int32(currentCapacity + 1)), + }, + ) + if err != nil { + return "", "", fmt.Errorf("failed to set desired capacity: %w", err) + } + + // poll until new instance is created with 30 second timeout + newInstance := types.Instance{} + for i := 0; i < 30; i++ { + groups, err := c.scalingClient.DescribeAutoScalingGroups( + ctx, + &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: []string{scalingGroupID}, + }, + ) + if err != nil { + return "", "", fmt.Errorf("failed to describe autoscaling group: %w", err) + } + + if len(groups.AutoScalingGroups) != 1 { + return "", "", fmt.Errorf("expected exactly one autoscaling group, got %d", len(groups.AutoScalingGroups)) + } + + for _, instance := range groups.AutoScalingGroups[0].Instances { + if !containsInstance(previousInstances, instance) { + newInstance = instance + break + } + } + + // break if new instance is found + if newInstance.InstanceId != nil { + break + } + + // wait 1 second + select { + case <-ctx.Done(): + return "", "", fmt.Errorf("context cancelled") + case <-time.After(1 * time.Second): + } + } + + if newInstance.InstanceId == nil { + return "", "", fmt.Errorf("timed out waiting for new instance") + } + + if newInstance.AvailabilityZone == nil { + return "", "", fmt.Errorf("new instance %s does not have availability zone", *newInstance.InstanceId) + } + + // return new instance + return *newInstance.InstanceId, fmt.Sprintf("aws:///%s/%s", *newInstance.AvailabilityZone, *newInstance.InstanceId), nil +} + +// DeleteNode deletes a node from the specified scaling group. +func (c *Client) DeleteNode(ctx context.Context, providerID string) error { + instanceID, err := getInstanceNameFromProviderID(providerID) + if err != nil { + return fmt.Errorf("failed to get instance name from providerID: %w", err) + } + + _, err = c.scalingClient.TerminateInstanceInAutoScalingGroup( + ctx, + &autoscaling.TerminateInstanceInAutoScalingGroupInput{ + InstanceId: &instanceID, + ShouldDecrementDesiredCapacity: toPtr(true), + }, + ) + if err != nil { + return fmt.Errorf("failed to terminate instance: %w", err) + } + + return nil +} + +func toPtr[T any](v T) *T { + return &v +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage_test.go b/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage_test.go new file mode 100644 index 000000000..1bae66ec0 --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/nodeimage_test.go @@ -0,0 +1,459 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + autoscalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetNodeImage(t *testing.T) { + ami := "ami-00000000000000000" + testCases := map[string]struct { + providerID string + describeInstancesErr error + describeInstancesOut *ec2.DescribeInstancesOutput + wantImage string + wantErr bool + }{ + "getting node image works": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + ImageId: &ami, + }, + }, + }, + }, + }, + wantImage: ami, + }, + "no reservations": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{}, + }, + wantErr: true, + }, + "no instances": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{}, + }, + }, + }, + wantErr: true, + }, + "no image": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + {}, + }, + }, + }, + }, + wantErr: true, + }, + "error describing instances": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesErr: assert.AnError, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + ec2Client: &stubEC2API{ + describeInstancesOut: tc.describeInstancesOut, + describeInstancesErr: tc.describeInstancesErr, + }, + } + gotImage, err := client.GetNodeImage(context.Background(), tc.providerID) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantImage, gotImage) + }) + } +} + +func TestGetScalingGroupID(t *testing.T) { + asgName := "my-asg" + testCases := map[string]struct { + providerID string + describeInstancesErr error + describeInstancesOut *ec2.DescribeInstancesOutput + wantASGID string + wantErr bool + }{ + "getting node's tag works": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + Tags: []ec2types.Tag{ + { + Key: toPtr("aws:autoscaling:groupName"), + Value: &asgName, + }, + }, + }, + }, + }, + }, + }, + wantASGID: asgName, + }, + "no valid tags": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + { + Tags: []ec2types.Tag{ + { + Key: toPtr("foo"), + Value: toPtr("bar"), + }, + }, + }, + }, + }, + }, + }, + wantErr: true, + }, + "no reservations": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{}, + }, + wantErr: true, + }, + "no instances": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{}, + }, + }, + }, + wantErr: true, + }, + "no image": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesOut: &ec2.DescribeInstancesOutput{ + Reservations: []ec2types.Reservation{ + { + Instances: []ec2types.Instance{ + {}, + }, + }, + }, + }, + wantErr: true, + }, + "error describing instances": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstancesErr: assert.AnError, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + ec2Client: &stubEC2API{ + describeInstancesOut: tc.describeInstancesOut, + describeInstancesErr: tc.describeInstancesErr, + }, + } + gotScalingID, err := client.GetScalingGroupID(context.Background(), tc.providerID) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantASGID, gotScalingID) + }) + } +} + +func TestCreateNode(t *testing.T) { + testCases := map[string]struct { + providerID string + describeAutoscalingOutFirst *autoscaling.DescribeAutoScalingGroupsOutput + describeAutoscalingFirstErr error + describeAutoscalingOutSecond *autoscaling.DescribeAutoScalingGroupsOutput + describeAutoscalingSecondErr error + setDesiredCapacityErr error + wantNodeName string + wantProviderID string + wantErr bool + }{ + "creating a new node works": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoscalingOutFirst: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ + { + AutoScalingGroupName: toPtr("my-asg"), + Instances: []autoscalingtypes.Instance{ + { + InstanceId: toPtr("i-00000000000000000"), + }, + }, + DesiredCapacity: toPtr(int32(1)), + }, + }, + }, + describeAutoscalingOutSecond: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ + { + AutoScalingGroupName: toPtr("my-asg"), + Instances: []autoscalingtypes.Instance{ + { + InstanceId: toPtr("i-00000000000000000"), + }, + { + InstanceId: toPtr("i-00000000000000001"), + AvailabilityZone: toPtr("us-east-2a"), + }, + }, + DesiredCapacity: toPtr(int32(2)), + }, + }, + }, + wantNodeName: "i-00000000000000001", + wantProviderID: "aws:///us-east-2a/i-00000000000000001", + }, + "creating a new node fails when describing the auto scaling group the first time": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoscalingFirstErr: assert.AnError, + wantErr: true, + }, + "creating a new node fails when describing the auto scaling group the second time": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoscalingOutFirst: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ + { + AutoScalingGroupName: toPtr("my-asg"), + Instances: []autoscalingtypes.Instance{ + { + InstanceId: toPtr("i-00000000000000000"), + }, + }, + DesiredCapacity: toPtr(int32(1)), + }, + }, + }, + describeAutoscalingSecondErr: assert.AnError, + wantErr: true, + }, + "creating a new node fails when the auto scaling group is not found": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoscalingOutFirst: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{}, + }, + wantErr: true, + }, + "creating a new node fails when set desired capacity fails": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoscalingOutFirst: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ + { + AutoScalingGroupName: toPtr("my-asg"), + Instances: []autoscalingtypes.Instance{ + { + InstanceId: toPtr("i-00000000000000000"), + }, + }, + DesiredCapacity: toPtr(int32(1)), + }, + }, + }, + setDesiredCapacityErr: assert.AnError, + wantErr: true, + }, + "creating a new node fails when the found vm does not contain an availability zone": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoscalingOutFirst: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ + { + AutoScalingGroupName: toPtr("my-asg"), + Instances: []autoscalingtypes.Instance{ + { + InstanceId: toPtr("i-00000000000000000"), + }, + }, + DesiredCapacity: toPtr(int32(1)), + }, + }, + }, + describeAutoscalingOutSecond: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []autoscalingtypes.AutoScalingGroup{ + { + AutoScalingGroupName: toPtr("my-asg"), + Instances: []autoscalingtypes.Instance{ + { + InstanceId: toPtr("i-00000000000000000"), + }, + { + InstanceId: toPtr("i-00000000000000001"), + }, + }, + DesiredCapacity: toPtr(int32(2)), + }, + }, + }, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + scalingClient: &stubAutoscalingAPI{ + describeAutoScalingGroupsOut: []*autoscaling.DescribeAutoScalingGroupsOutput{ + tc.describeAutoscalingOutFirst, + tc.describeAutoscalingOutSecond, + }, + describeAutoScalingGroupsErr: []error{ + tc.describeAutoscalingFirstErr, + tc.describeAutoscalingSecondErr, + }, + setDesiredCapacityErr: tc.setDesiredCapacityErr, + }, + } + nodeName, providerID, err := client.CreateNode(context.Background(), tc.providerID) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantNodeName, nodeName) + assert.Equal(tc.wantProviderID, providerID) + }) + } +} + +func TestDeleteNode(t *testing.T) { + testCases := map[string]struct { + providerID string + terminateInstanceErr error + wantErr bool + }{ + "deleting node works": { + providerID: "aws:///us-east-2a/i-00000000000000000", + }, + "deleting node fails when terminating the instance fails": { + providerID: "aws:///us-east-2a/i-00000000000000000", + terminateInstanceErr: assert.AnError, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + scalingClient: &stubAutoscalingAPI{ + terminateInstanceErr: tc.terminateInstanceErr, + }, + } + err := client.DeleteNode(context.Background(), tc.providerID) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + }) + } +} + +type stubEC2API struct { + describeInstancesOut *ec2.DescribeInstancesOutput + describeInstancesErr error + describeInstanceStatusOut *ec2.DescribeInstanceStatusOutput + describeInstanceStatusErr error + describeLaunchTemplateVersionsOut *ec2.DescribeLaunchTemplateVersionsOutput + describeLaunchTemplateVersionsErr error + createLaunchTemplateVersionOut *ec2.CreateLaunchTemplateVersionOutput + createLaunchTemplateVersionErr error + modifyLaunchTemplateErr error +} + +func (a *stubEC2API) DescribeInstances(_ context.Context, _ *ec2.DescribeInstancesInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstancesOutput, error) { + return a.describeInstancesOut, a.describeInstancesErr +} + +func (a *stubEC2API) DescribeInstanceStatus(_ context.Context, _ *ec2.DescribeInstanceStatusInput, _ ...func(*ec2.Options)) (*ec2.DescribeInstanceStatusOutput, error) { + return a.describeInstanceStatusOut, a.describeInstanceStatusErr +} + +func (a *stubEC2API) CreateLaunchTemplateVersion(_ context.Context, _ *ec2.CreateLaunchTemplateVersionInput, _ ...func(*ec2.Options)) (*ec2.CreateLaunchTemplateVersionOutput, error) { + return a.createLaunchTemplateVersionOut, a.createLaunchTemplateVersionErr +} + +func (a *stubEC2API) ModifyLaunchTemplate(_ context.Context, _ *ec2.ModifyLaunchTemplateInput, _ ...func(*ec2.Options)) (*ec2.ModifyLaunchTemplateOutput, error) { + return nil, a.modifyLaunchTemplateErr +} + +func (a *stubEC2API) DescribeLaunchTemplateVersions(_ context.Context, _ *ec2.DescribeLaunchTemplateVersionsInput, _ ...func(*ec2.Options)) (*ec2.DescribeLaunchTemplateVersionsOutput, error) { + return a.describeLaunchTemplateVersionsOut, a.describeLaunchTemplateVersionsErr +} + +type stubAutoscalingAPI struct { + describeAutoScalingGroupsOut []*autoscaling.DescribeAutoScalingGroupsOutput + describeAutoScalingGroupsErr []error + describeCounter int + setDesiredCapacityErr error + terminateInstanceErr error +} + +func (a *stubAutoscalingAPI) DescribeAutoScalingGroups(_ context.Context, _ *autoscaling.DescribeAutoScalingGroupsInput, _ ...func(*autoscaling.Options)) (*autoscaling.DescribeAutoScalingGroupsOutput, error) { + out := a.describeAutoScalingGroupsOut[a.describeCounter] + err := a.describeAutoScalingGroupsErr[a.describeCounter] + a.describeCounter++ + return out, err +} + +func (a *stubAutoscalingAPI) SetDesiredCapacity(_ context.Context, _ *autoscaling.SetDesiredCapacityInput, _ ...func(*autoscaling.Options)) (*autoscaling.SetDesiredCapacityOutput, error) { + return nil, a.setDesiredCapacityErr +} + +func (a *stubAutoscalingAPI) TerminateInstanceInAutoScalingGroup(_ context.Context, _ *autoscaling.TerminateInstanceInAutoScalingGroupInput, _ ...func(*autoscaling.Options)) (*autoscaling.TerminateInstanceInAutoScalingGroupOutput, error) { + return nil, a.terminateInstanceErr +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/pendingnode.go b/operators/constellation-node-operator/internal/cloud/aws/client/pendingnode.go new file mode 100644 index 000000000..662a6d6b0 --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/pendingnode.go @@ -0,0 +1,68 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +// GetNodeState returns the state of the node. +func (c *Client) GetNodeState(ctx context.Context, providerID string) (updatev1alpha1.CSPNodeState, error) { + logr := log.FromContext(ctx) + logr.Info("GetNodeState", "providerID", providerID) + instanceName, err := getInstanceNameFromProviderID(providerID) + if err != nil { + return updatev1alpha1.NodeStateUnknown, fmt.Errorf("failed to get instance name from providerID: %w", err) + } + + statusOut, err := c.ec2Client.DescribeInstanceStatus(ctx, &ec2.DescribeInstanceStatusInput{ + InstanceIds: []string{instanceName}, + IncludeAllInstances: toPtr(true), + }) + if err != nil { + if strings.Contains(err.Error(), "InvalidInstanceID.NotFound") { + return updatev1alpha1.NodeStateTerminated, nil + } + return updatev1alpha1.NodeStateUnknown, err + } + + if len(statusOut.InstanceStatuses) != 1 { + return updatev1alpha1.NodeStateUnknown, fmt.Errorf("expected 1 instance status, got %d", len(statusOut.InstanceStatuses)) + } + + if statusOut.InstanceStatuses[0].InstanceState == nil { + return updatev1alpha1.NodeStateUnknown, errors.New("instance state is nil") + } + + // Translate AWS instance state to node state. + switch statusOut.InstanceStatuses[0].InstanceState.Name { + case ec2types.InstanceStateNameRunning: + return updatev1alpha1.NodeStateReady, nil + case ec2types.InstanceStateNameTerminated: + return updatev1alpha1.NodeStateTerminated, nil + case ec2types.InstanceStateNameShuttingDown: + return updatev1alpha1.NodeStateTerminating, nil + case ec2types.InstanceStateNameStopped: + return updatev1alpha1.NodeStateStopped, nil + // For "Stopping" we can only know the next state in the state machine + // so we preemptively set it to "Stopped". + case ec2types.InstanceStateNameStopping: + return updatev1alpha1.NodeStateStopped, nil + case ec2types.InstanceStateNamePending: + return updatev1alpha1.NodeStateCreating, nil + default: + return updatev1alpha1.NodeStateUnknown, fmt.Errorf("unknown instance state %q", statusOut.InstanceStatuses[0].InstanceState.Name) + } +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/pendingnode_test.go b/operators/constellation-node-operator/internal/cloud/aws/client/pendingnode_test.go new file mode 100644 index 000000000..2d5f3bddb --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/pendingnode_test.go @@ -0,0 +1,173 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "context" + "errors" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + updatev1alpha1 "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api/v1alpha1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetNodeState(t *testing.T) { + testCases := map[string]struct { + providerID string + describeInstanceStatusOut *ec2.DescribeInstanceStatusOutput + describeInstanceStatusErr error + wantState updatev1alpha1.CSPNodeState + wantErr bool + }{ + "getting node state works for running VM": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{ + { + InstanceState: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameRunning, + }, + }, + }, + }, + wantState: updatev1alpha1.NodeStateReady, + }, + "getting node state works for terminated VM": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{ + { + InstanceState: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameTerminated, + }, + }, + }, + }, + wantState: updatev1alpha1.NodeStateTerminated, + }, + "getting node state works for stopping VM": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{ + { + InstanceState: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameStopping, + }, + }, + }, + }, + wantState: updatev1alpha1.NodeStateStopped, + }, + "getting node state works for stopped VM": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{ + { + InstanceState: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameStopped, + }, + }, + }, + }, + wantState: updatev1alpha1.NodeStateStopped, + }, + "getting node state works for pending VM": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{ + { + InstanceState: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNamePending, + }, + }, + }, + }, + wantState: updatev1alpha1.NodeStateCreating, + }, + "getting node state works for shutting-down VM": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{ + { + InstanceState: &ec2types.InstanceState{ + Name: ec2types.InstanceStateNameShuttingDown, + }, + }, + }, + }, + wantState: updatev1alpha1.NodeStateTerminating, + }, + "getting node state fails when the state is unknown": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{ + { + InstanceState: &ec2types.InstanceState{ + Name: "unknown", + }, + }, + }, + }, + wantState: updatev1alpha1.NodeStateUnknown, + wantErr: true, + }, + "cannot find instance": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusErr: errors.New("InvalidInstanceID.NotFound"), + wantState: updatev1alpha1.NodeStateTerminated, + }, + "unknown error when describing the instance error": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusErr: assert.AnError, + wantState: updatev1alpha1.NodeStateUnknown, + wantErr: true, + }, + "fails when getting no instances": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{}, + }, + wantState: updatev1alpha1.NodeStateUnknown, + wantErr: true, + }, + "fails when the instance state is nil": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeInstanceStatusOut: &ec2.DescribeInstanceStatusOutput{ + InstanceStatuses: []ec2types.InstanceStatus{ + { + InstanceState: nil, + }, + }, + }, + wantState: updatev1alpha1.NodeStateUnknown, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + ec2Client: &stubEC2API{ + describeInstanceStatusOut: tc.describeInstanceStatusOut, + describeInstanceStatusErr: tc.describeInstanceStatusErr, + }, + } + nodeState, err := client.GetNodeState(context.Background(), tc.providerID) + assert.Equal(tc.wantState, nodeState) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + }) + } +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/scalinggroup.go b/operators/constellation-node-operator/internal/cloud/aws/client/scalinggroup.go new file mode 100644 index 000000000..bea53a5a2 --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/scalinggroup.go @@ -0,0 +1,174 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "context" + "fmt" + "strings" + + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + scalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" +) + +// GetScalingGroupImage returns the image URI of the scaling group. +func (c *Client) GetScalingGroupImage(ctx context.Context, scalingGroupID string) (string, error) { + launchTemplate, err := c.getScalingGroupTemplate(ctx, scalingGroupID) + if err != nil { + return "", err + } + + if launchTemplate.LaunchTemplateData == nil { + return "", fmt.Errorf("launch template data is nil for scaling group %q", scalingGroupID) + } + + if launchTemplate.LaunchTemplateData.ImageId == nil { + return "", fmt.Errorf("image ID is nil for scaling group %q", scalingGroupID) + } + + return *launchTemplate.LaunchTemplateData.ImageId, nil +} + +// SetScalingGroupImage sets the image URI of the scaling group. +func (c *Client) SetScalingGroupImage(ctx context.Context, scalingGroupID, imageURI string) error { + launchTemplate, err := c.getScalingGroupTemplate(ctx, scalingGroupID) + if err != nil { + return fmt.Errorf("failed to get launch template for scaling group %q: %w", scalingGroupID, err) + } + + if launchTemplate.VersionNumber == nil { + return fmt.Errorf("version number is nil for scaling group %q", scalingGroupID) + } + + createLaunchTemplateOut, err := c.ec2Client.CreateLaunchTemplateVersion( + ctx, + &ec2.CreateLaunchTemplateVersionInput{ + LaunchTemplateData: &ec2types.RequestLaunchTemplateData{ + ImageId: &imageURI, + }, + LaunchTemplateId: launchTemplate.LaunchTemplateId, + SourceVersion: toPtr(fmt.Sprintf("%d", *launchTemplate.VersionNumber)), + }, + ) + if err != nil { + return fmt.Errorf("failed to create launch template version: %w", err) + } + + if createLaunchTemplateOut == nil { + return fmt.Errorf("create launch template version output is nil") + } + if createLaunchTemplateOut.LaunchTemplateVersion == nil { + return fmt.Errorf("created launch template version is nil") + } + if createLaunchTemplateOut.LaunchTemplateVersion.VersionNumber == nil { + return fmt.Errorf("created launch template version number is nil") + } + + // set created version as default + _, err = c.ec2Client.ModifyLaunchTemplate( + ctx, + &ec2.ModifyLaunchTemplateInput{ + LaunchTemplateId: launchTemplate.LaunchTemplateId, + DefaultVersion: toPtr(fmt.Sprintf("%d", createLaunchTemplateOut.LaunchTemplateVersion.VersionNumber)), + }, + ) + if err != nil { + return fmt.Errorf("failed to modify launch template: %w", err) + } + + return nil +} + +func (c *Client) getScalingGroupTemplate(ctx context.Context, scalingGroupID string) (ec2types.LaunchTemplateVersion, error) { + groupOutput, err := c.scalingClient.DescribeAutoScalingGroups( + ctx, + &autoscaling.DescribeAutoScalingGroupsInput{ + AutoScalingGroupNames: []string{scalingGroupID}, + }, + ) + if err != nil { + return ec2types.LaunchTemplateVersion{}, fmt.Errorf("failed to describe scaling group %q: %w", scalingGroupID, err) + } + + if len(groupOutput.AutoScalingGroups) != 1 { + return ec2types.LaunchTemplateVersion{}, fmt.Errorf("expected exactly one scaling group, got %d", len(groupOutput.AutoScalingGroups)) + } + + if groupOutput.AutoScalingGroups[0].LaunchTemplate == nil { + return ec2types.LaunchTemplateVersion{}, fmt.Errorf("launch template is nil for scaling group %q", scalingGroupID) + } + + if groupOutput.AutoScalingGroups[0].LaunchTemplate.LaunchTemplateId == nil { + return ec2types.LaunchTemplateVersion{}, fmt.Errorf("launch template ID is nil for scaling group %q", scalingGroupID) + } + + launchTemplateID := groupOutput.AutoScalingGroups[0].LaunchTemplate.LaunchTemplateId + + launchTemplateOutput, err := c.ec2Client.DescribeLaunchTemplateVersions( + ctx, + &ec2.DescribeLaunchTemplateVersionsInput{ + LaunchTemplateId: launchTemplateID, + Versions: []string{"$Latest"}, + }, + ) + if err != nil { + return ec2types.LaunchTemplateVersion{}, fmt.Errorf("failed to describe launch template %q: %w", *launchTemplateID, err) + } + + if len(launchTemplateOutput.LaunchTemplateVersions) != 1 { + return ec2types.LaunchTemplateVersion{}, fmt.Errorf("expected exactly one launch template, got %d", len(launchTemplateOutput.LaunchTemplateVersions)) + } + return launchTemplateOutput.LaunchTemplateVersions[0], nil +} + +// GetScalingGroupName retrieves the name of a scaling group. +// This keeps the casing of the original name, but Kubernetes requires the name to be lowercase, +// so use strings.ToLower() on the result if using the name in a Kubernetes context. +func (c *Client) GetScalingGroupName(scalingGroupID string) (string, error) { + return strings.ToLower(scalingGroupID), nil +} + +// GetAutoscalingGroupName retrieves the name of a scaling group as needed by the cluster-autoscaler. +func (c *Client) GetAutoscalingGroupName(scalingGroupID string) (string, error) { + return scalingGroupID, nil +} + +// ListScalingGroups retrieves a list of scaling groups for the cluster. +func (c *Client) ListScalingGroups(ctx context.Context, uid string) (controlPlaneGroupIDs []string, workerGroupIDs []string, err error) { + output, err := c.scalingClient.DescribeAutoScalingGroups( + ctx, + &autoscaling.DescribeAutoScalingGroupsInput{ + Filters: []scalingtypes.Filter{ + { + Name: toPtr("tag:constellation-uid"), + Values: []string{uid}, + }, + }, + }, + ) + if err != nil { + return nil, nil, fmt.Errorf("failed to describe scaling groups: %w", err) + } + + for _, group := range output.AutoScalingGroups { + if group.Tags == nil { + continue + } + for _, tag := range group.Tags { + if *tag.Key == "constellation-role" { + if *tag.Value == "control-plane" { + controlPlaneGroupIDs = append(controlPlaneGroupIDs, *group.AutoScalingGroupName) + } else if *tag.Value == "worker" { + workerGroupIDs = append(workerGroupIDs, *group.AutoScalingGroupName) + } + } + } + } + return controlPlaneGroupIDs, workerGroupIDs, nil +} diff --git a/operators/constellation-node-operator/internal/cloud/aws/client/scalinggroup_test.go b/operators/constellation-node-operator/internal/cloud/aws/client/scalinggroup_test.go new file mode 100644 index 000000000..19025a68a --- /dev/null +++ b/operators/constellation-node-operator/internal/cloud/aws/client/scalinggroup_test.go @@ -0,0 +1,306 @@ +/* +Copyright (c) Edgeless Systems GmbH + +SPDX-License-Identifier: AGPL-3.0-only +*/ + +package client + +import ( + "context" + "testing" + + "github.com/aws/aws-sdk-go-v2/service/autoscaling" + scalingtypes "github.com/aws/aws-sdk-go-v2/service/autoscaling/types" + "github.com/aws/aws-sdk-go-v2/service/ec2" + ec2types "github.com/aws/aws-sdk-go-v2/service/ec2/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetScalingGroupImage(t *testing.T) { + testCases := map[string]struct { + providerID string + describeAutoScalingGroupsOut *autoscaling.DescribeAutoScalingGroupsOutput + describeAutoScalingGroupsErr error + describeLaunchTemplateVersionsOut *ec2.DescribeLaunchTemplateVersionsOutput + describeLaunchTemplateVersionsErr error + wantImage string + wantErr bool + }{ + "getting scaling group image works": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoScalingGroupsOut: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []scalingtypes.AutoScalingGroup{ + { + LaunchTemplate: &scalingtypes.LaunchTemplateSpecification{ + LaunchTemplateId: toPtr("lt-00000000000000000"), + }, + }, + }, + }, + describeLaunchTemplateVersionsOut: &ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + ImageId: toPtr("ami-00000000000000000"), + }, + }, + }, + }, + wantImage: "ami-00000000000000000", + }, + "fails when describing autoscaling group fails": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoScalingGroupsErr: assert.AnError, + wantErr: true, + }, + "fails when describing launch template versions fails": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoScalingGroupsOut: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []scalingtypes.AutoScalingGroup{ + { + LaunchTemplate: &scalingtypes.LaunchTemplateSpecification{ + LaunchTemplateId: toPtr("lt-00000000000000000"), + }, + }, + }, + }, + describeLaunchTemplateVersionsErr: assert.AnError, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + ec2Client: &stubEC2API{ + describeLaunchTemplateVersionsOut: tc.describeLaunchTemplateVersionsOut, + describeLaunchTemplateVersionsErr: tc.describeLaunchTemplateVersionsErr, + }, + scalingClient: &stubAutoscalingAPI{ + describeAutoScalingGroupsOut: []*autoscaling.DescribeAutoScalingGroupsOutput{ + tc.describeAutoScalingGroupsOut, + }, + describeAutoScalingGroupsErr: []error{ + tc.describeAutoScalingGroupsErr, + }, + }, + } + scalingGroupImage, err := client.GetScalingGroupImage(context.Background(), tc.providerID) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantImage, scalingGroupImage) + }) + } +} + +func TestSetScalingGroupImage(t *testing.T) { + testCases := map[string]struct { + providerID string + describeAutoScalingGroupsOut *autoscaling.DescribeAutoScalingGroupsOutput + describeAutoScalingGroupsErr error + describeLaunchTemplateVersionsOut *ec2.DescribeLaunchTemplateVersionsOutput + describeLaunchTemplateVersionsErr error + createLaunchTemplateVersionOut *ec2.CreateLaunchTemplateVersionOutput + createLaunchTemplateVersionErr error + modifyLaunchTemplateErr error + imageURI string + wantErr bool + }{ + "getting scaling group image works": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoScalingGroupsOut: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []scalingtypes.AutoScalingGroup{ + { + LaunchTemplate: &scalingtypes.LaunchTemplateSpecification{ + LaunchTemplateId: toPtr("lt-00000000000000000"), + }, + }, + }, + }, + describeLaunchTemplateVersionsOut: &ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + ImageId: toPtr("ami-00000000000000000"), + }, + VersionNumber: toPtr(int64(1)), + }, + }, + }, + createLaunchTemplateVersionOut: &ec2.CreateLaunchTemplateVersionOutput{ + LaunchTemplateVersion: &ec2types.LaunchTemplateVersion{ + VersionNumber: toPtr(int64(2)), + }, + }, + imageURI: "ami-00000000000000000", + }, + "fails when creating launch template version fails": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoScalingGroupsOut: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []scalingtypes.AutoScalingGroup{ + { + LaunchTemplate: &scalingtypes.LaunchTemplateSpecification{ + LaunchTemplateId: toPtr("lt-00000000000000000"), + }, + }, + }, + }, + describeLaunchTemplateVersionsOut: &ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + ImageId: toPtr("ami-00000000000000000"), + }, + VersionNumber: toPtr(int64(1)), + }, + }, + }, + imageURI: "ami-00000000000000000", + createLaunchTemplateVersionErr: assert.AnError, + wantErr: true, + }, + "fails when modifying launch template fails": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoScalingGroupsOut: &autoscaling.DescribeAutoScalingGroupsOutput{ + AutoScalingGroups: []scalingtypes.AutoScalingGroup{ + { + LaunchTemplate: &scalingtypes.LaunchTemplateSpecification{ + LaunchTemplateId: toPtr("lt-00000000000000000"), + }, + }, + }, + }, + describeLaunchTemplateVersionsOut: &ec2.DescribeLaunchTemplateVersionsOutput{ + LaunchTemplateVersions: []ec2types.LaunchTemplateVersion{ + { + LaunchTemplateData: &ec2types.ResponseLaunchTemplateData{ + ImageId: toPtr("ami-00000000000000000"), + }, + VersionNumber: toPtr(int64(1)), + }, + }, + }, + imageURI: "ami-00000000000000000", + modifyLaunchTemplateErr: assert.AnError, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + ec2Client: &stubEC2API{ + describeLaunchTemplateVersionsOut: tc.describeLaunchTemplateVersionsOut, + describeLaunchTemplateVersionsErr: tc.describeLaunchTemplateVersionsErr, + createLaunchTemplateVersionOut: tc.createLaunchTemplateVersionOut, + createLaunchTemplateVersionErr: tc.createLaunchTemplateVersionErr, + modifyLaunchTemplateErr: tc.modifyLaunchTemplateErr, + }, + scalingClient: &stubAutoscalingAPI{ + describeAutoScalingGroupsOut: []*autoscaling.DescribeAutoScalingGroupsOutput{ + tc.describeAutoScalingGroupsOut, + }, + describeAutoScalingGroupsErr: []error{ + tc.describeAutoScalingGroupsErr, + }, + }, + } + err := client.SetScalingGroupImage(context.Background(), tc.providerID, tc.imageURI) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + }) + } +} + +func TestListScalingGroups(t *testing.T) { + testCases := map[string]struct { + providerID string + describeAutoScalingGroupsOut []*autoscaling.DescribeAutoScalingGroupsOutput + describeAutoScalingGroupsErr []error + wantControlPlaneGroupIDs []string + wantWorkerGroupIDs []string + wantErr bool + }{ + "listing scaling groups work": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoScalingGroupsOut: []*autoscaling.DescribeAutoScalingGroupsOutput{ + { + AutoScalingGroups: []scalingtypes.AutoScalingGroup{ + { + AutoScalingGroupName: toPtr("control-plane-asg"), + Tags: []scalingtypes.TagDescription{ + { + Key: toPtr("constellation-role"), + Value: toPtr("control-plane"), + }, + }, + }, + { + AutoScalingGroupName: toPtr("worker-asg"), + Tags: []scalingtypes.TagDescription{ + { + Key: toPtr("constellation-role"), + Value: toPtr("worker"), + }, + }, + }, + { + AutoScalingGroupName: toPtr("worker-asg-2"), + Tags: []scalingtypes.TagDescription{ + { + Key: toPtr("constellation-role"), + Value: toPtr("worker"), + }, + }, + }, + { + AutoScalingGroupName: toPtr("other-asg"), + }, + }, + }, + }, + describeAutoScalingGroupsErr: []error{nil}, + wantControlPlaneGroupIDs: []string{"control-plane-asg"}, + wantWorkerGroupIDs: []string{"worker-asg", "worker-asg-2"}, + }, + "fails when describing scaling groups fails": { + providerID: "aws:///us-east-2a/i-00000000000000000", + describeAutoScalingGroupsOut: []*autoscaling.DescribeAutoScalingGroupsOutput{nil}, + describeAutoScalingGroupsErr: []error{assert.AnError}, + wantErr: true, + }, + } + for name, tc := range testCases { + t.Run(name, func(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + client := Client{ + scalingClient: &stubAutoscalingAPI{ + describeAutoScalingGroupsOut: tc.describeAutoScalingGroupsOut, + describeAutoScalingGroupsErr: tc.describeAutoScalingGroupsErr, + }, + } + controlPlaneGroupIDs, workerGroupIDs, err := client.ListScalingGroups(context.Background(), tc.providerID) + if tc.wantErr { + assert.Error(err) + return + } + require.NoError(err) + assert.Equal(tc.wantControlPlaneGroupIDs, controlPlaneGroupIDs) + assert.Equal(tc.wantWorkerGroupIDs, workerGroupIDs) + }) + } +} diff --git a/operators/constellation-node-operator/main.go b/operators/constellation-node-operator/main.go index b8d15a4a7..125d6c52f 100644 --- a/operators/constellation-node-operator/main.go +++ b/operators/constellation-node-operator/main.go @@ -25,6 +25,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" + awsclient "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/aws/client" azureclient "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/azure/client" cloudfake "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/fake/client" gcpclient "github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/internal/cloud/gcp/client" @@ -101,6 +102,12 @@ func main() { setupLog.Error(clientErr, "unable to create GCP client") os.Exit(1) } + case "aws": + cspClient, clientErr = awsclient.New(context.Background()) + if clientErr != nil { + setupLog.Error(clientErr, "unable to create AWS client") + os.Exit(1) + } default: setupLog.Info("CSP does not support upgrades", "csp", csp) cspClient = &cloudfake.Client{} @@ -142,7 +149,7 @@ func main() { os.Exit(1) } // Create Controllers - if csp == "azure" || csp == "gcp" { + if csp == "azure" || csp == "gcp" || csp == "aws" { if err = controllers.NewNodeVersionReconciler( cspClient, etcdClient, upgrade.NewClient(), discoveryClient, mgr.GetClient(), mgr.GetScheme(), ).SetupWithManager(mgr); err != nil {