From f164af29cfffd7a6f3c84e11ac8dd4b7dd70d419 Mon Sep 17 00:00:00 2001 From: Otto Bittner Date: Thu, 3 Nov 2022 16:42:19 +0100 Subject: [PATCH] AB#2583: deploy autoscaler via helm (#438) --- bootstrapper/cmd/bootstrapper/main.go | 8 +- .../k8sapi/kubectl/client/client_test.go | 6 - .../k8sapi/resources/cluster_autoscaler.go | 496 ------------------ .../resources/cluster_autoscaler_test.go | 47 -- bootstrapper/internal/kubernetes/k8sutil.go | 1 - .../internal/kubernetes/kubernetes.go | 45 +- .../internal/kubernetes/kubernetes_test.go | 4 +- .../constellation-services/Chart.yaml | 6 + .../charts/autoscaler/.helmignore | 23 + .../charts/autoscaler/Chart.yaml | 5 + .../autoscaler/templates/aws-deployment.yaml | 50 ++ .../templates/azure-deployment.yaml | 81 +++ .../autoscaler/templates/azure-secret.yaml | 15 + .../autoscaler/templates/clusterrole.yaml | 138 +++++ .../templates/clusterrolebinding.yaml | 17 + .../autoscaler/templates/gcp-deployment.yaml | 61 +++ .../templates/poddisruptionbudget.yaml | 15 + .../charts/autoscaler/templates/role.yaml | 26 + .../autoscaler/templates/rolebinding.yaml | 17 + .../charts/autoscaler/templates/service.yaml | 19 + .../autoscaler/templates/serviceaccount.yaml | 10 + .../charts/autoscaler/values.schema.json | 59 +++ .../charts/autoscaler/values.yaml | 0 .../charts/ccm/values.schema.json | 17 +- cli/internal/helm/loader.go | 6 + cli/internal/helm/loader_test.go | 13 +- .../templates/azure-deployment.yaml | 79 +++ .../autoscaler/templates/azure-secret.yaml | 13 + .../autoscaler/templates/clusterrole.yaml | 138 +++++ .../templates/clusterrolebinding.yaml | 17 + .../templates/poddisruptionbudget.yaml | 15 + .../charts/autoscaler/templates/role.yaml | 26 + .../autoscaler/templates/rolebinding.yaml | 17 + .../charts/autoscaler/templates/service.yaml | 19 + .../autoscaler/templates/serviceaccount.yaml | 10 + .../charts/ccm/templates/azure-daemonset.yaml | 51 +- .../charts/ccm/templates/azure-secret.yaml | 2 - .../autoscaler/templates/clusterrole.yaml | 138 +++++ .../templates/clusterrolebinding.yaml | 17 + .../autoscaler/templates/gcp-deployment.yaml | 59 +++ .../templates/poddisruptionbudget.yaml | 15 + .../charts/autoscaler/templates/role.yaml | 26 + .../autoscaler/templates/rolebinding.yaml | 17 + .../charts/autoscaler/templates/service.yaml | 19 + .../autoscaler/templates/serviceaccount.yaml | 10 + .../charts/ccm/templates/gcp-cm.yaml | 2 - .../charts/ccm/templates/gcp-daemonset.yaml | 7 +- .../charts/ccm/templates/gcp-secret.yaml | 2 - internal/cloud/aws/autoscaler.go | 47 -- internal/cloud/azure/autoscaler.go | 129 ----- internal/cloud/azure/autoscaler_test.go | 87 --- internal/cloud/gcp/autoscaler.go | 65 --- internal/cloud/gcp/autoscaler_test.go | 25 - internal/cloud/qemu/autoscaler.go | 45 -- 54 files changed, 1258 insertions(+), 1024 deletions(-) delete mode 100644 bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go delete mode 100644 bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler_test.go create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/.helmignore create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/Chart.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/aws-deployment.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/azure-deployment.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/azure-secret.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrole.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/role.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/rolebinding.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/service.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/serviceaccount.yaml create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/values.schema.json create mode 100644 cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/values.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/azure-deployment.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/azure-secret.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/clusterrole.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/role.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/rolebinding.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/service.yaml create mode 100644 cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/serviceaccount.yaml create mode 100644 cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/clusterrole.yaml create mode 100644 cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml create mode 100644 cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml create mode 100644 cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml create mode 100644 cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/role.yaml create mode 100644 cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/rolebinding.yaml create mode 100644 cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/service.yaml create mode 100644 cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/serviceaccount.yaml delete mode 100644 internal/cloud/aws/autoscaler.go delete mode 100644 internal/cloud/azure/autoscaler.go delete mode 100644 internal/cloud/azure/autoscaler_test.go delete mode 100644 internal/cloud/gcp/autoscaler.go delete mode 100644 internal/cloud/gcp/autoscaler_test.go delete mode 100644 internal/cloud/qemu/autoscaler.go diff --git a/bootstrapper/cmd/bootstrapper/main.go b/bootstrapper/cmd/bootstrapper/main.go index 6f49198ea..65e284830 100644 --- a/bootstrapper/cmd/bootstrapper/main.go +++ b/bootstrapper/cmd/bootstrapper/main.go @@ -107,7 +107,7 @@ func main() { cloudControllerManager := &awscloud.CloudControllerManager{} clusterInitJoiner = kubernetes.New( "aws", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), cloudControllerManager, - &gcpcloud.Autoscaler{}, metadata, pcrsJSON, helmClient, + metadata, pcrsJSON, helmClient, ) openTPM = vtpm.OpenVTPM fs = afero.NewOsFs() @@ -144,7 +144,7 @@ func main() { } clusterInitJoiner = kubernetes.New( "gcp", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), cloudControllerManager, - &gcpcloud.Autoscaler{}, metadata, pcrsJSON, helmClient, + metadata, pcrsJSON, helmClient, ) openTPM = vtpm.OpenVTPM fs = afero.NewOsFs() @@ -178,7 +178,7 @@ func main() { } clusterInitJoiner = kubernetes.New( "azure", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), azurecloud.NewCloudControllerManager(metadata), - &azurecloud.Autoscaler{}, metadata, pcrsJSON, helmClient, + metadata, pcrsJSON, helmClient, ) openTPM = vtpm.OpenVTPM @@ -200,7 +200,7 @@ func main() { } clusterInitJoiner = kubernetes.New( "qemu", k8sapi.NewKubernetesUtil(), &k8sapi.KubdeadmConfiguration{}, kubectl.New(), &qemucloud.CloudControllerManager{}, - &qemucloud.Autoscaler{}, metadata, pcrsJSON, helmClient, + metadata, pcrsJSON, helmClient, ) metadataAPI = metadata diff --git a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go index a52809cd4..b15276486 100644 --- a/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go +++ b/bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go @@ -14,7 +14,6 @@ import ( "net/http" "testing" - "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" "github.com/edgelesssys/constellation/v2/internal/kubernetes" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -275,11 +274,6 @@ func TestGetObjects(t *testing.T) { resourcesYAML string wantErr bool }{ - "GetObjects works on cluster-autoscaler deployment": { - wantResources: resources.NewDefaultAutoscalerDeployment(nil, nil, nil, ""), - resourcesYAML: string(nginxDeplYAML), - wantErr: false, - }, "GetObjects Marshal failure detected": { wantResources: &unmarshableResource{}, resourcesYAML: string(nginxDeplYAML), diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go b/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go deleted file mode 100644 index 2de635923..000000000 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler.go +++ /dev/null @@ -1,496 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package resources - -import ( - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/edgelesssys/constellation/v2/internal/versions" - "google.golang.org/protobuf/proto" - apps "k8s.io/api/apps/v1" - k8s "k8s.io/api/core/v1" - policy "k8s.io/api/policy/v1" - rbac "k8s.io/api/rbac/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" -) - -type AutoscalerDeployment struct { - PodDisruptionBudget policy.PodDisruptionBudget - ServiceAccount k8s.ServiceAccount - ClusterRole rbac.ClusterRole - ClusterRoleBinding rbac.ClusterRoleBinding - Role rbac.Role - RoleBinding rbac.RoleBinding - Service k8s.Service - Deployment apps.Deployment -} - -// NewDefaultAutoscalerDeployment creates a new *autoscalerDeployment, customized for the CSP. -func NewDefaultAutoscalerDeployment(extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar, k8sVersion versions.ValidK8sVersion) *AutoscalerDeployment { - return &AutoscalerDeployment{ - PodDisruptionBudget: policy.PodDisruptionBudget{ - TypeMeta: v1.TypeMeta{ - APIVersion: "policy/v1", - Kind: "PodDisruptionBudget", - }, - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - "app.kubernetes.io/managed-by": "Constellation", - }, - Name: "constellation-cluster-autoscaler", - Namespace: "default", - }, - Spec: policy.PodDisruptionBudgetSpec{ - Selector: &v1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - }, - }, - MaxUnavailable: &intstr.IntOrString{ - Type: intstr.Int, - IntVal: 1, - }, - }, - }, - ServiceAccount: k8s.ServiceAccount{ - TypeMeta: v1.TypeMeta{ - APIVersion: "v1", - Kind: "ServiceAccount", - }, - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - "app.kubernetes.io/managed-by": "Constellation", - }, - Name: "constellation-cluster-autoscaler", - Namespace: "kube-system", - }, - AutomountServiceAccountToken: proto.Bool(true), - }, - ClusterRole: rbac.ClusterRole{ - TypeMeta: v1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRole", - }, - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - "app.kubernetes.io/managed-by": "Constellation", - }, - Name: "constellation-cluster-autoscaler", - }, - Rules: []rbac.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{ - "events", - "endpoints", - }, - Verbs: []string{ - "create", - "patch", - }, - }, - { - APIGroups: []string{""}, - Resources: []string{ - "pods/eviction", - }, - Verbs: []string{ - "create", - }, - }, - { - APIGroups: []string{""}, - Resources: []string{ - "pods/status", - }, - Verbs: []string{ - "update", - }, - }, - { - APIGroups: []string{""}, - Resources: []string{ - "endpoints", - }, - ResourceNames: []string{ - "cluster-autoscaler", - }, - Verbs: []string{ - "get", - "update", - }, - }, - { - APIGroups: []string{""}, - Resources: []string{ - "nodes", - }, - Verbs: []string{ - "watch", - "list", - "get", - "update", - }, - }, - { - APIGroups: []string{""}, - Resources: []string{ - "namespaces", - "pods", - "services", - "replicationcontrollers", - "persistentvolumeclaims", - "persistentvolumes", - }, - Verbs: []string{ - "watch", - "list", - "get", - }, - }, - { - APIGroups: []string{ - "batch", - }, - Resources: []string{ - "jobs", - "cronjobs", - }, - Verbs: []string{ - "watch", - "list", - "get", - }, - }, - { - APIGroups: []string{ - "batch", - "extensions", - }, - Resources: []string{ - "jobs", - }, - Verbs: []string{ - "get", - "list", - "patch", - "watch", - }, - }, - { - APIGroups: []string{ - "extensions", - }, - Resources: []string{ - "replicasets", - "daemonsets", - }, - Verbs: []string{ - "watch", - "list", - "get", - }, - }, - { - APIGroups: []string{ - "policy", - }, - Resources: []string{ - "poddisruptionbudgets", - }, - Verbs: []string{ - "watch", - "list", - }, - }, - { - APIGroups: []string{ - "apps", - }, - Resources: []string{ - "daemonsets", - "replicasets", - "statefulsets", - }, - Verbs: []string{ - "watch", - "list", - "get", - }, - }, - { - APIGroups: []string{ - "storage.k8s.io", - }, - Resources: []string{ - "storageclasses", - "csinodes", - "csidrivers", - "csistoragecapacities", - }, - Verbs: []string{ - "watch", - "list", - "get", - }, - }, - { - APIGroups: []string{""}, - Resources: []string{ - "configmaps", - }, - Verbs: []string{ - "list", - "watch", - }, - }, - { - APIGroups: []string{ - "coordination.k8s.io", - }, - Resources: []string{ - "leases", - }, - Verbs: []string{ - "create", - }, - }, - { - APIGroups: []string{ - "coordination.k8s.io", - }, - ResourceNames: []string{ - "cluster-autoscaler", - }, - Resources: []string{ - "leases", - }, - Verbs: []string{ - "get", - "update", - }, - }, - }, - }, - ClusterRoleBinding: rbac.ClusterRoleBinding{ - TypeMeta: v1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "ClusterRoleBinding", - }, - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - "app.kubernetes.io/managed-by": "Constellation", - }, - Name: "constellation-cluster-autoscaler", - Namespace: "kube-system", - }, - RoleRef: rbac.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "ClusterRole", - Name: "constellation-cluster-autoscaler", - }, - Subjects: []rbac.Subject{ - { - Kind: "ServiceAccount", - Name: "constellation-cluster-autoscaler", - Namespace: "kube-system", - }, - }, - }, - Role: rbac.Role{ - TypeMeta: v1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "Role", - }, - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - "app.kubernetes.io/managed-by": "Constellation", - }, - Name: "constellation-cluster-autoscaler", - Namespace: "kube-system", - }, - Rules: []rbac.PolicyRule{ - { - APIGroups: []string{""}, - Resources: []string{ - "configmaps", - }, - Verbs: []string{ - "create", - }, - }, - { - APIGroups: []string{""}, - Resources: []string{ - "configmaps", - }, - ResourceNames: []string{ - "cluster-autoscaler-status", - }, - Verbs: []string{ - "delete", - "get", - "update", - }, - }, - }, - }, - RoleBinding: rbac.RoleBinding{ - TypeMeta: v1.TypeMeta{ - APIVersion: "rbac.authorization.k8s.io/v1", - Kind: "RoleBinding", - }, - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - "app.kubernetes.io/managed-by": "Constellation", - }, - Name: "constellation-cluster-autoscaler", - Namespace: "kube-system", - }, - RoleRef: rbac.RoleRef{ - APIGroup: "rbac.authorization.k8s.io", - Kind: "Role", - Name: "constellation-cluster-autoscaler", - }, - Subjects: []rbac.Subject{ - { - Kind: "ServiceAccount", - Name: "constellation-cluster-autoscaler", - Namespace: "kube-system", - }, - }, - }, - Service: k8s.Service{ - TypeMeta: v1.TypeMeta{ - APIVersion: "v1", - Kind: "Service", - }, - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - "app.kubernetes.io/managed-by": "Constellation", - }, - Name: "constellation-cluster-autoscaler", - Namespace: "default", - }, - Spec: k8s.ServiceSpec{ - Ports: []k8s.ServicePort{ - { - Port: 8085, - Protocol: k8s.ProtocolTCP, - TargetPort: intstr.FromInt(8085), - Name: "http", - }, - }, - Selector: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - }, - Type: k8s.ServiceTypeClusterIP, - }, - }, - Deployment: apps.Deployment{ - TypeMeta: v1.TypeMeta{ - APIVersion: "apps/v1", - Kind: "Deployment", - }, - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - "app.kubernetes.io/managed-by": "Constellation", - }, - Name: "constellation-cluster-autoscaler", - Namespace: "kube-system", - }, - Spec: apps.DeploymentSpec{ - Replicas: proto.Int32(0), - Selector: &v1.LabelSelector{ - MatchLabels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - }, - }, - Template: k8s.PodTemplateSpec{ - ObjectMeta: v1.ObjectMeta{ - Labels: map[string]string{ - "app.kubernetes.io/instance": "constellation", - "app.kubernetes.io/name": "cluster-autoscaler", - }, - }, - Spec: k8s.PodSpec{ - PriorityClassName: "system-cluster-critical", - DNSPolicy: k8s.DNSClusterFirst, - Containers: []k8s.Container{ - { - Name: "cluster-autoscaler", - Image: versions.VersionConfigs[k8sVersion].ClusterAutoscalerImage, - ImagePullPolicy: k8s.PullIfNotPresent, - LivenessProbe: &k8s.Probe{ - ProbeHandler: k8s.ProbeHandler{ - HTTPGet: &k8s.HTTPGetAction{ - Path: "/health-check", - Port: intstr.FromInt(8085), - }, - }, - }, - Ports: []k8s.ContainerPort{ - { - ContainerPort: 8085, - }, - }, - VolumeMounts: extraVolumeMounts, - Env: env, - }, - }, - Volumes: extraVolumes, - ServiceAccountName: "constellation-cluster-autoscaler", - Tolerations: []k8s.Toleration{ - { - Key: "node-role.kubernetes.io/master", - Operator: k8s.TolerationOpExists, - Effect: k8s.TaintEffectNoSchedule, - }, - { - Key: "node-role.kubernetes.io/control-plane", - Operator: k8s.TolerationOpExists, - Effect: k8s.TaintEffectNoSchedule, - }, - { - Key: "node.cloudprovider.kubernetes.io/uninitialized", - Operator: k8s.TolerationOpEqual, - Value: "true", - Effect: k8s.TaintEffectNoSchedule, - }, - }, - NodeSelector: map[string]string{ - "node-role.kubernetes.io/control-plane": "", - }, - }, - }, - }, - }, - } -} - -func (a *AutoscalerDeployment) Marshal() ([]byte, error) { - return kubernetes.MarshalK8SResources(a) -} diff --git a/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler_test.go b/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler_test.go deleted file mode 100644 index 6bf2bef9b..000000000 --- a/bootstrapper/internal/kubernetes/k8sapi/resources/cluster_autoscaler_test.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package resources - -import ( - "testing" - - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -func TestAutoscalerDeploymentMarshalUnmarshal(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - - autoscalerDepl := NewDefaultAutoscalerDeployment(nil, nil, nil, "") - - data, err := autoscalerDepl.Marshal() - require.NoError(err) - - t.Log(string(data)) - - var recreated AutoscalerDeployment - require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated)) - assert.Equal(autoscalerDepl, &recreated) -} - -func TestAutoscalerDeploymentWithCommandMarshalUnmarshal(t *testing.T) { - require := require.New(t) - assert := assert.New(t) - - autoscalerDepl := NewDefaultAutoscalerDeployment(nil, nil, nil, "") - - data, err := autoscalerDepl.Marshal() - require.NoError(err) - - t.Log(string(data)) - - var recreated AutoscalerDeployment - require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated)) - assert.Equal(autoscalerDepl, &recreated) -} diff --git a/bootstrapper/internal/kubernetes/k8sutil.go b/bootstrapper/internal/kubernetes/k8sutil.go index 6fb54ac3e..f09974f40 100644 --- a/bootstrapper/internal/kubernetes/k8sutil.go +++ b/bootstrapper/internal/kubernetes/k8sutil.go @@ -23,7 +23,6 @@ type clusterUtil interface { InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error SetupAccessManager(kubectl k8sapi.Client, sshUsers kubernetes.Marshaler) error - SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error SetupVerificationService(kubectl k8sapi.Client, verificationServiceConfiguration kubernetes.Marshaler) error SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration kubernetes.Marshaler) error diff --git a/bootstrapper/internal/kubernetes/kubernetes.go b/bootstrapper/internal/kubernetes/kubernetes.go index 19397294e..e361b1e49 100644 --- a/bootstrapper/internal/kubernetes/kubernetes.go +++ b/bootstrapper/internal/kubernetes/kubernetes.go @@ -19,6 +19,7 @@ import ( "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi" "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources" + "github.com/edgelesssys/constellation/v2/internal/azureshared" "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider" "github.com/edgelesssys/constellation/v2/internal/cloud/metadata" "github.com/edgelesssys/constellation/v2/internal/constants" @@ -54,7 +55,6 @@ type KubeWrapper struct { client k8sapi.Client kubeconfigReader configReader cloudControllerManager CloudControllerManager - clusterAutoscaler ClusterAutoscaler providerMetadata ProviderMetadata initialMeasurementsJSON []byte getIPAddr func() (string, error) @@ -62,7 +62,7 @@ type KubeWrapper struct { // New creates a new KubeWrapper with real values. func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, cloudControllerManager CloudControllerManager, - clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmClient helmClient, + providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmClient helmClient, ) *KubeWrapper { return &KubeWrapper{ cloudProvider: cloudProvider, @@ -72,7 +72,6 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura client: client, kubeconfigReader: &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}}, cloudControllerManager: cloudControllerManager, - clusterAutoscaler: clusterAutoscaler, providerMetadata: providerMetadata, initialMeasurementsJSON: initialMeasurementsJSON, getIPAddr: getIPAddr, @@ -204,10 +203,6 @@ func (k *KubeWrapper) InitCluster( return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err) } - if err := k.setupClusterAutoscaler(instance, cloudServiceAccountURI, k8sVersion); err != nil { - return nil, fmt.Errorf("setting up cluster autoscaler: %w", err) - } - // TODO: remove access manager or re-enable with support for readonly /etc // accessManager := resources.NewAccessManagerDeployment(sshUsers) // if err := k.clusterUtil.SetupAccessManager(k.client, accessManager); err != nil { @@ -312,23 +307,6 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) { return k.kubeconfigReader.ReadKubeconfig() } -func (k *KubeWrapper) setupClusterAutoscaler(instance metadata.InstanceMetadata, cloudServiceAccountURI string, k8sVersion versions.ValidK8sVersion) error { - if !k.clusterAutoscaler.Supported() { - return nil - } - caSecrets, err := k.clusterAutoscaler.Secrets(instance.ProviderID, cloudServiceAccountURI) - if err != nil { - return fmt.Errorf("defining Secrets for cluster-autoscaler: %w", err) - } - - clusterAutoscalerConfiguration := resources.NewDefaultAutoscalerDeployment(k.clusterAutoscaler.Volumes(), k.clusterAutoscaler.VolumeMounts(), k.clusterAutoscaler.Env(), k8sVersion) - if err := k.clusterUtil.SetupAutoscaling(k.client, clusterAutoscalerConfiguration, caSecrets); err != nil { - return fmt.Errorf("setting up cluster-autoscaler: %w", err) - } - - return nil -} - // setupK8sVersionConfigMap applies a ConfigMap (cf. server-side apply) to consistently store the installed k8s version. func (k *KubeWrapper) setupK8sVersionConfigMap(ctx context.Context, k8sVersion versions.ValidK8sVersion) error { config := corev1.ConfigMap{ @@ -510,6 +488,25 @@ func (k *KubeWrapper) setupExtraVals(ctx context.Context, initialMeasurementsJSO return nil, errors.New("invalid join-service values") } joinVals["idkeydigest"] = hex.EncodeToString(idkeydigest) + + subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(instance.ProviderID) + if err != nil { + return nil, err + } + creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI) + if err != nil { + return nil, err + } + + extraVals["autoscaler"] = map[string]any{ + "Azure": map[string]any{ + "clientID": creds.AppClientID, + "clientSecret": creds.ClientSecretValue, + "resourceGroup": resourceGroup, + "subscriptionID": subscriptionID, + "tenantID": creds.TenantID, + }, + } } } return extraVals, nil diff --git a/bootstrapper/internal/kubernetes/kubernetes_test.go b/bootstrapper/internal/kubernetes/kubernetes_test.go index ac1ed2b06..b4ff2f16f 100644 --- a/bootstrapper/internal/kubernetes/kubernetes_test.go +++ b/bootstrapper/internal/kubernetes/kubernetes_test.go @@ -192,7 +192,8 @@ func TestInitCluster(t *testing.T) { k8sVersion: versions.Default, }, "kubeadm init fails when setting the cluster autoscaler": { - clusterUtil: stubClusterUtil{setupAutoscalingError: someErr}, + clusterUtil: stubClusterUtil{}, + helmClient: stubHelmClient{servicesError: someErr}, kubeconfigReader: &stubKubeconfigReader{ Kubeconfig: []byte("someKubeconfig"), }, @@ -258,7 +259,6 @@ func TestInitCluster(t *testing.T) { helmClient: &tc.helmClient, providerMetadata: tc.providerMetadata, cloudControllerManager: tc.CloudControllerManager, - clusterAutoscaler: tc.ClusterAutoscaler, configProvider: &stubConfigProvider{InitConfig: k8sapi.KubeadmInitYAML{}}, client: &tc.kubectl, kubeconfigReader: tc.kubeconfigReader, diff --git a/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml b/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml index 45f43bb09..1edbe979b 100644 --- a/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml +++ b/cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml @@ -29,3 +29,9 @@ dependencies: version: 2.2.0-pre tags: - Azure + - name: autoscaler + version: 2.2.0-pre + tags: + - Azure + - GCP + - AWS diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/.helmignore b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/.helmignore new file mode 100644 index 000000000..0e8a0eb36 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/Chart.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/Chart.yaml new file mode 100644 index 000000000..f618bcb6b --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/Chart.yaml @@ -0,0 +1,5 @@ +apiVersion: v2 +name: autoscaler +description: A Helm chart to deploy the cluster autoscaler. +type: application +version: 2.2.0-pre diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/aws-deployment.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/aws-deployment.yaml new file mode 100644 index 000000000..2625a736b --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/aws-deployment.yaml @@ -0,0 +1,50 @@ +{{- if eq .Values.csp "AWS" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + replicas: 0 + selector: + matchLabels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + template: + metadata: + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + spec: + containers: + - name: cluster-autoscaler + image: {{ .Values.image }} + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /health-check + port: 8085 + ports: + - containerPort: 8085 + resources: {} + dnsPolicy: ClusterFirst + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-cluster-critical + serviceAccountName: constellation-cluster-autoscaler + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Equal + value: "true" +{{- end -}} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/azure-deployment.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/azure-deployment.yaml new file mode 100644 index 000000000..7a287bed6 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/azure-deployment.yaml @@ -0,0 +1,81 @@ +{{- if eq .Values.csp "Azure" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + replicas: 0 + selector: + matchLabels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + template: + metadata: + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + spec: + containers: + - name: cluster-autoscaler + image: {{ .Values.image }} + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /health-check + port: 8085 + ports: + - containerPort: 8085 + env: + - name: ARM_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + key: SubscriptionID + name: cluster-autoscaler-azure + - name: ARM_RESOURCE_GROUP + valueFrom: + secretKeyRef: + key: ResourceGroup + name: cluster-autoscaler-azure + - name: ARM_TENANT_ID + valueFrom: + secretKeyRef: + key: TenantID + name: cluster-autoscaler-azure + - name: ARM_CLIENT_ID + valueFrom: + secretKeyRef: + key: ClientID + name: cluster-autoscaler-azure + - name: ARM_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: ClientSecret + name: cluster-autoscaler-azure + - name: ARM_VM_TYPE + valueFrom: + secretKeyRef: + key: VMType + name: cluster-autoscaler-azure + resources: {} + dnsPolicy: ClusterFirst + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-cluster-critical + serviceAccountName: constellation-cluster-autoscaler + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Equal + value: "true" +{{- end -}} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/azure-secret.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/azure-secret.yaml new file mode 100644 index 000000000..62fa109af --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/azure-secret.yaml @@ -0,0 +1,15 @@ +{{- if eq .Values.csp "Azure" -}} +apiVersion: v1 +kind: Secret +metadata: + name: cluster-autoscaler-azure + namespace: {{ .Release.Namespace }} +data: + ClientID: {{ .Values.Azure.clientID | b64enc }} + ClientSecret: {{ .Values.Azure.clientSecret | b64enc }} + ResourceGroup: {{ .Values.Azure.resourceGroup | b64enc }} + SubscriptionID: {{ .Values.Azure.subscriptionID | b64enc }} + TenantID: {{ .Values.Azure.tenantID | b64enc }} + # b64encode("vmss") + VMType: dm1zcw== +{{- end -}} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrole.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrole.yaml new file mode 100644 index 000000000..501c719d5 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrole.yaml @@ -0,0 +1,138 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: constellation-cluster-autoscaler + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +rules: +- apiGroups: + - "" + resources: + - events + - endpoints + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - "" + resources: + - pods/status + verbs: + - update +- apiGroups: + - "" + resourceNames: + - cluster-autoscaler + resources: + - endpoints + verbs: + - get + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - watch + - list + - get + - update +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - replicationcontrollers + - persistentvolumeclaims + - persistentvolumes + verbs: + - watch + - list + - get +- apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - watch + - list + - get +- apiGroups: + - batch + - extensions + resources: + - jobs + verbs: + - get + - list + - patch + - watch +- apiGroups: + - extensions + resources: + - replicasets + - daemonsets + verbs: + - watch + - list + - get +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - watch + - list +- apiGroups: + - apps + resources: + - daemonsets + - replicasets + - statefulsets + verbs: + - watch + - list + - get +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + - csidrivers + - csistoragecapacities + verbs: + - watch + - list + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resourceNames: + - cluster-autoscaler + resources: + - leases + verbs: + - get + - update diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..df0ebe615 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: constellation-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml new file mode 100644 index 000000000..8a97d1c20 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml @@ -0,0 +1,61 @@ +{{- if eq .Values.csp "GCP" -}} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + replicas: 0 + selector: + matchLabels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + template: + metadata: + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + spec: + containers: + - name: cluster-autoscaler + image: {{ .Values.image }} + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /health-check + port: 8085 + ports: + - containerPort: 8085 + resources: {} + volumeMounts: + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json + dnsPolicy: ClusterFirst + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-cluster-critical + serviceAccountName: constellation-cluster-autoscaler + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Equal + value: "true" + volumes: + - name: gcekey + secret: + secretName: gcekey +{{- end -}} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..9e3c11e1e --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml @@ -0,0 +1,15 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/role.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/role.yaml new file mode 100644 index 000000000..dec5805a3 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/role.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create +- apiGroups: + - "" + resourceNames: + - cluster-autoscaler-status + resources: + - configmaps + verbs: + - delete + - get + - update diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/rolebinding.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/rolebinding.yaml new file mode 100644 index 000000000..6d1c2a273 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: constellation-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: constellation-cluster-autoscaler + namespace: kube-system diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/service.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/service.yaml new file mode 100644 index 000000000..d58b79bfa --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + ports: + - name: http + port: 8085 + protocol: TCP + targetPort: 8085 + selector: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + type: ClusterIP diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/serviceaccount.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/serviceaccount.yaml new file mode 100644 index 000000000..1b6a0ffb3 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: constellation-cluster-autoscaler + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +automountServiceAccountToken: true diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/values.schema.json b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/values.schema.json new file mode 100644 index 000000000..d01913788 --- /dev/null +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/values.schema.json @@ -0,0 +1,59 @@ +{ + "$schema": "https://json-schema.org/draft-07/schema#", + "properties": { + "csp": { + "description": "CSP to which the chart is deployed.", + "enum": ["Azure", "GCP", "AWS", "QEMU"] + }, + "image": { + "description": "Container image to use for the spawned pods.", + "type": "string", + "examples": ["k8s.gcr.io/autoscaling/cluster-autoscaler:v1.23.1"] + + }, + "Azure": { + "description": "Config values required for deployment on Azure", + "type": "object", + "properties": { + "clientID": { + "description": "Client ID of the service account used to access the Azure API.", + "type": "string" + }, + "clientSecret": { + "description": "Secret of the service account used to access the Azure API.", + "type": "string" + }, + "resourceGroup": { + "description": "Resource group in which the cluster is running.", + "type": "string" + }, + "subscriptionID": { + "description": "Subscription ID of the Azure subscription.", + "type": "string" + }, + "tenantID": { + "description": "Tenant ID of the Azure subscription.", + "type": "string" + } + }, + "required": [ + "clientID", + "clientSecret", + "resourceGroup", + "subscriptionID", + "tenantID" + ] + } + }, + "required": [ + "csp", + "image" + ], + "if": { + "properties": { "csp": { "const": "Azure" } }, + "required": ["csp"] + }, + "then": { "required": ["Azure"] }, + "title": "Values", + "type": "object" +} diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/values.yaml b/cli/internal/helm/charts/edgeless/constellation-services/charts/autoscaler/values.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json index fefbaf73f..44a4f2579 100644 --- a/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json +++ b/cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json @@ -5,12 +5,6 @@ "description": "CSP to which the chart is deployed.", "enum": ["Azure", "GCP", "AWS", "QEMU"] }, - "subnetworkPodCIDR": { - "description": "CIDR Range for Pods in cluster", - "type": "string", - "examples": ["192.0.2.0/24"], - "pattern": "[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/[0-9]{1,2}" - }, "GCP": { "description": "Config values required for deployment on GCP", "type": "object", @@ -31,7 +25,13 @@ "secretData": { "description": "GCP service account key as a json-string", "type": "string" - } + }, + "subnetworkPodCIDR": { + "description": "CIDR Range for Pods in cluster", + "type": "string", + "examples": ["192.0.2.0/24"], + "pattern": "[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/[0-9]{1,2}" + } }, "required": [ "image", @@ -57,8 +57,7 @@ }, "required": [ "image", - "azureConfig", - "subnetworkPodCIDR" + "azureConfig" ] }, "AWS": { diff --git a/cli/internal/helm/loader.go b/cli/internal/helm/loader.go index 669fe4cef..2be39c7f6 100644 --- a/cli/internal/helm/loader.go +++ b/cli/internal/helm/loader.go @@ -39,6 +39,7 @@ type ChartLoader struct { kmsImage string ccmImage string cnmImage string + autoscalerImage string } func New(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion) *ChartLoader { @@ -58,6 +59,7 @@ func New(csp cloudprovider.Provider, k8sVersion versions.ValidK8sVersion) *Chart kmsImage: versions.KmsImage, ccmImage: ccmImage, cnmImage: cnmImage, + autoscalerImage: versions.VersionConfigs[k8sVersion].ClusterAutoscalerImage, } } @@ -164,6 +166,10 @@ func (i *ChartLoader) loadConstellationServices(csp cloudprovider.Provider, "ccm": map[string]any{ "csp": csp, }, + "autoscaler": map[string]any{ + "csp": csp, + "image": i.autoscalerImage, + }, } switch csp { diff --git a/cli/internal/helm/loader_test.go b/cli/internal/helm/loader_test.go index 2a5097f77..c5bd704f1 100644 --- a/cli/internal/helm/loader_test.go +++ b/cli/internal/helm/loader_test.go @@ -77,7 +77,7 @@ func TestTemplate(t *testing.T) { assert := assert.New(t) require := require.New(t) - chartLoader := ChartLoader{joinServiceImage: "joinServiceImage", kmsImage: "kmsImage", ccmImage: tc.ccmImage, cnmImage: tc.cnmImage} + chartLoader := ChartLoader{joinServiceImage: "joinServiceImage", kmsImage: "kmsImage", ccmImage: tc.ccmImage, cnmImage: tc.cnmImage, autoscalerImage: "autoscalerImage"} release, err := chartLoader.Load(tc.csp, true, []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []byte("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"), []uint32{1, 11}, tc.enforceIDKeyDigest) require.NoError(err) @@ -161,6 +161,17 @@ func prepareAzureValues(values map[string]any) error { ccmVals["Azure"].(map[string]any)["subnetworkPodCIDR"] = "192.0.2.0/24" ccmVals["Azure"].(map[string]any)["azureConfig"] = "baaaaaad" + autoscalerVals, ok := values["autoscaler"].(map[string]any) + if !ok { + return errors.New("missing 'autoscaler' key") + } + autoscalerVals["Azure"] = map[string]any{ + "clientID": "AppClientID", + "clientSecret": "ClientSecretValue", + "resourceGroup": "resourceGroup", + "subscriptionID": "subscriptionID", + "tenantID": "TenantID", + } return nil } diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/azure-deployment.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/azure-deployment.yaml new file mode 100644 index 000000000..45dd42bae --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/azure-deployment.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + replicas: 0 + selector: + matchLabels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + template: + metadata: + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + spec: + containers: + - name: cluster-autoscaler + image: autoscalerImage + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /health-check + port: 8085 + ports: + - containerPort: 8085 + env: + - name: ARM_SUBSCRIPTION_ID + valueFrom: + secretKeyRef: + key: SubscriptionID + name: cluster-autoscaler-azure + - name: ARM_RESOURCE_GROUP + valueFrom: + secretKeyRef: + key: ResourceGroup + name: cluster-autoscaler-azure + - name: ARM_TENANT_ID + valueFrom: + secretKeyRef: + key: TenantID + name: cluster-autoscaler-azure + - name: ARM_CLIENT_ID + valueFrom: + secretKeyRef: + key: ClientID + name: cluster-autoscaler-azure + - name: ARM_CLIENT_SECRET + valueFrom: + secretKeyRef: + key: ClientSecret + name: cluster-autoscaler-azure + - name: ARM_VM_TYPE + valueFrom: + secretKeyRef: + key: VMType + name: cluster-autoscaler-azure + resources: {} + dnsPolicy: ClusterFirst + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-cluster-critical + serviceAccountName: constellation-cluster-autoscaler + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Equal + value: "true" diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/azure-secret.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/azure-secret.yaml new file mode 100644 index 000000000..d757253d2 --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/azure-secret.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Secret +metadata: + name: cluster-autoscaler-azure + namespace: testNamespace +data: + ClientID: QXBwQ2xpZW50SUQ= + ClientSecret: Q2xpZW50U2VjcmV0VmFsdWU= + ResourceGroup: cmVzb3VyY2VHcm91cA== + SubscriptionID: c3Vic2NyaXB0aW9uSUQ= + TenantID: VGVuYW50SUQ= + # b64encode("vmss") + VMType: dm1zcw== diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/clusterrole.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/clusterrole.yaml new file mode 100644 index 000000000..501c719d5 --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/clusterrole.yaml @@ -0,0 +1,138 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: constellation-cluster-autoscaler + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +rules: +- apiGroups: + - "" + resources: + - events + - endpoints + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - "" + resources: + - pods/status + verbs: + - update +- apiGroups: + - "" + resourceNames: + - cluster-autoscaler + resources: + - endpoints + verbs: + - get + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - watch + - list + - get + - update +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - replicationcontrollers + - persistentvolumeclaims + - persistentvolumes + verbs: + - watch + - list + - get +- apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - watch + - list + - get +- apiGroups: + - batch + - extensions + resources: + - jobs + verbs: + - get + - list + - patch + - watch +- apiGroups: + - extensions + resources: + - replicasets + - daemonsets + verbs: + - watch + - list + - get +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - watch + - list +- apiGroups: + - apps + resources: + - daemonsets + - replicasets + - statefulsets + verbs: + - watch + - list + - get +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + - csidrivers + - csistoragecapacities + verbs: + - watch + - list + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resourceNames: + - cluster-autoscaler + resources: + - leases + verbs: + - get + - update diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..f2d509015 --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: constellation-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: constellation-cluster-autoscaler + namespace: testNamespace diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..3c474b05d --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml @@ -0,0 +1,15 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/role.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/role.yaml new file mode 100644 index 000000000..e5cdd8ae7 --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/role.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create +- apiGroups: + - "" + resourceNames: + - cluster-autoscaler-status + resources: + - configmaps + verbs: + - delete + - get + - update diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/rolebinding.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/rolebinding.yaml new file mode 100644 index 000000000..5f567e23f --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: constellation-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: constellation-cluster-autoscaler + namespace: kube-system diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/service.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/service.yaml new file mode 100644 index 000000000..84d228ff2 --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + ports: + - name: http + port: 8085 + protocol: TCP + targetPort: 8085 + selector: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + type: ClusterIP diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/serviceaccount.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/serviceaccount.yaml new file mode 100644 index 000000000..79634e292 --- /dev/null +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/autoscaler/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +automountServiceAccountToken: true diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/ccm/templates/azure-daemonset.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/ccm/templates/azure-daemonset.yaml index e75fb3cf4..9d3fe58cd 100644 --- a/cli/internal/helm/testdata/Azure/constellation-services/charts/ccm/templates/azure-daemonset.yaml +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/ccm/templates/azure-daemonset.yaml @@ -1,4 +1,3 @@ - apiVersion: apps/v1 kind: DaemonSet metadata: @@ -16,31 +15,31 @@ spec: k8s-app: cloud-controller-manager spec: containers: - - name: cloud-controller-manager - image: ccmImageForAzure - command: - - cloud-controller-manager - - --cloud-provider=azure - - --leader-elect=true - - -v=2 - - --controllers=*,-cloud-node - - --cloud-config=/etc/azure/azure.json - - --allocate-node-cidrs=false - - --configure-cloud-routes=false - resources: {} - volumeMounts: - - name: etckubernetes - mountPath: /etc/kubernetes - readOnly: true - - name: etcssl - mountPath: /etc/ssl - readOnly: true - - name: etcpki - mountPath: /etc/pki - readOnly: true - - name: azureconfig - mountPath: /etc/azure - readOnly: true + - name: cloud-controller-manager + image: ccmImageForAzure + command: + - cloud-controller-manager + - --cloud-provider=azure + - --leader-elect=true + - -v=2 + - --controllers=*,-cloud-node + - --cloud-config=/etc/azure/azure.json + - --allocate-node-cidrs=false + - --configure-cloud-routes=false + volumeMounts: + - name: etckubernetes + mountPath: /etc/kubernetes + readOnly: true + - name: etcssl + mountPath: /etc/ssl + readOnly: true + - name: etcpki + mountPath: /etc/pki + readOnly: true + - name: azureconfig + mountPath: /etc/azure + readOnly: true + resources: {} nodeSelector: node-role.kubernetes.io/control-plane: "" serviceAccountName: cloud-controller-manager diff --git a/cli/internal/helm/testdata/Azure/constellation-services/charts/ccm/templates/azure-secret.yaml b/cli/internal/helm/testdata/Azure/constellation-services/charts/ccm/templates/azure-secret.yaml index 8ee103766..8b591ff21 100644 --- a/cli/internal/helm/testdata/Azure/constellation-services/charts/ccm/templates/azure-secret.yaml +++ b/cli/internal/helm/testdata/Azure/constellation-services/charts/ccm/templates/azure-secret.yaml @@ -1,4 +1,3 @@ - apiVersion: v1 kind: Secret metadata: @@ -6,4 +5,3 @@ metadata: namespace: testNamespace data: azure.json: YmFhYWFhYWQ= - diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/clusterrole.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/clusterrole.yaml new file mode 100644 index 000000000..501c719d5 --- /dev/null +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/clusterrole.yaml @@ -0,0 +1,138 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: constellation-cluster-autoscaler + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +rules: +- apiGroups: + - "" + resources: + - events + - endpoints + verbs: + - create + - patch +- apiGroups: + - "" + resources: + - pods/eviction + verbs: + - create +- apiGroups: + - "" + resources: + - pods/status + verbs: + - update +- apiGroups: + - "" + resourceNames: + - cluster-autoscaler + resources: + - endpoints + verbs: + - get + - update +- apiGroups: + - "" + resources: + - nodes + verbs: + - watch + - list + - get + - update +- apiGroups: + - "" + resources: + - namespaces + - pods + - services + - replicationcontrollers + - persistentvolumeclaims + - persistentvolumes + verbs: + - watch + - list + - get +- apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - watch + - list + - get +- apiGroups: + - batch + - extensions + resources: + - jobs + verbs: + - get + - list + - patch + - watch +- apiGroups: + - extensions + resources: + - replicasets + - daemonsets + verbs: + - watch + - list + - get +- apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - watch + - list +- apiGroups: + - apps + resources: + - daemonsets + - replicasets + - statefulsets + verbs: + - watch + - list + - get +- apiGroups: + - storage.k8s.io + resources: + - storageclasses + - csinodes + - csidrivers + - csistoragecapacities + verbs: + - watch + - list + - get +- apiGroups: + - "" + resources: + - configmaps + verbs: + - list + - watch +- apiGroups: + - coordination.k8s.io + resources: + - leases + verbs: + - create +- apiGroups: + - coordination.k8s.io + resourceNames: + - cluster-autoscaler + resources: + - leases + verbs: + - get + - update diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml new file mode 100644 index 000000000..f2d509015 --- /dev/null +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/clusterrolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: constellation-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: constellation-cluster-autoscaler + namespace: testNamespace diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml new file mode 100644 index 000000000..f6da44fc9 --- /dev/null +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/gcp-deployment.yaml @@ -0,0 +1,59 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + replicas: 0 + selector: + matchLabels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + template: + metadata: + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + spec: + containers: + - name: cluster-autoscaler + image: autoscalerImage + imagePullPolicy: IfNotPresent + livenessProbe: + httpGet: + path: /health-check + port: 8085 + ports: + - containerPort: 8085 + resources: {} + volumeMounts: + - mountPath: /var/secrets/google + name: gcekey + readOnly: true + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json + dnsPolicy: ClusterFirst + nodeSelector: + node-role.kubernetes.io/control-plane: "" + priorityClassName: system-cluster-critical + serviceAccountName: constellation-cluster-autoscaler + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + operator: Exists + - effect: NoSchedule + key: node.cloudprovider.kubernetes.io/uninitialized + operator: Equal + value: "true" + volumes: + - name: gcekey + secret: + secretName: gcekey diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml new file mode 100644 index 000000000..3c474b05d --- /dev/null +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/poddisruptionbudget.yaml @@ -0,0 +1,15 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + maxUnavailable: 1 + selector: + matchLabels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/role.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/role.yaml new file mode 100644 index 000000000..e5cdd8ae7 --- /dev/null +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/role.yaml @@ -0,0 +1,26 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +rules: +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create +- apiGroups: + - "" + resourceNames: + - cluster-autoscaler-status + resources: + - configmaps + verbs: + - delete + - get + - update diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/rolebinding.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/rolebinding.yaml new file mode 100644 index 000000000..5f567e23f --- /dev/null +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/rolebinding.yaml @@ -0,0 +1,17 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: constellation-cluster-autoscaler +subjects: +- kind: ServiceAccount + name: constellation-cluster-autoscaler + namespace: kube-system diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/service.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/service.yaml new file mode 100644 index 000000000..84d228ff2 --- /dev/null +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +spec: + ports: + - name: http + port: 8085 + protocol: TCP + targetPort: 8085 + selector: + app.kubernetes.io/instance: constellation + app.kubernetes.io/name: cluster-autoscaler + type: ClusterIP diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/serviceaccount.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/serviceaccount.yaml new file mode 100644 index 000000000..79634e292 --- /dev/null +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/autoscaler/templates/serviceaccount.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: constellation-cluster-autoscaler + namespace: testNamespace + labels: + app.kubernetes.io/instance: constellation + app.kubernetes.io/managed-by: Constellation + app.kubernetes.io/name: cluster-autoscaler +automountServiceAccountToken: true diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-cm.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-cm.yaml index 65e1c71b9..24c0cdd00 100644 --- a/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-cm.yaml +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-cm.yaml @@ -1,4 +1,3 @@ - apiVersion: v1 kind: ConfigMap metadata: @@ -6,4 +5,3 @@ metadata: namespace: testNamespace data: gce.conf: "[global]\nproject-id = 42424242424242\nuse-metadata-server = true\nnode-tags = constellation-242424242424\n" - diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-daemonset.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-daemonset.yaml index cb7f52722..9a2a1ad8e 100644 --- a/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-daemonset.yaml +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-daemonset.yaml @@ -1,4 +1,3 @@ - apiVersion: apps/v1 kind: DaemonSet metadata: @@ -30,9 +29,6 @@ spec: - --cidr-allocator-type=CloudAllocator - --allocate-node-cidrs=true - --configure-cloud-routes=false - env: - - name: GOOGLE_APPLICATION_CREDENTIALS - value: /var/secrets/google/key.json volumeMounts: - mountPath: /etc/kubernetes name: etckubernetes @@ -50,6 +46,9 @@ spec: name: gcekey readOnly: true resources: {} + env: + - name: GOOGLE_APPLICATION_CREDENTIALS + value: /var/secrets/google/key.json serviceAccountName: cloud-controller-manager nodeSelector: node-role.kubernetes.io/control-plane: "" diff --git a/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-secret.yaml b/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-secret.yaml index 080a34ed3..19d56a3ab 100644 --- a/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-secret.yaml +++ b/cli/internal/helm/testdata/GCP/constellation-services/charts/ccm/templates/gcp-secret.yaml @@ -1,4 +1,3 @@ - apiVersion: v1 kind: Secret metadata: @@ -6,4 +5,3 @@ metadata: namespace: testNamespace data: key.json: YmFhYWFhYWQ= - diff --git a/internal/cloud/aws/autoscaler.go b/internal/cloud/aws/autoscaler.go deleted file mode 100644 index 6080d03e3..000000000 --- a/internal/cloud/aws/autoscaler.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package aws - -import ( - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - k8s "k8s.io/api/core/v1" -) - -// TODO: Implement for AWS. - -// Autoscaler holds the AWS cluster-autoscaler configuration. -type Autoscaler struct{} - -// Name returns the cloud-provider name as used by k8s cluster-autoscaler. -func (a Autoscaler) Name() string { - return "aws" -} - -// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Secrets(providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - return kubernetes.Secrets{}, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -// Supported is used to determine if we support autoscaling for the cloud provider. -func (a Autoscaler) Supported() bool { - return false -} diff --git a/internal/cloud/azure/autoscaler.go b/internal/cloud/azure/autoscaler.go deleted file mode 100644 index 8c8b26a41..000000000 --- a/internal/cloud/azure/autoscaler.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azure - -import ( - "github.com/edgelesssys/constellation/v2/internal/azureshared" - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - k8s "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// Autoscaler holds the Azure cluster-autoscaler configuration. -type Autoscaler struct{} - -// Name returns the cloud-provider name as used by k8s cluster-autoscaler. -func (a *Autoscaler) Name() string { - return "azure" -} - -// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) Secrets(providerID string, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - subscriptionID, resourceGroup, err := azureshared.BasicsFromProviderID(providerID) - if err != nil { - return kubernetes.Secrets{}, err - } - creds, err := azureshared.ApplicationCredentialsFromURI(cloudServiceAccountURI) - if err != nil { - return kubernetes.Secrets{}, err - } - return kubernetes.Secrets{ - &k8s.Secret{ - TypeMeta: meta.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "cluster-autoscaler-azure", - Namespace: "kube-system", - }, - Data: map[string][]byte{ - "ClientID": []byte(creds.AppClientID), - "ClientSecret": []byte(creds.ClientSecretValue), - "ResourceGroup": []byte(resourceGroup), - "SubscriptionID": []byte(subscriptionID), - "TenantID": []byte(creds.TenantID), - "VMType": []byte("vmss"), - }, - }, - }, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) Env() []k8s.EnvVar { - return []k8s.EnvVar{ - { - Name: "ARM_SUBSCRIPTION_ID", - ValueFrom: &k8s.EnvVarSource{ - SecretKeyRef: &k8s.SecretKeySelector{ - Key: "SubscriptionID", - LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"}, - }, - }, - }, - { - Name: "ARM_RESOURCE_GROUP", - ValueFrom: &k8s.EnvVarSource{ - SecretKeyRef: &k8s.SecretKeySelector{ - Key: "ResourceGroup", - LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"}, - }, - }, - }, - { - Name: "ARM_TENANT_ID", - ValueFrom: &k8s.EnvVarSource{ - SecretKeyRef: &k8s.SecretKeySelector{ - Key: "TenantID", - LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"}, - }, - }, - }, - { - Name: "ARM_CLIENT_ID", - ValueFrom: &k8s.EnvVarSource{ - SecretKeyRef: &k8s.SecretKeySelector{ - Key: "ClientID", - LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"}, - }, - }, - }, - { - Name: "ARM_CLIENT_SECRET", - ValueFrom: &k8s.EnvVarSource{ - SecretKeyRef: &k8s.SecretKeySelector{ - Key: "ClientSecret", - LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"}, - }, - }, - }, - { - Name: "ARM_VM_TYPE", - ValueFrom: &k8s.EnvVarSource{ - SecretKeyRef: &k8s.SecretKeySelector{ - Key: "VMType", - LocalObjectReference: k8s.LocalObjectReference{Name: "cluster-autoscaler-azure"}, - }, - }, - }, - } -} - -// Supported is used to determine if we support autoscaling for the cloud provider. -func (a *Autoscaler) Supported() bool { - return true -} diff --git a/internal/cloud/azure/autoscaler_test.go b/internal/cloud/azure/autoscaler_test.go deleted file mode 100644 index d2d0c2305..000000000 --- a/internal/cloud/azure/autoscaler_test.go +++ /dev/null @@ -1,87 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package azure - -import ( - "testing" - - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - k8s "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -func TestAutoscalerSecrets(t *testing.T) { - testCases := map[string]struct { - providerID string - cloudServiceAccountURI string - wantSecrets kubernetes.Secrets - wantErr bool - }{ - "Secrets works": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachineScaleSets/scaleset/virtualMachines/instance-name", - cloudServiceAccountURI: "serviceaccount://azure?tenant_id=tenant-id&client_id=client-id&client_secret=client-secret", - wantSecrets: kubernetes.Secrets{ - &k8s.Secret{ - TypeMeta: meta.TypeMeta{ - Kind: "Secret", - APIVersion: "v1", - }, - ObjectMeta: meta.ObjectMeta{ - Name: "cluster-autoscaler-azure", - Namespace: "kube-system", - }, - Data: map[string][]byte{ - "ClientID": []byte("client-id"), - "ClientSecret": []byte("client-secret"), - "ResourceGroup": []byte("resource-group"), - "SubscriptionID": []byte("subscription-id"), - "TenantID": []byte("tenant-id"), - "VMType": []byte("vmss"), - }, - }, - }, - }, - "invalid providerID fails": { - providerID: "invalid", - wantErr: true, - }, - "invalid cloudServiceAccountURI fails": { - providerID: "azure:///subscriptions/subscription-id/resourceGroups/resource-group/providers/Microsoft.Compute/virtualMachines/instance-name", - cloudServiceAccountURI: "invalid", - wantErr: true, - }, - } - - for name, tc := range testCases { - t.Run(name, func(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - autoscaler := Autoscaler{} - secrets, err := autoscaler.Secrets(tc.providerID, tc.cloudServiceAccountURI) - if tc.wantErr { - assert.Error(err) - return - } - require.NoError(err) - assert.Equal(tc.wantSecrets, secrets) - }) - } -} - -func TestTrivialAutoscalerFunctions(t *testing.T) { - assert := assert.New(t) - autoscaler := Autoscaler{} - - assert.NotEmpty(autoscaler.Name()) - assert.Empty(autoscaler.Volumes()) - assert.Empty(autoscaler.VolumeMounts()) - assert.NotEmpty(autoscaler.Env()) - assert.True(autoscaler.Supported()) -} diff --git a/internal/cloud/gcp/autoscaler.go b/internal/cloud/gcp/autoscaler.go deleted file mode 100644 index 758cc4836..000000000 --- a/internal/cloud/gcp/autoscaler.go +++ /dev/null @@ -1,65 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - k8s "k8s.io/api/core/v1" -) - -// Autoscaler holds the GCP cluster-autoscaler configuration. -type Autoscaler struct{} - -// Name returns the cloud-provider name as used by k8s cluster-autoscaler. -func (a *Autoscaler) Name() string { - return "gce" -} - -// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) Secrets(instance, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - return kubernetes.Secrets{}, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) Volumes() []k8s.Volume { - return []k8s.Volume{ - { - Name: "gcekey", - VolumeSource: k8s.VolumeSource{ - Secret: &k8s.SecretVolumeSource{ - SecretName: "gcekey", - }, - }, - }, - } -} - -// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{ - { - Name: "gcekey", - ReadOnly: true, - MountPath: "/var/secrets/google", - }, - } -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler. -func (a *Autoscaler) Env() []k8s.EnvVar { - return []k8s.EnvVar{ - { - Name: "GOOGLE_APPLICATION_CREDENTIALS", - Value: "/var/secrets/google/key.json", - }, - } -} - -// Supported is used to determine if we support autoscaling for the cloud provider. -func (a *Autoscaler) Supported() bool { - return true -} diff --git a/internal/cloud/gcp/autoscaler_test.go b/internal/cloud/gcp/autoscaler_test.go deleted file mode 100644 index b129e0aa0..000000000 --- a/internal/cloud/gcp/autoscaler_test.go +++ /dev/null @@ -1,25 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package gcp - -import ( - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestTrivialAutoscalerFunctions(t *testing.T) { - assert := assert.New(t) - autoscaler := Autoscaler{} - - assert.NotEmpty(autoscaler.Name()) - assert.Empty(autoscaler.Secrets("", "")) - assert.NotEmpty(autoscaler.Volumes()) - assert.NotEmpty(autoscaler.VolumeMounts()) - assert.NotEmpty(autoscaler.Env()) - assert.True(autoscaler.Supported()) -} diff --git a/internal/cloud/qemu/autoscaler.go b/internal/cloud/qemu/autoscaler.go deleted file mode 100644 index 8cbe98331..000000000 --- a/internal/cloud/qemu/autoscaler.go +++ /dev/null @@ -1,45 +0,0 @@ -/* -Copyright (c) Edgeless Systems GmbH - -SPDX-License-Identifier: AGPL-3.0-only -*/ - -package qemu - -import ( - "github.com/edgelesssys/constellation/v2/internal/kubernetes" - k8s "k8s.io/api/core/v1" -) - -// Autoscaler holds the QEMU cluster-autoscaler configuration. -type Autoscaler struct{} - -// Name returns the cloud-provider name as used by k8s cluster-autoscaler. -func (a Autoscaler) Name() string { - return "qemu" -} - -// Secrets returns a list of secrets to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Secrets(providerID, cloudServiceAccountURI string) (kubernetes.Secrets, error) { - return kubernetes.Secrets{}, nil -} - -// Volumes returns a list of volumes to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Volumes() []k8s.Volume { - return []k8s.Volume{} -} - -// VolumeMounts returns a list of volume mounts to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) VolumeMounts() []k8s.VolumeMount { - return []k8s.VolumeMount{} -} - -// Env returns a list of k8s environment key-value pairs to deploy together with the k8s cluster-autoscaler. -func (a Autoscaler) Env() []k8s.EnvVar { - return []k8s.EnvVar{} -} - -// Supported is used to determine if we support autoscaling for the cloud provider. -func (a Autoscaler) Supported() bool { - return false -}