Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-24 06:59:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

AB#2538: deploy CCM via Helm

Browse Source 
Also move helmloader interface/stubs
This commit is contained in:
Otto Bittner 2022-10-26 10:37:10 +02:00
parent 009b2e67e3
commit 091e3b2b2b
26 changed files with 579 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
CMakeLists.txt
View File

@ -63,7 +63,9 @@ add_test(NAME integration-node-operator COMMAND make test WORKING_DIRECTORY ${CM
add_test(NAME integration-csi COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/csi)
add_test(NAME integration-dm COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/disk-mapper/internal)
add_test(NAME integration-license COMMAND bash -c "go test -tags integration" WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/internal/license)
add_test(NAME helm-lint COMMAND bash -c "helm lint * --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set join-service.csp='QEMU' \
add_test(NAME helm-lint COMMAND bash -c "helm lint * --set tags.GCP=true --set kms.image='ghcr.io/edgelesssys/constellation/kms:latest' --set join-service.csp='GCP' \
--set join-service.enforcedPCRs='[]' --set join-service.image='ghcr.io/edgelesssys/constellation/join-service:latest' --set join-service.measurements='[]' \
--set join-service.measurementSalt='deadbeef' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'"
--set join-service.measurementSalt='deadbeef' --set kms.salt='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' --set kms.masterSecret='aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' \
--set ccm.csp='GCP' --set ccm.subnetworkCIDR='192.0.2.0/24' --set ccm.GCP.image='ghcr.io/edgelesssys/cloud-provider-gcp:latest' --set ccm.GCP.projectID='demoproject-581925' \
--set ccm.GCP.uid='foobar' --set ccm.GCP.secretData='jsonstring'"
WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/cli/internal/helm/charts/edgeless/)

1
bootstrapper/internal/kubernetes/k8sapi/constants.go
View File

@ -14,5 +14,4 @@ const (
kubeletPath = "/run/state/bin/kubelet"
kubeletServicePath = "/usr/lib/systemd/system/kubelet.service"
executablePerm = 0o544
systemdUnitPerm = 0o644
)

5
bootstrapper/internal/kubernetes/k8sapi/kubectl/client/client_test.go
View File

@ -280,11 +280,6 @@ func TestGetObjects(t *testing.T) {
resourcesYAML: string(nginxDeplYAML),
wantErr: false,
},
"GetObjects works on cloud-controller-manager deployment": {
wantResources: resources.NewDefaultCloudControllerManagerDeployment("someProvider", "someImage", "somePath", "someCIDR", nil, nil, nil, nil),
resourcesYAML: string(nginxDeplYAML),
wantErr: false,
},
"GetObjects Marshal failure detected": {
wantResources: &unmarshableResource{},
resourcesYAML: string(nginxDeplYAML),

179
bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager.go
View File

@ -1,179 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"fmt"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
apps "k8s.io/api/apps/v1"
k8s "k8s.io/api/core/v1"
rbac "k8s.io/api/rbac/v1"
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type CloudControllerManagerDeployment struct {
ServiceAccount k8s.ServiceAccount
ClusterRoleBinding rbac.ClusterRoleBinding
DaemonSet apps.DaemonSet
}
// references:
// https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/admin/cloud/ccm-example.yaml
// https://kubernetes.io/docs/tasks/administer-cluster/running-cloud-controller/#cloud-controller-manager
// NewDefaultCloudControllerManagerDeployment creates a new *cloudControllerManagerDeployment, customized for the CSP.
func NewDefaultCloudControllerManagerDeployment(cloudProvider, image, path, podCIDR string, extraArgs []string, extraVolumes []k8s.Volume, extraVolumeMounts []k8s.VolumeMount, env []k8s.EnvVar) *CloudControllerManagerDeployment {
command := []string{
path,
fmt.Sprintf("--cloud-provider=%s", cloudProvider),
"--leader-elect=true",
fmt.Sprintf("--cluster-cidr=%s", podCIDR),
"-v=2",
}
command = append(command, extraArgs...)
volumes := []k8s.Volume{
{
Name: "etckubernetes",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/kubernetes"},
},
},
{
Name: "etcssl",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/ssl"},
},
},
{
Name: "etcpki",
VolumeSource: k8s.VolumeSource{
HostPath: &k8s.HostPathVolumeSource{Path: "/etc/pki"},
},
},
}
volumes = append(volumes, extraVolumes...)
volumeMounts := []k8s.VolumeMount{
{
MountPath: "/etc/kubernetes",
Name: "etckubernetes",
ReadOnly: true,
},
{
MountPath: "/etc/ssl",
Name: "etcssl",
ReadOnly: true,
},
{
MountPath: "/etc/pki",
Name: "etcpki",
ReadOnly: true,
},
}
volumeMounts = append(volumeMounts, extraVolumeMounts...)
return &CloudControllerManagerDeployment{
ServiceAccount: k8s.ServiceAccount{
TypeMeta: meta.TypeMeta{
APIVersion: "v1",
Kind: "ServiceAccount",
},
ObjectMeta: meta.ObjectMeta{
Name: "cloud-controller-manager",
Namespace: "kube-system",
},
},
ClusterRoleBinding: rbac.ClusterRoleBinding{
TypeMeta: meta.TypeMeta{
APIVersion: "rbac.authorization.k8s.io/v1",
Kind: "ClusterRoleBinding",
},
ObjectMeta: meta.ObjectMeta{
Name: "system:cloud-controller-manager",
},
RoleRef: rbac.RoleRef{
APIGroup: "rbac.authorization.k8s.io",
Kind: "ClusterRole",
Name: "cluster-admin",
},
Subjects: []rbac.Subject{
{
Kind: "ServiceAccount",
Name: "cloud-controller-manager",
Namespace: "kube-system",
},
},
},
DaemonSet: apps.DaemonSet{
TypeMeta: meta.TypeMeta{
APIVersion: "apps/v1",
Kind: "DaemonSet",
},
ObjectMeta: meta.ObjectMeta{
Labels: map[string]string{
"k8s-app": "cloud-controller-manager",
},
Name: "cloud-controller-manager",
Namespace: "kube-system",
},
Spec: apps.DaemonSetSpec{
Selector: &meta.LabelSelector{
MatchLabels: map[string]string{
"k8s-app": "cloud-controller-manager",
},
},
Template: k8s.PodTemplateSpec{
ObjectMeta: meta.ObjectMeta{
Labels: map[string]string{
"k8s-app": "cloud-controller-manager",
},
},
Spec: k8s.PodSpec{
ServiceAccountName: "cloud-controller-manager",
Containers: []k8s.Container{
{
Name: "cloud-controller-manager",
Image: image,
Command: command,
VolumeMounts: volumeMounts,
Env: env,
},
},
Volumes: volumes,
Tolerations: []k8s.Toleration{
{
Key: "node.cloudprovider.kubernetes.io/uninitialized",
Value: "true",
Effect: k8s.TaintEffectNoSchedule,
},
{
Key: "node-role.kubernetes.io/master",
Effect: k8s.TaintEffectNoSchedule,
},
{
Key: "node-role.kubernetes.io/control-plane",
Operator: k8s.TolerationOpExists,
Effect: k8s.TaintEffectNoSchedule,
},
{
Key: "node.kubernetes.io/not-ready",
Effect: k8s.TaintEffectNoSchedule,
},
},
NodeSelector: map[string]string{
"node-role.kubernetes.io/control-plane": "",
},
},
},
},
},
}
}
func (c *CloudControllerManagerDeployment) Marshal() ([]byte, error) {
return kubernetes.MarshalK8SResources(c)
}

29
bootstrapper/internal/kubernetes/k8sapi/resources/cloud_controller_manager_test.go
View File

@ -1,29 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package resources
import (
"testing"
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
k8s "k8s.io/api/core/v1"
)
func TestCloudControllerMarshalUnmarshal(t *testing.T) {
require := require.New(t)
assert := assert.New(t)
cloudControllerManagerDepl := NewDefaultCloudControllerManagerDeployment("dummy-cloudprovider", "some-image:latest", "/dummy_path", "192.0.2.0/24", []string{}, []k8s.Volume{}, []k8s.VolumeMount{}, nil)
data, err := cloudControllerManagerDepl.Marshal()
require.NoError(err)
var recreated CloudControllerManagerDeployment
require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated))
assert.Equal(cloudControllerManagerDepl, &recreated)
}

115
bootstrapper/internal/kubernetes/kubernetes.go
View File

@ -11,6 +11,7 @@ import (
"encoding/base64"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"net"
"strconv"
@ -18,9 +19,11 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/gcpshared"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions"
@ -201,7 +204,10 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("setting up konnectivity: %w", err)
}
extraVals := setupExtraVals(k.initialMeasurementsJSON, idKeyDigest, measurementSalt)
extraVals, err := k.setupExtraVals(ctx, k.initialMeasurementsJSON, idKeyDigest, measurementSalt, subnetworkPodCIDR, cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("setting up extraVals: %w", err)
}
if err = k.helmClient.InstallConstellationServices(ctx, helmReleases.ConstellationServices, extraVals); err != nil {
return nil, fmt.Errorf("installing constellation-services: %w", err)
@ -211,9 +217,6 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("failed to setup internal ConfigMap: %w", err)
}
if err := k.setupCCM(ctx, subnetworkPodCIDR, cloudServiceAccountURI, instance, k8sVersion); err != nil {
return nil, fmt.Errorf("setting up cloud controller manager: %w", err)
}
if err := k.setupCloudNodeManager(k8sVersion); err != nil {
return nil, fmt.Errorf("setting up cloud node manager: %w", err)
}
@ -328,34 +331,6 @@ func (k *KubeWrapper) GetKubeconfig() ([]byte, error) {
return k.kubeconfigReader.ReadKubeconfig()
}
func (k *KubeWrapper) setupCCM(ctx context.Context, subnetworkPodCIDR, cloudServiceAccountURI string, instance metadata.InstanceMetadata, k8sVersion versions.ValidK8sVersion) error {
if !k.cloudControllerManager.Supported() {
return nil
}
ccmConfigMaps, err := k.cloudControllerManager.ConfigMaps()
if err != nil {
return fmt.Errorf("defining ConfigMaps for CCM: %w", err)
}
ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return fmt.Errorf("defining Secrets for CCM: %w", err)
}
ccmImage, err := k.cloudControllerManager.Image(k8sVersion)
if err != nil {
return fmt.Errorf("defining Image for CCM: %w", err)
}
cloudControllerManagerConfiguration := resources.NewDefaultCloudControllerManagerDeployment(
k.cloudControllerManager.Name(), ccmImage, k.cloudControllerManager.Path(), subnetworkPodCIDR,
k.cloudControllerManager.ExtraArgs(), k.cloudControllerManager.Volumes(), k.cloudControllerManager.VolumeMounts(), k.cloudControllerManager.Env(),
)
if err := k.clusterUtil.SetupCloudControllerManager(k.client, cloudControllerManagerConfiguration, ccmConfigMaps, ccmSecrets); err != nil {
return fmt.Errorf("setting up cloud-controller-manager: %w", err)
}
return nil
}
func (k *KubeWrapper) setupCloudNodeManager(k8sVersion versions.ValidK8sVersion) error {
if !k.cloudNodeManager.Supported() {
return nil
@ -498,12 +473,82 @@ func getIPAddr() (string, error) {
// setupExtraVals create a helm values map for consumption by helm-install.
// Will move to a more dedicated place once that place becomes apparent.
func setupExtraVals(initialMeasurementsJSON []byte, idkeydigest []byte, measurementSalt []byte) map[string]any {
return map[string]any{
func (k *KubeWrapper) setupExtraVals(ctx context.Context, initialMeasurementsJSON []byte, idkeydigest []byte, measurementSalt []byte, subnetworkCIDR string, cloudServiceAccountURI string) (map[string]any, error) {
extraVals := map[string]any{
"join-service": map[string]any{
"measurements": string(initialMeasurementsJSON),
"idkeydigest": hex.EncodeToString(idkeydigest),
"measurementSalt": base64.StdEncoding.EncodeToString(measurementSalt),
},
"ccm": map[string]any{
"subnetworkCIDR": subnetworkCIDR,
},
}
instance, err := k.providerMetadata.Self(ctx)
if err != nil {
return nil, fmt.Errorf("retrieving current instance: %w", err)
}
switch cloudprovider.FromString(k.cloudProvider) {
case cloudprovider.GCP:
{
uid, err := k.providerMetadata.UID(ctx)
if err != nil {
return nil, fmt.Errorf("getting uid: %w", err)
}
projectID, _, _, err := gcpshared.SplitProviderID(instance.ProviderID)
if err != nil {
return nil, fmt.Errorf("splitting providerID: %w", err)
}
serviceAccountKey, err := gcpshared.ServiceAccountKeyFromURI(cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("getting service account key: %w", err)
}
rawKey, err := json.Marshal(serviceAccountKey)
if err != nil {
return nil, fmt.Errorf("marshaling service account key: %w", err)
}
ccmVals, ok := extraVals["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["GCP"] = map[string]any{
"projectID": projectID,
"uid": uid,
"secretData": string(rawKey),
}
}
case cloudprovider.Azure:
{
// TODO: After refactoring the ProviderMetadata interface this section should be rewritten.
// Currently, we have to rely on the Secrets(..) method, as GetNetworkSecurityGroupName & GetLoadBalancerName
// rely on Azure specific API endpoints.
ccmSecrets, err := k.cloudControllerManager.Secrets(ctx, instance.ProviderID, cloudServiceAccountURI)
if err != nil {
return nil, fmt.Errorf("creating ccm secret: %w", err)
}
if len(ccmSecrets) < 1 {
return nil, errors.New("missing secret")
}
rawConfig := ccmSecrets[0].Data["azure.json"]
ccmVals, ok := extraVals["ccm"].(map[string]any)
if !ok {
return nil, errors.New("invalid ccm values")
}
ccmVals["Azure"] = map[string]any{
"azureConfig": string(rawConfig),
}
joinVals, ok := extraVals["join-service"].(map[string]any)
if !ok {
return nil, errors.New("invalid join-service values")
}
joinVals["idkeydigest"] = hex.EncodeToString(idkeydigest)
}
}
return extraVals, nil
}

27
bootstrapper/internal/kubernetes/kubernetes_test.go
View File

@ -191,7 +191,7 @@ func TestInitCluster(t *testing.T) {
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up the join service": {
"kubeadm init fails when setting up constellation-services chart": {
clusterUtil: stubClusterUtil{},
helmClient: stubHelmClient{servicesError: someErr},
kubeconfigReader: &stubKubeconfigReader{
@ -204,18 +204,6 @@ func TestInitCluster(t *testing.T) {
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting the cloud contoller manager": {
clusterUtil: stubClusterUtil{setupCloudControllerManagerError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{},
CloudControllerManager: &stubCloudControllerManager{SupportedResp: true},
CloudNodeManager: &stubCloudNodeManager{},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting the cloud node manager": {
clusterUtil: stubClusterUtil{setupCloudNodeManagerError: someErr},
kubeconfigReader: &stubKubeconfigReader{
@ -252,19 +240,6 @@ func TestInitCluster(t *testing.T) {
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up the kms": {
clusterUtil: stubClusterUtil{},
helmClient: stubHelmClient{servicesError: someErr},
kubeconfigReader: &stubKubeconfigReader{
Kubeconfig: []byte("someKubeconfig"),
},
providerMetadata: &stubProviderMetadata{SupportedResp: false},
CloudControllerManager: &stubCloudControllerManager{},
CloudNodeManager: &stubCloudNodeManager{SupportedResp: false},
ClusterAutoscaler: &stubClusterAutoscaler{},
wantErr: true,
k8sVersion: versions.Default,
},
"kubeadm init fails when setting up konnectivity": {
clusterUtil: stubClusterUtil{setupKonnectivityError: someErr},
kubeconfigReader: &stubKubeconfigReader{

21
cli/internal/cmd/helmloader.go
View File

@ -1,21 +0,0 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package cmd
import "github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
type helmLoader interface {
Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error)
}
type stubHelmLoader struct {
loadErr error
}
func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) {
return nil, d.loadErr
}

6
cli/internal/cmd/init.go
View File

@ -126,7 +126,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
return fmt.Errorf("parsing or generating master secret from file %s: %w", flags.masterSecretPath, err)
}
helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt, getEnforcedPCRs(provider, config), getEnforceIDKeyDigest(provider, config))
helmDeployments, err := helmLoader.Load(provider, flags.conformance, masterSecret.Key, masterSecret.Salt, getEnforcedPCRs(provider, config), getEnforceIDKeyDigest(provider, config), k8sVersion)
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
@ -368,3 +368,7 @@ func getMarshaledServiceAccountURI(provider cloudprovider.Provider, config *conf
type grpcDialer interface {
Dial(ctx context.Context, target string) (*grpc.ClientConn, error)
}
type helmLoader interface {
Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error)
}

9
cli/internal/cmd/init_test.go
View File

@ -31,6 +31,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/grpc/testdialer"
"github.com/edgelesssys/constellation/v2/internal/license"
"github.com/edgelesssys/constellation/v2/internal/oid"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
@ -503,3 +504,11 @@ func (c *stubLicenseClient) QuotaCheck(ctx context.Context, checkRequest license
Quota: 25,
}, nil
}
type stubHelmLoader struct {
loadErr error
}
func (d *stubHelmLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error) {
return nil, d.loadErr
}

16
cli/internal/helm/charts/edgeless/constellation-services/Chart.yaml
View File

@ -7,5 +7,21 @@ version: 2.2.0-pre
dependencies:
- name: kms
version: 2.2.0-pre
tags:
- Azure
- GCP
- AWS
- QEMU
- name: join-service
version: 2.2.0-pre
tags:
- Azure
- GCP
- AWS
- QEMU
- name: ccm
version: 2.2.0-pre
tags:
- Azure
- GCP
- AWS

23
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/.helmignore Normal file
View File

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

5
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/Chart.yaml Normal file
View File

@ -0,0 +1,5 @@
apiVersion: v2
name: ccm
description: A Helm chart to deploy the cloud controller manager.
type: application
version: 2.2.0-pre

63
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/aws-daemonset.yaml Normal file
View File

@ -0,0 +1,63 @@
{{ if eq .Values.csp "AWS" }}
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: cloud-controller-manager
namespace: {{ .Release.Namespace }}
labels:
k8s-app: cloud-controller-manager
spec:
selector:
matchLabels:
k8s-app: cloud-controller-manager
template:
metadata:
labels:
k8s-app: cloud-controller-manager
spec:
containers:
- name: cloud-controller-manager
image: {{ .Values.AWS.image }}
command:
- /aws-cloud-controller-manager
- --cloud-provider=aws
- --leader-elect=true
- --cluster-cidr={{ .Values.subnetworkCIDR }}
- -v=2
volumeMounts:
- name: etckubernetes
mountPath: /etc/kubernetes
readOnly: true
- name: etcssl
mountPath: /etc/ssl
readOnly: true
- name: etcpki
mountPath: /etc/pki
readOnly: true
resources: {}
nodeSelector:
node-role.kubernetes.io/control-plane: ""
serviceAccountName: cloud-controller-manager
tolerations:
- effect: NoSchedule
key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
- effect: NoSchedule
key: node.kubernetes.io/not-ready
volumes:
- name: etckubernetes
hostPath:
path: /etc/kubernetes
- name: etcssl
hostPath:
path: /etc/ssl
- name: etcpki
hostPath:
path: /etc/pki
updateStrategy: {}
{{ end }}

73
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-daemonset.yaml Normal file
View File

@ -0,0 +1,73 @@
{{ if eq .Values.csp "Azure" }}
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: cloud-controller-manager
namespace: {{ .Release.Namespace}}
labels:
k8s-app: cloud-controller-manager
spec:
selector:
matchLabels:
k8s-app: cloud-controller-manager
template:
metadata:
labels:
k8s-app: cloud-controller-manager
spec:
containers:
- name: cloud-controller-manager
image: {{ .Values.Azure.image }}
command:
- cloud-controller-manager
- --cloud-provider=azure
- --leader-elect=true
- --cluster-cidr={{ .Values.subnetworkCIDR }}
- -v=2
- --controllers=*,-cloud-node
- --cloud-config=/etc/azure/azure.json
- --allocate-node-cidrs=false
- --configure-cloud-routes=true
volumeMounts:
- name: etckubernetes
mountPath: /etc/kubernetes
readOnly: true
- name: etcssl
mountPath: /etc/ssl
readOnly: true
- name: etcpki
mountPath: /etc/pki
readOnly: true
- name: azureconfig
mountPath: /etc/azure
readOnly: true
resources: {}
nodeSelector:
node-role.kubernetes.io/control-plane: ""
serviceAccountName: cloud-controller-manager
tolerations:
- effect: NoSchedule
key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
- effect: NoSchedule
key: node.kubernetes.io/not-ready
volumes:
- name: etckubernetes
hostPath:
path: /etc/kubernetes
- name: etcssl
hostPath:
path: /etc/ssl
- name: etcpki
hostPath:
path: /etc/pki
- name: azureconfig
secret:
secretName: azureconfig
updateStrategy: {}
{{ end }}

9
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/azure-secret.yaml Normal file
View File

@ -0,0 +1,9 @@
{{ if eq .Values.csp "Azure" }}
apiVersion: v1
kind: Secret
metadata:
name: azureconfig
namespace: {{ .Release.Namespace }}
data:
azure.json: {{ .Values.Azure.azureConfig | b64enc }}
{{ end }}

12
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/clusterrolebinding.yaml Normal file
View File

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: system:cloud-controller-manager
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: cluster-admin
subjects:
- kind: ServiceAccount
name: cloud-controller-manager
namespace: {{ .Release.Namespace }}

9
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-cm.yaml Normal file
View File

@ -0,0 +1,9 @@
{{ if eq .Values.csp "GCP" }}
apiVersion: v1
kind: ConfigMap
metadata:
name: gceconf
namespace: {{ .Release.Namespace }}
data:
gce.conf: "[global]\nproject-id = {{.Values.GCP.projectID }}\nuse-metadata-server = true\nnode-tags = constellation-{{ .Values.GCP.uid }}\n"
{{ end }}

84
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-daemonset.yaml Normal file
View File

@ -0,0 +1,84 @@
{{ if eq .Values.csp "GCP" }}
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: cloud-controller-manager
namespace: {{ .Release.Namespace }}
labels:
k8s-app: cloud-controller-manager
spec:
selector:
matchLabels:
k8s-app: cloud-controller-manager
template:
metadata:
labels:
k8s-app: cloud-controller-manager
spec:
containers:
- name: cloud-controller-manager
image: {{ .Values.GCP.image }}
command:
- /cloud-controller-manager
- --cloud-provider=gce
- --leader-elect=true
- --cluster-cidr={{ .Values.subnetworkCIDR }}
- -v=2
- --use-service-account-credentials
- --controllers=cloud-node,cloud-node-lifecycle,nodeipam,service,route
- --cloud-config=/etc/gce/gce.conf
- --cidr-allocator-type=CloudAllocator
- --allocate-node-cidrs=true
- --configure-cloud-routes=false
volumeMounts:
- mountPath: /etc/kubernetes
name: etckubernetes
readOnly: true
- mountPath: /etc/ssl
name: etcssl
readOnly: true
- mountPath: /etc/pki
name: etcpki
readOnly: true
- mountPath: /etc/gce
name: gceconf
readOnly: true
- mountPath: /var/secrets/google
name: gcekey
readOnly: true
resources: {}
env:
- name: GOOGLE_APPLICATION_CREDENTIALS
value: /var/secrets/google/key.json
serviceAccountName: cloud-controller-manager
nodeSelector:
node-role.kubernetes.io/control-plane: ""
tolerations:
- effect: NoSchedule
key: node.cloudprovider.kubernetes.io/uninitialized
value: "true"
- effect: NoSchedule
key: node-role.kubernetes.io/master
- effect: NoSchedule
key: node-role.kubernetes.io/control-plane
operator: Exists
- effect: NoSchedule
key: node.kubernetes.io/not-ready
volumes:
- name: etckubernetes
hostPath:
path: /etc/kubernetes
- name: etcssl
hostPath:
path: /etc/ssl
- name: etcpki
hostPath:
path: /etc/pki
- name: gceconf
configMap:
name: gceconf
- name: gcekey
secret:
secretName: gcekey
updateStrategy: {}
{{ end }}

9
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/gcp-secret.yaml Normal file
View File

@ -0,0 +1,9 @@
{{ if eq .Values.csp "GCP" }}
apiVersion: v1
kind: Secret
metadata:
name: gcekey
namespace: {{ .Release.Namespace }}
data:
key.json: {{ .Values.GCP.secretData | b64enc }}
{{ end }}

5
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/templates/serviceaccount.yaml Normal file
View File

@ -0,0 +1,5 @@
apiVersion: v1
kind: ServiceAccount
metadata:
name: cloud-controller-manager
namespace: {{ .Release.Namespace }}

105
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.schema.json Normal file
View File

@ -0,0 +1,105 @@
{
"$schema": "https://json-schema.org/draft-07/schema#",
"properties": {
"csp": {
"description": "CSP to which the chart is deployed.",
"enum": ["Azure", "GCP", "AWS", "QEMU"]
},
"subnetworkCIDR": {
"description": "CIDR for the subnetwork of the cluster",
"type": "string",
"examples": ["192.0.2.0/24"],
"pattern": "[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}.[0-9]{1,3}/[0-9]{1,2}"
},
"GCP": {
"description": "Config values required for deployment on GCP",
"type": "object",
"properties": {
"image": {
"description": "Container image to use for the spawned pods.",
"type": "string"
},
"projectID": {
"description": "ID of the GCP project into which the cluster is deployed",
"type": "string",
"examples": ["demoproject-581925"]
},
"uid": {
"description": "Unique identifier for the cluster",
"type": "string"
},
"secretData": {
"description": "GCP service account key as a json-string",
"type": "string"
}
},
"required": [
"image",
"projectID",
"uid",
"secretData"
]
},
"Azure": {
"description": "Config values required for deployment on Azure",
"type": "object",
"properties": {
"image": {
"description": "Container image to use for the spawned pods.",
"type": "string",
"examples": ["mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager:latest"]
},
"azureConfig": {
"description": "Base64 encoded json string that hold required config parameters for Azure CCM.",
"type": "string"
}
},
"required": [
"image",
"azureConfig"
]
},
"AWS": {
"description": "Config values required for deployment on AWS",
"type": "object",
"properties": {
"image": {
"description": "Container image to use for the spawned pods.",
"type": "string"
}
},
"required": [
"image"
]
}
},
"required": [
"csp",
"subnetworkCIDR"
],
"allOf": [
{
"if": {
"properties": { "csp": { "const": "GCP" } },
"required": ["csp"]
},
"then": { "required": ["GCP"] }
},
{
"if": {
"properties": { "csp": { "const": "Azure" } },
"required": ["csp"]
},
"then": { "required": ["Azure"] }
},
{
"if": {
"properties": { "csp": { "const": "AWS" } },
"required": ["csp"]
},
"then": { "required": ["AWS"] }
}
],
"title": "Values",
"type": "object"
}

0
cli/internal/helm/charts/edgeless/constellation-services/charts/ccm/values.yaml Normal file
View File

7
cli/internal/helm/charts/edgeless/constellation-services/values.yaml
View File

@ -9,3 +9,10 @@ global:
k8sVersionCMName: k8s-version
# Name of the ConfigMap that holds configs that should not be modified by the user.
internalCMName: internal-config
# Set one of the tags to true to indicate which CSP you are deploying to.
tags:
Azure: false
GCP: false
AWS: false
QEMU: false

66
cli/internal/helm/loader.go
View File

@ -36,13 +36,13 @@ var HelmFS embed.FS
type ChartLoader struct{}
func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool) ([]byte, error) {
func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool, masterSecret []byte, salt []byte, enforcedPCRs []uint32, enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion) ([]byte, error) {
ciliumRelease, err := i.loadCilium(csp, conformanceMode)
if err != nil {
return nil, err
}
conServicesRelease, err := i.loadConstellationServices(csp, masterSecret, salt, enforcedPCRs, enforceIDKeyDigest)
conServicesRelease, err := i.loadConstellationServices(csp, masterSecret, salt, enforcedPCRs, enforceIDKeyDigest, k8sVersion)
if err != nil {
return nil, err
}
@ -93,7 +93,7 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
// loadConstellationServices loads the constellation-services chart from the embed.FS, marshals it into a helm-package .tgz and sets the values that can be set in the CLI.
func (i *ChartLoader) loadConstellationServices(csp cloudprovider.Provider,
masterSecret []byte, salt []byte, enforcedPCRs []uint32,
enforceIDKeyDigest bool,
enforceIDKeyDigest bool, k8sVersion versions.ValidK8sVersion,
) (helm.Release, error) {
chart, err := loadChartsDir(HelmFS, "charts/edgeless/constellation-services")
if err != nil {
@ -134,14 +134,64 @@ func (i *ChartLoader) loadConstellationServices(csp cloudprovider.Provider,
"image": versions.JoinImage,
"namespace": constants.ConstellationNamespace,
},
"ccm": map[string]interface{}{
"csp": csp,
},
}
if csp == cloudprovider.Azure {
joinServiceVals, ok := vals["join-service"].(map[string]any)
if !ok {
return helm.Release{}, errors.New("invalid join-service values")
switch csp {
case cloudprovider.Azure:
{
joinServiceVals, ok := vals["join-service"].(map[string]any)
if !ok {
return helm.Release{}, errors.New("invalid join-service values")
}
joinServiceVals["enforceIdKeyDigest"] = enforceIDKeyDigest
ccmVals, ok := vals["ccm"].(map[string]any)
if !ok {
return helm.Release{}, errors.New("invalid ccm values")
}
ccmVals["Azure"] = map[string]any{
"image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAzure,
}
vals["tags"] = map[string]any{
"Azure": true,
}
}
case cloudprovider.GCP:
{
ccmVals, ok := vals["ccm"].(map[string]any)
if !ok {
return helm.Release{}, errors.New("invalid ccm values")
}
ccmVals["GCP"] = map[string]any{
"image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageGCP,
}
vals["tags"] = map[string]any{
"GCP": true,
}
}
case cloudprovider.QEMU:
{
vals["tags"] = map[string]interface{}{
"QEMU": true,
}
}
case cloudprovider.AWS:
ccmVals, ok := vals["ccm"].(map[string]any)
if !ok {
return helm.Release{}, errors.New("invalid ccm values")
}
ccmVals["AWS"] = map[string]any{
"image": versions.VersionConfigs[k8sVersion].CloudControllerManagerImageAWS,
}
vals["tags"] = map[string]any{
"AWS": true,
}
joinServiceVals["enforceIdKeyDigest"] = enforceIDKeyDigest
}
return helm.Release{Chart: chartRaw, Values: vals, ReleaseName: "constellation-services", Wait: true}, nil

3
cli/internal/helm/loader_test.go
View File

@ -13,6 +13,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/versions"
"github.com/stretchr/testify/assert"
"helm.sh/helm/v3/pkg/chart/loader"
)
@ -21,7 +22,7 @@ func TestLoad(t *testing.T) {
assert := assert.New(t)
chartLoader := ChartLoader{}
release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"), nil, false)
release, err := chartLoader.Load(cloudprovider.GCP, true, []byte("secret"), []byte("salt"), nil, false, versions.Default)
assert.NoError(err)
var helmReleases helm.Releases