mirror of
https://github.com/edgelesssys/constellation.git
synced 2025-01-27 07:47:02 -05:00
AB#2490: deploy KMS via Helm
* Bundle helm-install related code in speparate package * Move cilium installation to new helm package
This commit is contained in:
parent
62168bbf98
commit
c6ccee1250
@ -14,6 +14,7 @@ import (
|
||||
"os"
|
||||
"strconv"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/initserver"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
|
||||
@ -69,6 +70,11 @@ func main() {
|
||||
var openTPM vtpm.TPMOpenFunc
|
||||
var fs afero.Fs
|
||||
|
||||
helmClient, err := helm.New(log)
|
||||
if err != nil {
|
||||
log.With(zap.Error(err)).Fatalf("Helm client could not be initialized")
|
||||
}
|
||||
|
||||
switch cloudprovider.FromString(os.Getenv(constellationCSP)) {
|
||||
case cloudprovider.AWS:
|
||||
panic("AWS cloud provider currently unsupported")
|
||||
@ -104,7 +110,7 @@ func main() {
|
||||
}
|
||||
clusterInitJoiner = kubernetes.New(
|
||||
"gcp", k8sapi.NewKubernetesUtil(), &k8sapi.CoreOSConfiguration{}, kubectl.New(), cloudControllerManager,
|
||||
&gcpcloud.CloudNodeManager{}, &gcpcloud.Autoscaler{}, metadata, pcrsJSON,
|
||||
&gcpcloud.CloudNodeManager{}, &gcpcloud.Autoscaler{}, metadata, pcrsJSON, helmClient,
|
||||
)
|
||||
openTPM = vtpm.OpenVTPM
|
||||
fs = afero.NewOsFs()
|
||||
@ -137,7 +143,7 @@ func main() {
|
||||
}
|
||||
clusterInitJoiner = kubernetes.New(
|
||||
"azure", k8sapi.NewKubernetesUtil(), &k8sapi.CoreOSConfiguration{}, kubectl.New(), azurecloud.NewCloudControllerManager(metadata),
|
||||
&azurecloud.CloudNodeManager{}, &azurecloud.Autoscaler{}, metadata, pcrsJSON,
|
||||
&azurecloud.CloudNodeManager{}, &azurecloud.Autoscaler{}, metadata, pcrsJSON, helmClient,
|
||||
)
|
||||
|
||||
openTPM = vtpm.OpenVTPM
|
||||
@ -158,7 +164,7 @@ func main() {
|
||||
}
|
||||
clusterInitJoiner = kubernetes.New(
|
||||
"qemu", k8sapi.NewKubernetesUtil(), &k8sapi.CoreOSConfiguration{}, kubectl.New(), &qemucloud.CloudControllerManager{},
|
||||
&qemucloud.CloudNodeManager{}, &qemucloud.Autoscaler{}, metadata, pcrsJSON,
|
||||
&qemucloud.CloudNodeManager{}, &qemucloud.Autoscaler{}, metadata, pcrsJSON, helmClient,
|
||||
)
|
||||
metadataAPI = metadata
|
||||
|
||||
|
@ -9,7 +9,7 @@ package main
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/edgelesssys/constellation/v2/internal/role"
|
||||
@ -22,7 +22,7 @@ type clusterFake struct{}
|
||||
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
|
||||
func (c *clusterFake) InitCluster(
|
||||
context.Context, string, string, []byte, []uint32, bool, []byte, bool,
|
||||
resources.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
|
||||
helm.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
|
||||
) ([]byte, error) {
|
||||
return []byte{}, nil
|
||||
}
|
||||
|
174
bootstrapper/internal/helm/client.go
Normal file
174
bootstrapper/internal/helm/client.go
Normal file
@ -0,0 +1,174 @@
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package helm
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"helm.sh/helm/v3/pkg/action"
|
||||
"helm.sh/helm/v3/pkg/cli"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
// timeout is the maximum time given to the helm client.
|
||||
timeout = 5 * time.Minute
|
||||
)
|
||||
|
||||
// Client is used to install microservice during cluster initialization. It is a wrapper for a helm install action.
|
||||
type Client struct {
|
||||
*action.Install
|
||||
}
|
||||
|
||||
// New creates a new client with the given logger.
|
||||
func New(log *logger.Logger) (*Client, error) {
|
||||
settings := cli.New()
|
||||
settings.KubeConfig = constants.CoreOSAdminConfFilename
|
||||
|
||||
actionConfig := &action.Configuration{}
|
||||
if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace,
|
||||
"secret", log.Infof); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &Client{
|
||||
action.NewInstall(actionConfig),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// InstallCilium sets up the cilium pod network.
|
||||
func (h *Client) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
|
||||
h.Namespace = constants.HelmNamespace
|
||||
h.ReleaseName = release.ReleaseName
|
||||
h.Wait = release.Wait
|
||||
h.Timeout = timeout
|
||||
|
||||
switch in.CloudProvider {
|
||||
case "gcp":
|
||||
return h.installlCiliumGCP(ctx, kubectl, release, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint)
|
||||
case "azure":
|
||||
return h.installCiliumAzure(ctx, release, in.LoadBalancerEndpoint)
|
||||
case "qemu":
|
||||
return h.installCiliumQEMU(ctx, release, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint)
|
||||
default:
|
||||
return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func (h *Client) installCiliumAzure(ctx context.Context, release helm.Release, kubeAPIEndpoint string) error {
|
||||
host := kubeAPIEndpoint
|
||||
release.Values["k8sServiceHost"] = host
|
||||
release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
|
||||
|
||||
_, err := h.RunWithContext(ctx, release.Chart, release.Values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("installing cilium: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Client) installlCiliumGCP(ctx context.Context, kubectl k8sapi.Client, release helm.Release, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIEndpoint string) error {
|
||||
out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", constants.CoreOSAdminConfFilename, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput()
|
||||
if err != nil {
|
||||
err = errors.New(string(out))
|
||||
return err
|
||||
}
|
||||
|
||||
timeoutS := int64(10)
|
||||
// allow coredns to run on uninitialized nodes (required by cloud-controller-manager)
|
||||
tolerations := []corev1.Toleration{
|
||||
{
|
||||
Key: "node.cloudprovider.kubernetes.io/uninitialized",
|
||||
Value: "true",
|
||||
Effect: corev1.TaintEffectNoSchedule,
|
||||
},
|
||||
{
|
||||
Key: "node.kubernetes.io/unreachable",
|
||||
Operator: corev1.TolerationOpExists,
|
||||
Effect: corev1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &timeoutS,
|
||||
},
|
||||
}
|
||||
if err = kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns", "kube-system"); err != nil {
|
||||
return err
|
||||
}
|
||||
selectors := map[string]string{
|
||||
"node-role.kubernetes.io/control-plane": "",
|
||||
}
|
||||
if err = kubectl.AddNodeSelectorsToDeployment(ctx, selectors, "coredns", "kube-system"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
host, port, err := net.SplitHostPort(kubeAPIEndpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// configure pod network CIDR
|
||||
release.Values["ipv4NativeRoutingCIDR"] = subnetworkPodCIDR
|
||||
release.Values["strictModeCIDR"] = subnetworkPodCIDR
|
||||
release.Values["k8sServiceHost"] = host
|
||||
if port != "" {
|
||||
release.Values["k8sServicePort"] = port
|
||||
}
|
||||
|
||||
_, err = h.RunWithContext(ctx, release.Chart, release.Values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("helm install cilium: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *Client) installCiliumQEMU(ctx context.Context, release helm.Release, subnetworkPodCIDR, kubeAPIEndpoint string) error {
|
||||
// configure pod network CIDR
|
||||
release.Values["ipam"] = map[string]interface{}{
|
||||
"operator": map[string]interface{}{
|
||||
"clusterPoolIPv4PodCIDRList": []interface{}{
|
||||
subnetworkPodCIDR,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
release.Values["k8sServiceHost"] = kubeAPIEndpoint
|
||||
release.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
|
||||
|
||||
_, err := h.RunWithContext(ctx, release.Chart, release.Values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("helm install cilium: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// InstallKMS deploys the KMS deployment.
|
||||
func (h *Client) InstallKMS(ctx context.Context, release helm.Release, kmsConfig KMSConfig) error {
|
||||
h.Namespace = constants.HelmNamespace
|
||||
h.ReleaseName = release.ReleaseName
|
||||
h.Wait = release.Wait
|
||||
h.Timeout = timeout
|
||||
|
||||
release.Values["masterSecret"] = base64.StdEncoding.EncodeToString(kmsConfig.MasterSecret[:])
|
||||
release.Values["salt"] = base64.StdEncoding.EncodeToString(kmsConfig.Salt[:])
|
||||
|
||||
_, err := h.RunWithContext(ctx, release.Chart, release.Values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("helm install kms: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
13
bootstrapper/internal/helm/kms.go
Normal file
13
bootstrapper/internal/helm/kms.go
Normal file
@ -0,0 +1,13 @@
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package helm
|
||||
|
||||
// KMSConfig is the configuration needed to set up Constellation's key management service.
|
||||
type KMSConfig struct {
|
||||
MasterSecret []byte
|
||||
Salt []byte
|
||||
}
|
@ -15,7 +15,7 @@ import (
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/diskencryption"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/atls"
|
||||
"github.com/edgelesssys/constellation/v2/internal/attestation"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/vmtype"
|
||||
@ -132,13 +132,9 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro
|
||||
req.EnforceIdkeydigest,
|
||||
s.issuerWrapper.IDKeyDigest(),
|
||||
s.issuerWrapper.VMType() == vmtype.AzureCVM,
|
||||
resources.KMSConfig{
|
||||
MasterSecret: req.MasterSecret,
|
||||
Salt: req.Salt,
|
||||
KMSURI: req.KmsUri,
|
||||
StorageURI: req.StorageUri,
|
||||
KeyEncryptionKeyID: req.KeyEncryptionKeyId,
|
||||
UseExistingKEK: req.UseExistingKek,
|
||||
helm.KMSConfig{
|
||||
MasterSecret: req.MasterSecret,
|
||||
Salt: req.Salt,
|
||||
},
|
||||
sshProtoKeysToMap(req.SshUserKeys),
|
||||
req.HelmDeployments,
|
||||
@ -240,7 +236,7 @@ type ClusterInitializer interface {
|
||||
enforceIDKeyDigest bool,
|
||||
idKeyDigest []byte,
|
||||
azureCVM bool,
|
||||
kmsConfig resources.KMSConfig,
|
||||
kmsConfig helm.KMSConfig,
|
||||
sshUserKeys map[string]string,
|
||||
helmDeployments []byte,
|
||||
conformanceMode bool,
|
||||
|
@ -16,7 +16,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/initproto"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/crypto/testvector"
|
||||
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
@ -290,7 +290,7 @@ type stubClusterInitializer struct {
|
||||
|
||||
func (i *stubClusterInitializer) InitCluster(
|
||||
context.Context, string, string, []byte, []uint32, bool, []byte, bool,
|
||||
resources.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
|
||||
helm.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
|
||||
) ([]byte, error) {
|
||||
return i.initClusterKubeconfig, i.initClusterErr
|
||||
}
|
||||
|
@ -12,7 +12,6 @@ const (
|
||||
binDir = "/run/state/bin"
|
||||
kubeadmPath = "/run/state/bin/kubeadm"
|
||||
kubeletPath = "/run/state/bin/kubelet"
|
||||
kubectlPath = "/run/state/bin/kubectl"
|
||||
kubeletServiceEtcPath = "/etc/systemd/system/kubelet.service"
|
||||
kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
|
||||
kubeadmConfEtcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
|
||||
|
@ -10,7 +10,6 @@ import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
@ -32,7 +31,6 @@ import (
|
||||
kubeconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/internal/crypto"
|
||||
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions"
|
||||
@ -40,20 +38,14 @@ import (
|
||||
"github.com/spf13/afero"
|
||||
"go.uber.org/zap"
|
||||
"golang.org/x/text/transform"
|
||||
"helm.sh/helm/v3/pkg/action"
|
||||
"helm.sh/helm/v3/pkg/cli"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
)
|
||||
|
||||
const (
|
||||
// kubeConfig is the path to the Kubernetes admin config (used for authentication).
|
||||
kubeConfig = "/etc/kubernetes/admin.conf"
|
||||
// kubeletStartTimeout is the maximum time given to the kubelet service to (re)start.
|
||||
kubeletStartTimeout = 10 * time.Minute
|
||||
// crdTimeout is the maximum time given to the CRDs to be created.
|
||||
crdTimeout = 30 * time.Second
|
||||
// helmTimeout is the maximum time given to the helm client.
|
||||
helmTimeout = 5 * time.Minute
|
||||
)
|
||||
|
||||
// Client provides the functions to talk to the k8s API.
|
||||
@ -122,7 +114,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions
|
||||
return fmt.Errorf("installing kubeadm: %w", err)
|
||||
}
|
||||
if err := k.inst.Install(
|
||||
ctx, versionConf.KubectlURL, []string{kubectlPath}, executablePerm, false,
|
||||
ctx, versionConf.KubectlURL, []string{constants.KubectlPath}, executablePerm, false,
|
||||
); err != nil {
|
||||
return fmt.Errorf("installing kubectl: %w", err)
|
||||
}
|
||||
@ -235,19 +227,19 @@ func (k *KubernetesUtil) prepareControlPlaneForKonnectivity(ctx context.Context,
|
||||
return fmt.Errorf("generating konnectivity server certificate: %w", err)
|
||||
}
|
||||
|
||||
if out, err := exec.CommandContext(ctx, kubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server",
|
||||
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "config", "set-credentials", "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "system:konnectivity-server",
|
||||
"--client-certificate", "/etc/kubernetes/konnectivity.crt", "--client-key", "/etc/kubernetes/konnectivity.key", "--embed-certs=true").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-credentials: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint,
|
||||
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-cluster", "kubernetes", "--server", "https://"+loadBalancerEndpoint,
|
||||
"--certificate-authority", "/etc/kubernetes/pki/ca.crt", "--embed-certs=true").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-cluster: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes",
|
||||
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "set-context", "system:konnectivity-server@kubernetes",
|
||||
"--cluster", "kubernetes", "--user", "system:konnectivity-server").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig set-context: %w, %s", err, string(out))
|
||||
}
|
||||
if out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil {
|
||||
if out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", "/etc/kubernetes/konnectivity-server.conf", "config", "use-context", "system:konnectivity-server@kubernetes").CombinedOutput(); err != nil {
|
||||
return fmt.Errorf("konnectivity kubeconfig use-context: %w, %s", err, string(out))
|
||||
}
|
||||
// cleanup
|
||||
@ -264,33 +256,6 @@ func (k *KubernetesUtil) SetupKonnectivity(kubectl Client, konnectivityAgentsDae
|
||||
return kubectl.Apply(konnectivityAgentsDaemonSet, true)
|
||||
}
|
||||
|
||||
func (k *KubernetesUtil) SetupHelmDeployments(ctx context.Context, kubectl Client, helmDeployments []byte, in SetupPodNetworkInput, log *logger.Logger) error {
|
||||
var helmDeploy helm.Deployments
|
||||
if err := json.Unmarshal(helmDeployments, &helmDeploy); err != nil {
|
||||
return fmt.Errorf("unmarshalling helm deployments: %w", err)
|
||||
}
|
||||
settings := cli.New()
|
||||
settings.KubeConfig = kubeConfig
|
||||
|
||||
actionConfig := new(action.Configuration)
|
||||
if err := actionConfig.Init(settings.RESTClientGetter(), constants.HelmNamespace,
|
||||
"secret", log.Infof); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
helmClient := action.NewInstall(actionConfig)
|
||||
helmClient.Namespace = constants.HelmNamespace
|
||||
helmClient.ReleaseName = "cilium"
|
||||
helmClient.Wait = true
|
||||
helmClient.Timeout = helmTimeout
|
||||
|
||||
if err := k.deployCilium(ctx, in, helmClient, helmDeploy.Cilium, kubectl); err != nil {
|
||||
return fmt.Errorf("deploying cilium: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type SetupPodNetworkInput struct {
|
||||
CloudProvider string
|
||||
NodeName string
|
||||
@ -300,85 +265,6 @@ type SetupPodNetworkInput struct {
|
||||
LoadBalancerEndpoint string
|
||||
}
|
||||
|
||||
// deployCilium sets up the cilium pod network.
|
||||
func (k *KubernetesUtil) deployCilium(ctx context.Context, in SetupPodNetworkInput, helmClient *action.Install, ciliumDeployment helm.Deployment, kubectl Client) error {
|
||||
switch in.CloudProvider {
|
||||
case "gcp":
|
||||
return k.deployCiliumGCP(ctx, helmClient, kubectl, ciliumDeployment, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint)
|
||||
case "azure":
|
||||
return k.deployCiliumAzure(ctx, helmClient, ciliumDeployment, in.LoadBalancerEndpoint)
|
||||
case "qemu":
|
||||
return k.deployCiliumQEMU(ctx, helmClient, ciliumDeployment, in.SubnetworkPodCIDR, in.LoadBalancerEndpoint)
|
||||
default:
|
||||
return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider)
|
||||
}
|
||||
}
|
||||
|
||||
func (k *KubernetesUtil) deployCiliumAzure(ctx context.Context, helmClient *action.Install, ciliumDeployment helm.Deployment, kubeAPIEndpoint string) error {
|
||||
host := kubeAPIEndpoint
|
||||
ciliumDeployment.Values["k8sServiceHost"] = host
|
||||
ciliumDeployment.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
|
||||
|
||||
_, err := helmClient.RunWithContext(ctx, ciliumDeployment.Chart, ciliumDeployment.Values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("installing cilium: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *KubernetesUtil) deployCiliumGCP(ctx context.Context, helmClient *action.Install, kubectl Client, ciliumDeployment helm.Deployment, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIEndpoint string) error {
|
||||
out, err := exec.CommandContext(ctx, kubectlPath, "--kubeconfig", kubeConfig, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput()
|
||||
if err != nil {
|
||||
err = errors.New(string(out))
|
||||
return err
|
||||
}
|
||||
|
||||
timeoutS := int64(10)
|
||||
// allow coredns to run on uninitialized nodes (required by cloud-controller-manager)
|
||||
tolerations := []corev1.Toleration{
|
||||
{
|
||||
Key: "node.cloudprovider.kubernetes.io/uninitialized",
|
||||
Value: "true",
|
||||
Effect: corev1.TaintEffectNoSchedule,
|
||||
},
|
||||
{
|
||||
Key: "node.kubernetes.io/unreachable",
|
||||
Operator: corev1.TolerationOpExists,
|
||||
Effect: corev1.TaintEffectNoExecute,
|
||||
TolerationSeconds: &timeoutS,
|
||||
},
|
||||
}
|
||||
if err = kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns", "kube-system"); err != nil {
|
||||
return err
|
||||
}
|
||||
selectors := map[string]string{
|
||||
"node-role.kubernetes.io/control-plane": "",
|
||||
}
|
||||
if err = kubectl.AddNodeSelectorsToDeployment(ctx, selectors, "coredns", "kube-system"); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
host, port, err := net.SplitHostPort(kubeAPIEndpoint)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// configure pod network CIDR
|
||||
ciliumDeployment.Values["ipv4NativeRoutingCIDR"] = subnetworkPodCIDR
|
||||
ciliumDeployment.Values["strictModeCIDR"] = subnetworkPodCIDR
|
||||
ciliumDeployment.Values["k8sServiceHost"] = host
|
||||
if port != "" {
|
||||
ciliumDeployment.Values["k8sServicePort"] = port
|
||||
}
|
||||
|
||||
_, err = helmClient.RunWithContext(ctx, ciliumDeployment.Chart, ciliumDeployment.Values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("installing cilium: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// FixCilium fixes https://github.com/cilium/cilium/issues/19958 but instead of a rollout restart of
|
||||
// the cilium daemonset, it only restarts the local cilium pod.
|
||||
func (k *KubernetesUtil) FixCilium(log *logger.Logger) {
|
||||
@ -423,26 +309,6 @@ func (k *KubernetesUtil) FixCilium(log *logger.Logger) {
|
||||
}
|
||||
}
|
||||
|
||||
func (k *KubernetesUtil) deployCiliumQEMU(ctx context.Context, helmClient *action.Install, ciliumDeployment helm.Deployment, subnetworkPodCIDR, kubeAPIEndpoint string) error {
|
||||
// configure pod network CIDR
|
||||
ciliumDeployment.Values["ipam"] = map[string]interface{}{
|
||||
"operator": map[string]interface{}{
|
||||
"clusterPoolIPv4PodCIDRList": []interface{}{
|
||||
subnetworkPodCIDR,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
ciliumDeployment.Values["k8sServiceHost"] = kubeAPIEndpoint
|
||||
ciliumDeployment.Values["k8sServicePort"] = strconv.Itoa(constants.KubernetesPort)
|
||||
|
||||
_, err := helmClient.RunWithContext(ctx, ciliumDeployment.Chart, ciliumDeployment.Values)
|
||||
if err != nil {
|
||||
return fmt.Errorf("installing cilium: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetupAutoscaling deploys the k8s cluster autoscaler.
|
||||
func (k *KubernetesUtil) SetupAutoscaling(kubectl Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error {
|
||||
if err := kubectl.Apply(secrets, true); err != nil {
|
||||
@ -485,14 +351,6 @@ func (k *KubernetesUtil) SetupAccessManager(kubectl Client, accessManagerConfigu
|
||||
return kubectl.Apply(accessManagerConfiguration, true)
|
||||
}
|
||||
|
||||
// SetupKMS deploys the KMS deployment.
|
||||
func (k *KubernetesUtil) SetupKMS(kubectl Client, kmsConfiguration kubernetes.Marshaler) error {
|
||||
if err := kubectl.Apply(kmsConfiguration, true); err != nil {
|
||||
return fmt.Errorf("applying KMS configuration: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// SetupVerificationService deploys the verification service.
|
||||
func (k *KubernetesUtil) SetupVerificationService(kubectl Client, verificationServiceConfiguration kubernetes.Marshaler) error {
|
||||
return kubectl.Apply(verificationServiceConfiguration, true)
|
@ -1,259 +0,0 @@
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package resources
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions"
|
||||
apps "k8s.io/api/apps/v1"
|
||||
k8s "k8s.io/api/core/v1"
|
||||
rbac "k8s.io/api/rbac/v1"
|
||||
meta "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
)
|
||||
|
||||
const kmsNamespace = "kube-system"
|
||||
|
||||
type KMSDeployment struct {
|
||||
ServiceAccount k8s.ServiceAccount
|
||||
Service k8s.Service
|
||||
ClusterRole rbac.ClusterRole
|
||||
ClusterRoleBinding rbac.ClusterRoleBinding
|
||||
Deployment apps.DaemonSet
|
||||
MasterSecret k8s.Secret
|
||||
}
|
||||
|
||||
// KMSConfig is the configuration needed to set up Constellation's key management service.
|
||||
type KMSConfig struct {
|
||||
MasterSecret []byte
|
||||
Salt []byte
|
||||
KMSURI string
|
||||
StorageURI string
|
||||
KeyEncryptionKeyID string
|
||||
UseExistingKEK bool
|
||||
}
|
||||
|
||||
// NewKMSDeployment creates a new *kmsDeployment to use as the key management system inside Constellation.
|
||||
func NewKMSDeployment(csp string, config KMSConfig) *KMSDeployment {
|
||||
return &KMSDeployment{
|
||||
ServiceAccount: k8s.ServiceAccount{
|
||||
TypeMeta: meta.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "ServiceAccount",
|
||||
},
|
||||
ObjectMeta: meta.ObjectMeta{
|
||||
Name: "kms",
|
||||
Namespace: kmsNamespace,
|
||||
},
|
||||
},
|
||||
Service: k8s.Service{
|
||||
TypeMeta: meta.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "Service",
|
||||
},
|
||||
ObjectMeta: meta.ObjectMeta{
|
||||
Name: "kms",
|
||||
Namespace: kmsNamespace,
|
||||
},
|
||||
Spec: k8s.ServiceSpec{
|
||||
Type: k8s.ServiceTypeClusterIP,
|
||||
Ports: []k8s.ServicePort{
|
||||
{
|
||||
Name: "grpc",
|
||||
Protocol: k8s.ProtocolTCP,
|
||||
Port: constants.KMSPort,
|
||||
TargetPort: intstr.FromInt(constants.KMSPort),
|
||||
},
|
||||
},
|
||||
Selector: map[string]string{
|
||||
"k8s-app": "kms",
|
||||
},
|
||||
},
|
||||
},
|
||||
ClusterRole: rbac.ClusterRole{
|
||||
TypeMeta: meta.TypeMeta{
|
||||
APIVersion: "rbac.authorization.k8s.io/v1",
|
||||
Kind: "ClusterRole",
|
||||
},
|
||||
ObjectMeta: meta.ObjectMeta{
|
||||
Name: "kms",
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "kms",
|
||||
},
|
||||
},
|
||||
Rules: []rbac.PolicyRule{
|
||||
{
|
||||
APIGroups: []string{""},
|
||||
Resources: []string{"secrets"},
|
||||
Verbs: []string{"get"},
|
||||
},
|
||||
},
|
||||
},
|
||||
ClusterRoleBinding: rbac.ClusterRoleBinding{
|
||||
TypeMeta: meta.TypeMeta{
|
||||
APIVersion: "rbac.authorization.k8s.io/v1",
|
||||
Kind: "ClusterRoleBinding",
|
||||
},
|
||||
ObjectMeta: meta.ObjectMeta{
|
||||
Name: "kms",
|
||||
},
|
||||
RoleRef: rbac.RoleRef{
|
||||
APIGroup: "rbac.authorization.k8s.io",
|
||||
Kind: "ClusterRole",
|
||||
Name: "kms",
|
||||
},
|
||||
Subjects: []rbac.Subject{
|
||||
{
|
||||
Kind: "ServiceAccount",
|
||||
Name: "kms",
|
||||
Namespace: kmsNamespace,
|
||||
},
|
||||
},
|
||||
},
|
||||
Deployment: apps.DaemonSet{
|
||||
TypeMeta: meta.TypeMeta{
|
||||
APIVersion: "apps/v1",
|
||||
Kind: "DaemonSet",
|
||||
},
|
||||
ObjectMeta: meta.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "kms",
|
||||
"component": "kms",
|
||||
"kubernetes.io/cluster-service": "true",
|
||||
},
|
||||
Name: "kms",
|
||||
Namespace: kmsNamespace,
|
||||
},
|
||||
Spec: apps.DaemonSetSpec{
|
||||
Selector: &meta.LabelSelector{
|
||||
MatchLabels: map[string]string{
|
||||
"k8s-app": "kms",
|
||||
},
|
||||
},
|
||||
Template: k8s.PodTemplateSpec{
|
||||
ObjectMeta: meta.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
"k8s-app": "kms",
|
||||
},
|
||||
},
|
||||
Spec: k8s.PodSpec{
|
||||
PriorityClassName: "system-cluster-critical",
|
||||
Tolerations: []k8s.Toleration{
|
||||
{
|
||||
Key: "CriticalAddonsOnly",
|
||||
Operator: k8s.TolerationOpExists,
|
||||
},
|
||||
{
|
||||
Key: "node-role.kubernetes.io/master",
|
||||
Operator: k8s.TolerationOpEqual,
|
||||
Value: "true",
|
||||
Effect: k8s.TaintEffectNoSchedule,
|
||||
},
|
||||
{
|
||||
Key: "node-role.kubernetes.io/control-plane",
|
||||
Operator: k8s.TolerationOpExists,
|
||||
Effect: k8s.TaintEffectNoSchedule,
|
||||
},
|
||||
{
|
||||
Operator: k8s.TolerationOpExists,
|
||||
Effect: k8s.TaintEffectNoExecute,
|
||||
},
|
||||
{
|
||||
Operator: k8s.TolerationOpExists,
|
||||
Effect: k8s.TaintEffectNoSchedule,
|
||||
},
|
||||
},
|
||||
// Only run on control plane nodes
|
||||
NodeSelector: map[string]string{
|
||||
"node-role.kubernetes.io/control-plane": "",
|
||||
},
|
||||
Volumes: []k8s.Volume{
|
||||
{
|
||||
Name: "config",
|
||||
VolumeSource: k8s.VolumeSource{
|
||||
Projected: &k8s.ProjectedVolumeSource{
|
||||
Sources: []k8s.VolumeProjection{
|
||||
{
|
||||
ConfigMap: &k8s.ConfigMapProjection{
|
||||
LocalObjectReference: k8s.LocalObjectReference{
|
||||
Name: "join-config",
|
||||
},
|
||||
Items: []k8s.KeyToPath{
|
||||
{
|
||||
Key: constants.MeasurementsFilename,
|
||||
Path: constants.MeasurementsFilename,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
Secret: &k8s.SecretProjection{
|
||||
LocalObjectReference: k8s.LocalObjectReference{
|
||||
Name: constants.ConstellationMasterSecretStoreName,
|
||||
},
|
||||
Items: []k8s.KeyToPath{
|
||||
{
|
||||
Key: constants.ConstellationMasterSecretKey,
|
||||
Path: constants.ConstellationMasterSecretKey,
|
||||
},
|
||||
{
|
||||
Key: constants.ConstellationMasterSecretSalt,
|
||||
Path: constants.ConstellationMasterSecretSalt,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
ServiceAccountName: "kms",
|
||||
Containers: []k8s.Container{
|
||||
{
|
||||
Name: "kms",
|
||||
Image: versions.KmsImage,
|
||||
Args: []string{
|
||||
fmt.Sprintf("--port=%d", constants.KMSPort),
|
||||
},
|
||||
VolumeMounts: []k8s.VolumeMount{
|
||||
{
|
||||
Name: "config",
|
||||
ReadOnly: true,
|
||||
MountPath: constants.ServiceBasePath,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
MasterSecret: k8s.Secret{
|
||||
TypeMeta: meta.TypeMeta{
|
||||
APIVersion: "v1",
|
||||
Kind: "Secret",
|
||||
},
|
||||
ObjectMeta: meta.ObjectMeta{
|
||||
Name: constants.ConstellationMasterSecretStoreName,
|
||||
Namespace: kmsNamespace,
|
||||
},
|
||||
Data: map[string][]byte{
|
||||
constants.ConstellationMasterSecretKey: config.MasterSecret,
|
||||
constants.ConstellationMasterSecretSalt: config.Salt,
|
||||
},
|
||||
Type: "Opaque",
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func (c *KMSDeployment) Marshal() ([]byte, error) {
|
||||
return kubernetes.MarshalK8SResources(c)
|
||||
}
|
@ -1,28 +0,0 @@
|
||||
/*
|
||||
Copyright (c) Edgeless Systems GmbH
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only
|
||||
*/
|
||||
|
||||
package resources
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestKMSMarshalUnmarshal(t *testing.T) {
|
||||
require := require.New(t)
|
||||
assert := assert.New(t)
|
||||
|
||||
kmsDepl := NewKMSDeployment("test", KMSConfig{MasterSecret: []byte{0x0, 0x1, 0x2}, Salt: []byte{0x3, 0x4, 0x5}})
|
||||
data, err := kmsDepl.Marshal()
|
||||
require.NoError(err)
|
||||
|
||||
var recreated KMSDeployment
|
||||
require.NoError(kubernetes.UnmarshalK8SResources(data, &recreated))
|
||||
assert.Equal(kmsDepl, &recreated)
|
||||
}
|
@ -10,7 +10,9 @@ import (
|
||||
"context"
|
||||
"net"
|
||||
|
||||
helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
|
||||
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/edgelesssys/constellation/v2/internal/role"
|
||||
@ -21,19 +23,25 @@ type clusterUtil interface {
|
||||
InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
|
||||
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error
|
||||
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
|
||||
SetupHelmDeployments(ctx context.Context, client k8sapi.Client, helmDeployments []byte, in k8sapi.SetupPodNetworkInput, log *logger.Logger) error
|
||||
SetupAccessManager(kubectl k8sapi.Client, sshUsers kubernetes.Marshaler) error
|
||||
SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error
|
||||
SetupJoinService(kubectl k8sapi.Client, joinServiceConfiguration kubernetes.Marshaler) error
|
||||
SetupCloudControllerManager(kubectl k8sapi.Client, cloudControllerManagerConfiguration kubernetes.Marshaler, configMaps kubernetes.Marshaler, secrets kubernetes.Marshaler) error
|
||||
SetupCloudNodeManager(kubectl k8sapi.Client, cloudNodeManagerConfiguration kubernetes.Marshaler) error
|
||||
SetupKonnectivity(kubectl k8sapi.Client, konnectivityAgentsDaemonSet kubernetes.Marshaler) error
|
||||
SetupKMS(kubectl k8sapi.Client, kmsConfiguration kubernetes.Marshaler) error
|
||||
SetupVerificationService(kubectl k8sapi.Client, verificationServiceConfiguration kubernetes.Marshaler) error
|
||||
SetupGCPGuestAgent(kubectl k8sapi.Client, gcpGuestAgentConfiguration kubernetes.Marshaler) error
|
||||
SetupOperatorLifecycleManager(ctx context.Context, kubectl k8sapi.Client, olmCRDs, olmConfiguration kubernetes.Marshaler, crdNames []string) error
|
||||
SetupNodeMaintenanceOperator(kubectl k8sapi.Client, nodeMaintenanceOperatorConfiguration kubernetes.Marshaler) error
|
||||
SetupNodeOperator(ctx context.Context, kubectl k8sapi.Client, nodeOperatorConfiguration kubernetes.Marshaler) error
|
||||
StartKubelet() error
|
||||
FixCilium(log *logger.Logger)
|
||||
StartKubelet() error
|
||||
}
|
||||
|
||||
// HelmUtil bundles functions related to microservice deployment. Only microservices that can be deployed purely via Helm are deployed with this interface.
|
||||
// Currently only a subset of microservices is deployed via Helm.
|
||||
// Naming is inspired by Helm.
|
||||
type HelmUtil interface {
|
||||
InstallCilium(context.Context, k8sapi.Client, helm.Release, k8sapi.SetupPodNetworkInput) error
|
||||
InstallKMS(context.Context, helm.Release, helmClient.KMSConfig) error
|
||||
}
|
||||
|
@ -15,10 +15,12 @@ import (
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/edgelesssys/constellation/v2/internal/role"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions"
|
||||
@ -44,6 +46,7 @@ type configurationProvider interface {
|
||||
type KubeWrapper struct {
|
||||
cloudProvider string
|
||||
clusterUtil clusterUtil
|
||||
helmUtil HelmUtil
|
||||
configProvider configurationProvider
|
||||
client k8sapi.Client
|
||||
kubeconfigReader configReader
|
||||
@ -57,11 +60,12 @@ type KubeWrapper struct {
|
||||
|
||||
// New creates a new KubeWrapper with real values.
|
||||
func New(cloudProvider string, clusterUtil clusterUtil, configProvider configurationProvider, client k8sapi.Client, cloudControllerManager CloudControllerManager,
|
||||
cloudNodeManager CloudNodeManager, clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte,
|
||||
cloudNodeManager CloudNodeManager, clusterAutoscaler ClusterAutoscaler, providerMetadata ProviderMetadata, initialMeasurementsJSON []byte, helmUtil HelmUtil,
|
||||
) *KubeWrapper {
|
||||
return &KubeWrapper{
|
||||
cloudProvider: cloudProvider,
|
||||
clusterUtil: clusterUtil,
|
||||
helmUtil: helmUtil,
|
||||
configProvider: configProvider,
|
||||
client: client,
|
||||
kubeconfigReader: &KubeconfigReader{fs: afero.Afero{Fs: afero.NewOsFs()}},
|
||||
@ -77,8 +81,8 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
|
||||
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
|
||||
func (k *KubeWrapper) InitCluster(
|
||||
ctx context.Context, cloudServiceAccountURI, versionString string, measurementSalt []byte, enforcedPCRs []uint32,
|
||||
enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig resources.KMSConfig, sshUsers map[string]string,
|
||||
helmDeployments []byte, conformanceMode bool, log *logger.Logger,
|
||||
enforceIDKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig helmClient.KMSConfig, sshUsers map[string]string,
|
||||
helmReleasesRaw []byte, conformanceMode bool, log *logger.Logger,
|
||||
) ([]byte, error) {
|
||||
k8sVersion, err := versions.NewValidK8sVersion(versionString)
|
||||
if err != nil {
|
||||
@ -174,8 +178,14 @@ func (k *KubeWrapper) InitCluster(
|
||||
SubnetworkPodCIDR: subnetworkPodCIDR,
|
||||
LoadBalancerEndpoint: controlPlaneEndpoint,
|
||||
}
|
||||
if err = k.clusterUtil.SetupHelmDeployments(ctx, k.client, helmDeployments, setupPodNetworkInput, log); err != nil {
|
||||
return nil, fmt.Errorf("setting up pod network: %w", err)
|
||||
|
||||
var helmReleases helm.Releases
|
||||
if err := json.Unmarshal(helmReleasesRaw, &helmReleases); err != nil {
|
||||
return nil, fmt.Errorf("unmarshalling helm releases: %w", err)
|
||||
}
|
||||
|
||||
if err = k.helmUtil.InstallCilium(ctx, k.client, helmReleases.Cilium, setupPodNetworkInput); err != nil {
|
||||
return nil, fmt.Errorf("installing pod network: %w", err)
|
||||
}
|
||||
|
||||
var controlPlaneIP string
|
||||
@ -191,9 +201,8 @@ func (k *KubeWrapper) InitCluster(
|
||||
return nil, fmt.Errorf("setting up konnectivity: %w", err)
|
||||
}
|
||||
|
||||
kms := resources.NewKMSDeployment(k.cloudProvider, kmsConfig)
|
||||
if err = k.clusterUtil.SetupKMS(k.client, kms); err != nil {
|
||||
return nil, fmt.Errorf("setting up kms: %w", err)
|
||||
if err = k.helmUtil.InstallKMS(ctx, helmReleases.KMS, kmsConfig); err != nil {
|
||||
return nil, fmt.Errorf("installing kms: %w", err)
|
||||
}
|
||||
|
||||
if err := k.setupInternalConfigMap(ctx, strconv.FormatBool(azureCVM)); err != nil {
|
||||
|
@ -14,10 +14,11 @@ import (
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
helmClient "github.com/edgelesssys/constellation/v2/bootstrapper/internal/helm"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
|
||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/resources"
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/metadata"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/kubernetes"
|
||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||
"github.com/edgelesssys/constellation/v2/internal/role"
|
||||
@ -47,6 +48,7 @@ func TestInitCluster(t *testing.T) {
|
||||
|
||||
testCases := map[string]struct {
|
||||
clusterUtil stubClusterUtil
|
||||
helmUtil stubHelmClient
|
||||
kubectl stubKubectl
|
||||
providerMetadata ProviderMetadata
|
||||
CloudControllerManager CloudControllerManager
|
||||
@ -178,8 +180,9 @@ func TestInitCluster(t *testing.T) {
|
||||
wantErr: true,
|
||||
k8sVersion: versions.Default,
|
||||
},
|
||||
"kubeadm init fails when deploying helm charts": {
|
||||
clusterUtil: stubClusterUtil{setupHelmDeploymentsErr: someErr},
|
||||
"kubeadm init fails when deploying cilium": {
|
||||
clusterUtil: stubClusterUtil{},
|
||||
helmUtil: stubHelmClient{ciliumError: someErr},
|
||||
kubeconfigReader: &stubKubeconfigReader{
|
||||
Kubeconfig: []byte("someKubeconfig"),
|
||||
},
|
||||
@ -251,7 +254,8 @@ func TestInitCluster(t *testing.T) {
|
||||
k8sVersion: versions.Default,
|
||||
},
|
||||
"kubeadm init fails when setting up the kms": {
|
||||
clusterUtil: stubClusterUtil{setupKMSError: someErr},
|
||||
clusterUtil: stubClusterUtil{},
|
||||
helmUtil: stubHelmClient{kmsError: someErr},
|
||||
kubeconfigReader: &stubKubeconfigReader{
|
||||
Kubeconfig: []byte("someKubeconfig"),
|
||||
},
|
||||
@ -307,6 +311,7 @@ func TestInitCluster(t *testing.T) {
|
||||
|
||||
kube := KubeWrapper{
|
||||
clusterUtil: &tc.clusterUtil,
|
||||
helmUtil: &tc.helmUtil,
|
||||
providerMetadata: tc.providerMetadata,
|
||||
cloudControllerManager: tc.CloudControllerManager,
|
||||
cloudNodeManager: tc.CloudNodeManager,
|
||||
@ -319,7 +324,7 @@ func TestInitCluster(t *testing.T) {
|
||||
|
||||
_, err := kube.InitCluster(
|
||||
context.Background(), serviceAccountURI, string(tc.k8sVersion),
|
||||
nil, nil, false, nil, true, resources.KMSConfig{MasterSecret: masterSecret}, nil, nil, false, logger.NewTest(t),
|
||||
nil, nil, false, nil, true, helmClient.KMSConfig{MasterSecret: masterSecret}, nil, []byte("{}"), false, logger.NewTest(t),
|
||||
)
|
||||
|
||||
if tc.wantErr {
|
||||
@ -528,13 +533,11 @@ func TestK8sCompliantHostname(t *testing.T) {
|
||||
type stubClusterUtil struct {
|
||||
installComponentsErr error
|
||||
initClusterErr error
|
||||
setupHelmDeploymentsErr error
|
||||
setupAutoscalingError error
|
||||
setupJoinServiceError error
|
||||
setupCloudControllerManagerError error
|
||||
setupCloudNodeManagerError error
|
||||
setupKonnectivityError error
|
||||
setupKMSError error
|
||||
setupAccessManagerError error
|
||||
setupVerificationServiceErr error
|
||||
setupGCPGuestAgentErr error
|
||||
@ -561,10 +564,6 @@ func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, no
|
||||
return s.initClusterErr
|
||||
}
|
||||
|
||||
func (s *stubClusterUtil) SetupHelmDeployments(context.Context, k8sapi.Client, []byte, k8sapi.SetupPodNetworkInput, *logger.Logger) error {
|
||||
return s.setupHelmDeploymentsErr
|
||||
}
|
||||
|
||||
func (s *stubClusterUtil) SetupAutoscaling(kubectl k8sapi.Client, clusterAutoscalerConfiguration kubernetes.Marshaler, secrets kubernetes.Marshaler) error {
|
||||
return s.setupAutoscalingError
|
||||
}
|
||||
@ -581,10 +580,6 @@ func (s *stubClusterUtil) SetupCloudControllerManager(kubectl k8sapi.Client, clo
|
||||
return s.setupCloudControllerManagerError
|
||||
}
|
||||
|
||||
func (s *stubClusterUtil) SetupKMS(kubectl k8sapi.Client, kmsDeployment kubernetes.Marshaler) error {
|
||||
return s.setupKMSError
|
||||
}
|
||||
|
||||
func (s *stubClusterUtil) SetupAccessManager(kubectl k8sapi.Client, accessManagerConfiguration kubernetes.Marshaler) error {
|
||||
return s.setupAccessManagerError
|
||||
}
|
||||
@ -685,3 +680,16 @@ type stubKubeconfigReader struct {
|
||||
func (s *stubKubeconfigReader) ReadKubeconfig() ([]byte, error) {
|
||||
return s.Kubeconfig, s.ReadErr
|
||||
}
|
||||
|
||||
type stubHelmClient struct {
|
||||
ciliumError error
|
||||
kmsError error
|
||||
}
|
||||
|
||||
func (s *stubHelmClient) InstallCilium(ctx context.Context, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
|
||||
return s.ciliumError
|
||||
}
|
||||
|
||||
func (s *stubHelmClient) InstallKMS(ctx context.Context, release helm.Release, kmsConfig helmClient.KMSConfig) error {
|
||||
return s.kmsError
|
||||
}
|
||||
|
@ -16,7 +16,9 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
|
||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
||||
"github.com/edgelesssys/constellation/v2/internal/versions"
|
||||
"github.com/pkg/errors"
|
||||
"helm.sh/helm/pkg/ignore"
|
||||
"helm.sh/helm/v3/pkg/chart"
|
||||
@ -26,28 +28,33 @@ import (
|
||||
// Run `go generate` to deterministically create the patched Helm deployment for cilium
|
||||
//go:generate ./generateCilium.sh
|
||||
|
||||
//go:embed all:charts/cilium/*
|
||||
//go:embed all:charts/*
|
||||
var HelmFS embed.FS
|
||||
|
||||
type ChartLoader struct{}
|
||||
|
||||
func (i *ChartLoader) Load(csp cloudprovider.Provider, conformanceMode bool) ([]byte, error) {
|
||||
ciliumDeployment, err := i.loadCilium(csp, conformanceMode)
|
||||
ciliumRelease, err := i.loadCilium(csp, conformanceMode)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
deployments := helm.Deployments{Cilium: ciliumDeployment}
|
||||
depl, err := json.Marshal(deployments)
|
||||
|
||||
kmsRelease, err := i.loadKMS()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return depl, nil
|
||||
releases := helm.Releases{Cilium: ciliumRelease, KMS: kmsRelease}
|
||||
rel, err := json.Marshal(releases)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return rel, nil
|
||||
}
|
||||
|
||||
func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Deployment, error) {
|
||||
func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode bool) (helm.Release, error) {
|
||||
chart, err := loadChartsDir(HelmFS, "charts/cilium")
|
||||
if err != nil {
|
||||
return helm.Deployment{}, err
|
||||
return helm.Release{}, err
|
||||
}
|
||||
var ciliumVals map[string]interface{}
|
||||
switch csp {
|
||||
@ -58,7 +65,7 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
|
||||
case cloudprovider.QEMU:
|
||||
ciliumVals = qemuVals
|
||||
default:
|
||||
return helm.Deployment{}, fmt.Errorf("unknown csp: %s", csp)
|
||||
return helm.Release{}, fmt.Errorf("unknown csp: %s", csp)
|
||||
}
|
||||
if conformanceMode {
|
||||
ciliumVals["kubeProxyReplacementHealthzBindAddr"] = ""
|
||||
@ -70,7 +77,27 @@ func (i *ChartLoader) loadCilium(csp cloudprovider.Provider, conformanceMode boo
|
||||
|
||||
}
|
||||
|
||||
return helm.Deployment{Chart: chart, Values: ciliumVals}, nil
|
||||
return helm.Release{Chart: chart, Values: ciliumVals, ReleaseName: "cilium", Wait: true}, nil
|
||||
}
|
||||
|
||||
func (i *ChartLoader) loadKMS() (helm.Release, error) {
|
||||
chart, err := loadChartsDir(HelmFS, "charts/edgeless/kms")
|
||||
if err != nil {
|
||||
return helm.Release{}, err
|
||||
}
|
||||
kmsVals := map[string]interface{}{
|
||||
"namespace": constants.ConstellationNamespace,
|
||||
"kmsPort": constants.KMSPort,
|
||||
"joinConfigCMName": constants.JoinConfigMap,
|
||||
"serviceBasePath": constants.ServiceBasePath,
|
||||
"kmsImage": versions.KmsImage,
|
||||
"masterSecretName": constants.ConstellationMasterSecretStoreName,
|
||||
"masterSecretKeyName": constants.ConstellationMasterSecretKey,
|
||||
"saltKeyName": constants.ConstellationSaltKey,
|
||||
"measurementsFilename": constants.MeasurementsFilename,
|
||||
}
|
||||
|
||||
return helm.Release{Chart: chart, Values: kmsVals, ReleaseName: "kms", Wait: true}, nil
|
||||
}
|
||||
|
||||
// taken from loader.LoadDir from the helm go module
|
||||
|
@ -23,10 +23,10 @@ const (
|
||||
ConstellationNameLength = 37
|
||||
// ConstellationMasterSecretStoreName is the name for the Constellation secrets in Kubernetes.
|
||||
ConstellationMasterSecretStoreName = "constellation-mastersecret"
|
||||
// ConstellationMasterSecretKey is the name of the key for master secret in the master secret store secret.
|
||||
// ConstellationMasterSecretKey is the name of the key for the master secret in the master secret kubernetes secret.
|
||||
ConstellationMasterSecretKey = "mastersecret"
|
||||
// ConstellationMasterSecretSalt is the name of the key for salt in the master secret store secret.
|
||||
ConstellationMasterSecretSalt = "salt"
|
||||
// ConstellationSaltKey is the name of the key for the salt in the master secret kubernetes secret.
|
||||
ConstellationSaltKey = "salt"
|
||||
|
||||
//
|
||||
// Ports.
|
||||
@ -67,6 +67,7 @@ const (
|
||||
WGQuickConfigFilename = "wg0.conf"
|
||||
CoreOSAdminConfFilename = "/etc/kubernetes/admin.conf"
|
||||
KubeadmCertificateDir = "/etc/kubernetes/pki"
|
||||
KubectlPath = "/run/state/bin/kubectl"
|
||||
|
||||
//
|
||||
// Filenames for Constellation's micro services.
|
||||
|
@ -8,11 +8,15 @@ package helm
|
||||
|
||||
import "helm.sh/helm/v3/pkg/chart"
|
||||
|
||||
type Deployment struct {
|
||||
Chart *chart.Chart
|
||||
Values map[string]interface{}
|
||||
// Release bundles all information necessary to create a helm release.
|
||||
type Release struct {
|
||||
Chart *chart.Chart
|
||||
Values map[string]interface{}
|
||||
ReleaseName string
|
||||
Wait bool
|
||||
}
|
||||
|
||||
type Deployments struct {
|
||||
Cilium Deployment
|
||||
type Releases struct {
|
||||
Cilium Release
|
||||
KMS Release
|
||||
}
|
||||
|
@ -28,7 +28,7 @@ import (
|
||||
func main() {
|
||||
port := flag.String("port", strconv.Itoa(constants.KMSPort), "Port gRPC server listens on")
|
||||
masterSecretPath := flag.String("master-secret", filepath.Join(constants.ServiceBasePath, constants.ConstellationMasterSecretKey), "Path to the Constellation master secret")
|
||||
saltPath := flag.String("salt", filepath.Join(constants.ServiceBasePath, constants.ConstellationMasterSecretSalt), "Path to the Constellation salt")
|
||||
saltPath := flag.String("salt", filepath.Join(constants.ServiceBasePath, constants.ConstellationSaltKey), "Path to the Constellation salt")
|
||||
verbosity := flag.Int("v", 0, logger.CmdLineVerbosityDescription)
|
||||
|
||||
flag.Parse()
|
||||
|
Loading…
x
Reference in New Issue
Block a user