Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2025-09-19 04:24:39 -04:00
Code Issues Projects Releases Packages Wiki Activity

aws: use new LB controller to fix SecurityGroup cleanup on K8s service deletion (#2090)

Browse source 
* add current chart

add current helm chart

* disable service controller for aws ccm

* add new iam roles

* doc AWS internet LB + add to LB test

* pass clusterName to helm for AWS LB

* fix update-aws-lb chart to also include .helmignore

* move chart outside services

* working state

* add subnet tags for AWS subnet discovery

* fix .helmignore load rule with file in subdirectory

* upgrade iam profile

* revert new loader impl since cilium is not correctly loaded

* install chart if not already present during `upgrade apply`

* cleanup PR + fix build + add todos

cleanup PR + add todos

* shared helm pkg for cli install and bootstrapper

* add link to eks docs

* refactor iamMigrationCmd

* delete unused helm.symwallk

* move iammigrate to upgrade pkg

* fixup! delete unused helm.symwallk

* add to upgradecheck

* remove nodeSelector from go code (Otto)

* update iam docs and sort permission + remove duplicate roles

* fix bug in `upgrade check`

* better upgrade check output when svc version upgrade not possible

* pr feedback

* remove force flag in upgrade_test

* use upgrader.GetUpgradeID instead of extra type

* remove todos + fix check

* update doc lb (leo)

* remove bootstrapper helm package

* Update cli/internal/cmd/upgradecheck.go

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* final nits

* add docs for e2e upgrade test setup

* Apply suggestions from code review

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* Update cli/internal/helm/loader.go

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* Update cli/internal/cmd/tfmigrationclient.go

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

* fix daniel review

* link to the iam permissions instead of manually updating them (agreed with leo)

* disable iam upgrade in upgrade apply

---------

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>
Co-authored-by: Malte Poll
This commit is contained in:
Adrian Stobbe 2023-07-24 10:30:53 +02:00 • committed by GitHub
parent 8da6a23aa5
commit a87b7894db
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
67 changed files with 3018 additions and 451 deletions
Download patch file Download diff file Expand all files Collapse all files

1
bootstrapper/internal/kubernetes/BUILD.bazel
View file

@ -4,6 +4,7 @@ load("//bazel/go:go_test.bzl", "go_test")
go_library(
name = "kubernetes",
srcs = [
"ciliuminstallation.go",
"cloud_provider.go",
"k8sutil.go",
"kubernetes.go",

81
bootstrapper/internal/kubernetes/ciliuminstallation.go Normal file
View file

@ -0,0 +1,81 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
// Package kubernetes provides functionality to bootstrap a Kubernetes cluster, or join an exiting one.
package kubernetes
import (
"context"
"errors"
"fmt"
"os/exec"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
corev1 "k8s.io/api/core/v1"
)
// installCilium sets up the cilium pod network.
func installCilium(ctx context.Context, helmInstaller helmClient, kubectl k8sapi.Client, release helm.Release, in k8sapi.SetupPodNetworkInput) error {
timeoutS := int64(10)
// allow coredns to run on uninitialized nodes (required by cloud-controller-manager)
tolerations := []corev1.Toleration{
{
Key: "node.cloudprovider.kubernetes.io/uninitialized",
Value: "true",
Effect: corev1.TaintEffectNoSchedule,
},
{
Key: "node.kubernetes.io/unreachable",
Operator: corev1.TolerationOpExists,
Effect: corev1.TaintEffectNoExecute,
TolerationSeconds: &timeoutS,
},
}
if err := kubectl.AddTolerationsToDeployment(ctx, tolerations, "coredns", "kube-system"); err != nil {
return fmt.Errorf("failed to add tolerations to coredns deployment: %w", err)
}
if err := kubectl.EnforceCoreDNSSpread(ctx); err != nil {
return fmt.Errorf("failed to enforce CoreDNS spread: %w", err)
}
switch in.CloudProvider {
case "aws", "azure", "openstack", "qemu":
return installCiliumGeneric(ctx, helmInstaller, release, in.LoadBalancerHost, in.LoadBalancerPort)
case "gcp":
return installCiliumGCP(ctx, helmInstaller, release, in.NodeName, in.FirstNodePodCIDR, in.SubnetworkPodCIDR, in.LoadBalancerHost, in.LoadBalancerPort)
default:
return fmt.Errorf("unsupported cloud provider %q", in.CloudProvider)
}
}
// installCiliumGeneric installs cilium with the given load balancer endpoint.
// This is used for cloud providers that do not require special server-side configuration.
// Currently this is AWS, Azure, and QEMU.
func installCiliumGeneric(ctx context.Context, helmInstaller helmClient, release helm.Release, kubeAPIHost, kubeAPIPort string) error {
if release.Values != nil {
release.Values["k8sServiceHost"] = kubeAPIHost
release.Values["k8sServicePort"] = kubeAPIPort
}
return helmInstaller.InstallChart(ctx, release)
}
func installCiliumGCP(ctx context.Context, helmInstaller helmClient, release helm.Release, nodeName, nodePodCIDR, subnetworkPodCIDR, kubeAPIHost, kubeAPIPort string) error {
out, err := exec.CommandContext(ctx, constants.KubectlPath, "--kubeconfig", constants.ControlPlaneAdminConfFilename, "patch", "node", nodeName, "-p", "{\"spec\":{\"podCIDR\": \""+nodePodCIDR+"\"}}").CombinedOutput()
if err != nil {
err = errors.New(string(out))
return err
}
// configure pod network CIDR
release.Values["ipv4NativeRoutingCIDR"] = subnetworkPodCIDR
release.Values["strictModeCIDR"] = subnetworkPodCIDR
release.Values["k8sServiceHost"] = kubeAPIHost
release.Values["k8sServicePort"] = kubeAPIPort
return helmInstaller.InstallChart(ctx, release)
}

5
bootstrapper/internal/kubernetes/k8sutil.go
View file

@ -10,7 +10,6 @@ import (
"context"
"net"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
@ -29,6 +28,6 @@ type clusterUtil interface {
// helmClient bundles functions related to microservice deployment.
// Only microservices that can be deployed purely via Helm are deployed with this interface.
type helmClient interface {
InstallCilium(context.Context, k8sapi.Client, helm.Release, k8sapi.SetupPodNetworkInput) error
InstallChart(ctx context.Context, release helm.Release, extraVals map[string]any) error
InstallChart(context.Context, helm.Release) error
InstallChartWithValues(ctx context.Context, release helm.Release, extraValues map[string]any) error
}

28
bootstrapper/internal/kubernetes/kubernetes.go
View file

@ -18,11 +18,6 @@ import (
"strings"
"time"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubewaiter"
"github.com/edgelesssys/constellation/v2/internal/cloud/cloudprovider"
@ -34,6 +29,10 @@ import (
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/role"
"github.com/edgelesssys/constellation/v2/internal/versions/components"
"go.uber.org/zap"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubeadm "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm/v1beta3"
)
var validHostnameRegex = regexp.MustCompile(`^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$`)
@ -193,7 +192,7 @@ func (k *KubeWrapper) InitCluster(
}
log.Infof("Installing Cilium")
if err = k.helmClient.InstallCilium(ctx, k.client, helmReleases.Cilium, setupPodNetworkInput); err != nil {
if err = installCilium(ctx, k.helmClient, k.client, helmReleases.Cilium, setupPodNetworkInput); err != nil {
return nil, fmt.Errorf("installing pod network: %w", err)
}
@ -221,7 +220,7 @@ func (k *KubeWrapper) InitCluster(
cloudServiceAccountURI: cloudServiceAccountURI,
loadBalancerIP: controlPlaneHost,
}
extraVals, err := k.setupExtraVals(ctx, serviceConfig)
constellationVals, err := k.setupExtraVals(ctx, serviceConfig)
if err != nil {
return nil, fmt.Errorf("setting up extraVals: %w", err)
}
@ -232,14 +231,14 @@ func (k *KubeWrapper) InitCluster(
}
log.Infof("Installing Constellation microservices")
if err = k.helmClient.InstallChart(ctx, helmReleases.ConstellationServices, extraVals); err != nil {
if err = k.helmClient.InstallChartWithValues(ctx, helmReleases.ConstellationServices, constellationVals); err != nil {
return nil, fmt.Errorf("installing constellation-services: %w", err)
}
// cert-manager provides CRDs used by other deployments,
// so it should be installed as early as possible, but after the services cert-manager depends on.
log.Infof("Installing cert-manager")
if err = k.helmClient.InstallChart(ctx, helmReleases.CertManager, nil); err != nil {
if err = k.helmClient.InstallChart(ctx, helmReleases.CertManager); err != nil {
return nil, fmt.Errorf("installing cert-manager: %w", err)
}
@ -260,11 +259,18 @@ func (k *KubeWrapper) InitCluster(
}
log.Infof("Installing CSI deployments")
if err := k.helmClient.InstallChart(ctx, *helmReleases.CSI, csiVals); err != nil {
if err := k.helmClient.InstallChartWithValues(ctx, *helmReleases.CSI, csiVals); err != nil {
return nil, fmt.Errorf("installing CSI snapshot CRDs: %w", err)
}
}
if helmReleases.AWSLoadBalancerController != nil {
log.Infof("Installing AWS Load Balancer Controller")
if err = k.helmClient.InstallChart(ctx, *helmReleases.AWSLoadBalancerController); err != nil {
return nil, fmt.Errorf("installing AWS Load Balancer Controller: %w", err)
}
}
operatorVals, err := k.setupOperatorVals(ctx)
if err != nil {
return nil, fmt.Errorf("setting up operator vals: %w", err)
@ -273,7 +279,7 @@ func (k *KubeWrapper) InitCluster(
// Constellation operators require CRDs from cert-manager.
// They must be installed after it.
log.Infof("Installing operators")
if err = k.helmClient.InstallChart(ctx, helmReleases.Operators, operatorVals); err != nil {
if err = k.helmClient.InstallChartWithValues(ctx, helmReleases.ConstellationOperators, operatorVals); err != nil {
return nil, fmt.Errorf("installing operators: %w", err)
}

14
bootstrapper/internal/kubernetes/kubernetes_test.go
View file

@ -182,6 +182,7 @@ func TestInitCluster(t *testing.T) {
require := require.New(t)
kube := KubeWrapper{
cloudProvider: "aws", // provide a valid cloud provider for cilium installation
clusterUtil: &tc.clusterUtil,
helmClient: &tc.helmClient,
providerMetadata: tc.providerMetadata,
@ -566,8 +567,17 @@ func (s *stubHelmClient) InstallCilium(_ context.Context, _ k8sapi.Client, _ hel
return s.ciliumError
}
func (s *stubHelmClient) InstallChart(_ context.Context, _ helm.Release, _ map[string]any) error {
return s.installChartError
func (s *stubHelmClient) InstallChart(ctx context.Context, release helm.Release) error {
return s.InstallChartWithValues(ctx, release, release.Values)
}
func (s *stubHelmClient) InstallChartWithValues(_ context.Context, release helm.Release, _ map[string]any) error {
switch release.ReleaseName {
case "cilium":
return s.ciliumError
default:
return s.installChartError
}
}
type stubKubeAPIWaiter struct {