From 807bbbfd16850385ec0df55935fc1b860d9a855d Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Wed, 3 Jul 2024 19:37:51 +0200 Subject: [PATCH] cli: annotate CoreDNS resources for Helm (#3236) --- cli/internal/cmd/apply.go | 4 ++ cli/internal/cmd/apply_test.go | 1 + cli/internal/cmd/init_test.go | 4 ++ cli/internal/cmd/upgradeapply_test.go | 4 ++ internal/constellation/BUILD.bazel | 5 ++ internal/constellation/apply.go | 12 +++++ internal/constellation/helm.go | 46 +++++++++++++++++++ .../internal/provider/cluster_resource.go | 5 ++ 8 files changed, 81 insertions(+) diff --git a/cli/internal/cmd/apply.go b/cli/internal/cmd/apply.go index 2db2f318c..f22e25a60 100644 --- a/cli/internal/cmd/apply.go +++ b/cli/internal/cmd/apply.go @@ -422,6 +422,9 @@ func (a *applyCmd) apply( // Apply Helm Charts if !a.flags.skipPhases.contains(skipHelmPhase) { + if err := a.applier.AnnotateCoreDNSResources(cmd.Context()); err != nil { + return fmt.Errorf("annotating CoreDNS: %w", err) + } if err := a.runHelmApply(cmd, conf, stateFile, upgradeDir); err != nil { return err } @@ -843,6 +846,7 @@ type applier interface { // methods required to install/upgrade Helm charts + AnnotateCoreDNSResources(context.Context) error PrepareHelmCharts( flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) (helm.Applier, bool, error) diff --git a/cli/internal/cmd/apply_test.go b/cli/internal/cmd/apply_test.go index 064e1f42b..33fe6ba90 100644 --- a/cli/internal/cmd/apply_test.go +++ b/cli/internal/cmd/apply_test.go @@ -553,6 +553,7 @@ func (s *stubConstellApplier) Init(context.Context, atls.Validator, *state.State } type helmApplier interface { + AnnotateCoreDNSResources(context.Context) error PrepareHelmCharts( flags helm.Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) ( diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index 8d6d2b1bb..ad2a95be5 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -278,6 +278,10 @@ type stubHelmApplier struct { err error } +func (s stubHelmApplier) AnnotateCoreDNSResources(_ context.Context) error { + return nil +} + func (s stubHelmApplier) PrepareHelmCharts( _ helm.Options, _ *state.State, _ string, _ uri.MasterSecret, ) (helm.Applier, bool, error) { diff --git a/cli/internal/cmd/upgradeapply_test.go b/cli/internal/cmd/upgradeapply_test.go index f396cc828..8a4341c2c 100644 --- a/cli/internal/cmd/upgradeapply_test.go +++ b/cli/internal/cmd/upgradeapply_test.go @@ -375,6 +375,10 @@ type mockApplier struct { mock.Mock } +func (m *mockApplier) AnnotateCoreDNSResources(_ context.Context) error { + return nil +} + func (m *mockApplier) PrepareHelmCharts( helmOpts helm.Options, stateFile *state.State, str string, masterSecret uri.MasterSecret, ) (helm.Applier, bool, error) { diff --git a/internal/constellation/BUILD.bazel b/internal/constellation/BUILD.bazel index 0e59b388d..02afff0e9 100644 --- a/internal/constellation/BUILD.bazel +++ b/internal/constellation/BUILD.bazel @@ -37,6 +37,11 @@ go_library( "//internal/semver", "//internal/versions", "@io_k8s_apiextensions_apiserver//pkg/apis/apiextensions/v1:apiextensions", + "@io_k8s_apimachinery//pkg/api/errors", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_apimachinery//pkg/runtime/schema", + "@io_k8s_apimachinery//pkg/types", + "@io_k8s_client_go//dynamic", "@io_k8s_client_go//tools/clientcmd", "@org_golang_google_grpc//:grpc", ], diff --git a/internal/constellation/apply.go b/internal/constellation/apply.go index c9844b435..a9f9228dd 100644 --- a/internal/constellation/apply.go +++ b/internal/constellation/apply.go @@ -18,6 +18,8 @@ import ( "github.com/edgelesssys/constellation/v2/internal/grpc/dialer" "github.com/edgelesssys/constellation/v2/internal/kms/uri" "github.com/edgelesssys/constellation/v2/internal/license" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/tools/clientcmd" ) // ApplyContext denotes the context in which the apply command is run. @@ -44,6 +46,7 @@ type Applier struct { newDialer func(validator atls.Validator) *dialer.Dialer kubecmdClient kubecmdClient helmClient helmApplier + dynamicClient dynamic.Interface } type licenseChecker interface { @@ -79,8 +82,17 @@ func (a *Applier) SetKubeConfig(kubeConfig []byte) error { if err != nil { return err } + restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeConfig) + if err != nil { + return err + } + dynamicClient, err := dynamic.NewForConfig(restConfig) + if err != nil { + return err + } a.kubecmdClient = kubecmdClient a.helmClient = helmClient + a.dynamicClient = dynamicClient return nil } diff --git a/internal/constellation/helm.go b/internal/constellation/helm.go index 1378ce3a0..4551e6ef4 100644 --- a/internal/constellation/helm.go +++ b/internal/constellation/helm.go @@ -7,13 +7,59 @@ SPDX-License-Identifier: AGPL-3.0-only package constellation import ( + "context" "errors" + "fmt" "github.com/edgelesssys/constellation/v2/internal/constellation/helm" "github.com/edgelesssys/constellation/v2/internal/constellation/state" "github.com/edgelesssys/constellation/v2/internal/kms/uri" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" ) +var patch = []byte(fmt.Sprintf(`{"metadata": {"labels": {%q: %q}, "annotations": {%q: %q, %q: %q}}}`, + "app.kubernetes.io/managed-by", "Helm", + "meta.helm.sh/release-name", "coredns", + "meta.helm.sh/release-namespace", "kube-system")) + +var namespacedCoreDNSResources = map[schema.GroupVersionResource]string{ + {Group: "apps", Version: "v1", Resource: "deployments"}: "coredns", + {Group: "", Version: "v1", Resource: "services"}: "kube-dns", + {Group: "", Version: "v1", Resource: "configmaps"}: "coredns", + {Group: "", Version: "v1", Resource: "serviceaccounts"}: "coredns", + {Group: "apps", Version: "v1", Resource: "statefulsets"}: "foobarbax", +} + +var coreDNSResources = map[schema.GroupVersionResource]string{ + {Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"}: "system:coredns", + {Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterrolebindings"}: "system:coredns", +} + +// AnnotateCoreDNSResources imports existing CoreDNS resources into the Helm release. +// +// This is only required when CoreDNS was installed by kubeadm directly. +// TODO(burgerdev): remove after v2.19 is released. +func (a *Applier) AnnotateCoreDNSResources(ctx context.Context) error { + for gvk, name := range coreDNSResources { + _, err := a.dynamicClient.Resource(gvk).Patch(ctx, name, types.StrategicMergePatchType, patch, v1.PatchOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + + for gvk, name := range namespacedCoreDNSResources { + _, err := a.dynamicClient.Resource(gvk).Namespace("kube-system").Patch(ctx, name, types.StrategicMergePatchType, patch, v1.PatchOptions{}) + if err != nil && !k8serrors.IsNotFound(err) { + return err + } + } + + return nil +} + // PrepareHelmCharts loads Helm charts for Constellation and returns an executor to apply them. func (a *Applier) PrepareHelmCharts( flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret, diff --git a/terraform-provider-constellation/internal/provider/cluster_resource.go b/terraform-provider-constellation/internal/provider/cluster_resource.go index a12fe38da..096621af6 100644 --- a/terraform-provider-constellation/internal/provider/cluster_resource.go +++ b/terraform-provider-constellation/internal/provider/cluster_resource.go @@ -1211,6 +1211,11 @@ func (r *ClusterResource) applyHelmCharts(ctx context.Context, applier *constell OpenStackValues: payload.openStackHelmValues, } + if err := applier.AnnotateCoreDNSResources(ctx); err != nil { + diags.AddError("Annotating CoreDNS resources", err.Error()) + return diags + } + executor, _, err := applier.PrepareHelmCharts(options, state, payload.serviceAccURI, payload.masterSecret) var upgradeErr *compatibility.InvalidUpgradeError