From 8ef5ea2efe48f44bc5cbdacc5bcd15511cc7440a Mon Sep 17 00:00:00 2001 From: Markus Rudy Date: Fri, 13 Sep 2024 09:47:33 +0200 Subject: [PATCH] helm: fix kubeadm bugs caused by CoreDNS installation (#3353) * helm: rename CoreDNS configmap * upgrade-agent: ignore CoreDNS preflight errors * fixup! helm: rename CoreDNS configmap --- cli/internal/cmd/apply.go | 4 +++ cli/internal/cmd/apply_test.go | 1 + cli/internal/cmd/init_test.go | 4 +++ cli/internal/cmd/upgradeapply_test.go | 4 +++ internal/constellation/helm.go | 15 +++++++++ .../charts/coredns/templates/configmap.yaml | 10 +++--- .../charts/coredns/templates/deployment.yaml | 2 +- .../helm/corednsgen/corednsgen.go | 32 +++++++++++++++++-- .../internal/provider/cluster_resource.go | 5 +++ upgrade-agent/internal/server/server.go | 10 ++++-- 10 files changed, 77 insertions(+), 10 deletions(-) diff --git a/cli/internal/cmd/apply.go b/cli/internal/cmd/apply.go index f22e25a60..aba3aa378 100644 --- a/cli/internal/cmd/apply.go +++ b/cli/internal/cmd/apply.go @@ -428,6 +428,9 @@ func (a *applyCmd) apply( if err := a.runHelmApply(cmd, conf, stateFile, upgradeDir); err != nil { return err } + if err := a.applier.CleanupCoreDNSResources(cmd.Context()); err != nil { + return fmt.Errorf("cleaning up CoreDNS: %w", err) + } } // Upgrade node image @@ -847,6 +850,7 @@ type applier interface { // methods required to install/upgrade Helm charts AnnotateCoreDNSResources(context.Context) error + CleanupCoreDNSResources(context.Context) error PrepareHelmCharts( flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) (helm.Applier, bool, error) diff --git a/cli/internal/cmd/apply_test.go b/cli/internal/cmd/apply_test.go index 33fe6ba90..17c03f33f 100644 --- a/cli/internal/cmd/apply_test.go +++ b/cli/internal/cmd/apply_test.go @@ -554,6 +554,7 @@ func (s *stubConstellApplier) Init(context.Context, atls.Validator, *state.State type helmApplier interface { AnnotateCoreDNSResources(context.Context) error + CleanupCoreDNSResources(ctx context.Context) error PrepareHelmCharts( flags helm.Options, stateFile *state.State, serviceAccURI string, masterSecret uri.MasterSecret, ) ( diff --git a/cli/internal/cmd/init_test.go b/cli/internal/cmd/init_test.go index ccad3eb30..568c31ff8 100644 --- a/cli/internal/cmd/init_test.go +++ b/cli/internal/cmd/init_test.go @@ -282,6 +282,10 @@ func (s stubHelmApplier) AnnotateCoreDNSResources(_ context.Context) error { return nil } +func (s stubHelmApplier) CleanupCoreDNSResources(_ context.Context) error { + return nil +} + func (s stubHelmApplier) PrepareHelmCharts( _ helm.Options, _ *state.State, _ string, _ uri.MasterSecret, ) (helm.Applier, bool, error) { diff --git a/cli/internal/cmd/upgradeapply_test.go b/cli/internal/cmd/upgradeapply_test.go index 8a4341c2c..db4012596 100644 --- a/cli/internal/cmd/upgradeapply_test.go +++ b/cli/internal/cmd/upgradeapply_test.go @@ -379,6 +379,10 @@ func (m *mockApplier) AnnotateCoreDNSResources(_ context.Context) error { return nil } +func (m *mockApplier) CleanupCoreDNSResources(_ context.Context) error { + return nil +} + func (m *mockApplier) PrepareHelmCharts( helmOpts helm.Options, stateFile *state.State, str string, masterSecret uri.MasterSecret, ) (helm.Applier, bool, error) { diff --git a/internal/constellation/helm.go b/internal/constellation/helm.go index 4551e6ef4..ed1345459 100644 --- a/internal/constellation/helm.go +++ b/internal/constellation/helm.go @@ -60,6 +60,21 @@ func (a *Applier) AnnotateCoreDNSResources(ctx context.Context) error { return nil } +// CleanupCoreDNSResources removes CoreDNS resources that are not managed by Helm. +// +// This is only required when CoreDNS was installed by kubeadm directly. +// TODO(burgerdev): remove after v2.19 is released. +func (a *Applier) CleanupCoreDNSResources(ctx context.Context) error { + err := a.dynamicClient. + Resource(schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}). + Namespace("kube-system"). + Delete(ctx, "coredns", v1.DeleteOptions{}) + if !k8serrors.IsNotFound(err) { + return err + } + return nil +} + // PrepareHelmCharts loads Helm charts for Constellation and returns an executor to apply them. func (a *Applier) PrepareHelmCharts( flags helm.Options, state *state.State, serviceAccURI string, masterSecret uri.MasterSecret, diff --git a/internal/constellation/helm/charts/coredns/templates/configmap.yaml b/internal/constellation/helm/charts/coredns/templates/configmap.yaml index 3cec1e9e4..58a48a318 100644 --- a/internal/constellation/helm/charts/coredns/templates/configmap.yaml +++ b/internal/constellation/helm/charts/coredns/templates/configmap.yaml @@ -1,9 +1,4 @@ - apiVersion: v1 -kind: ConfigMap -metadata: - name: coredns - namespace: kube-system data: Corefile: | .:53 { @@ -26,3 +21,8 @@ data: reload loadbalance } +kind: ConfigMap +metadata: + creationTimestamp: null + name: edg-coredns + namespace: kube-system diff --git a/internal/constellation/helm/charts/coredns/templates/deployment.yaml b/internal/constellation/helm/charts/coredns/templates/deployment.yaml index c8e7dbff0..b7fd735df 100644 --- a/internal/constellation/helm/charts/coredns/templates/deployment.yaml +++ b/internal/constellation/helm/charts/coredns/templates/deployment.yaml @@ -104,6 +104,6 @@ spec: items: - key: Corefile path: Corefile - name: coredns + name: edg-coredns name: config-volume status: {} diff --git a/internal/constellation/helm/corednsgen/corednsgen.go b/internal/constellation/helm/corednsgen/corednsgen.go index f777914c4..5c7bc08f2 100644 --- a/internal/constellation/helm/corednsgen/corednsgen.go +++ b/internal/constellation/helm/corednsgen/corednsgen.go @@ -29,6 +29,8 @@ import ( "sigs.k8s.io/yaml" ) +const configMapName = "edg-coredns" + var chartDir = flag.String("charts", "./charts", "target directory to create charts in") func main() { @@ -44,9 +46,9 @@ func main() { writeTemplate(kubedns.CoreDNSServiceAccount, "serviceaccount.yaml") writeTemplate(kubedns.CoreDNSClusterRole, "clusterrole.yaml") writeTemplate(kubedns.CoreDNSClusterRoleBinding, "clusterrolebinding.yaml") - writeTemplate(kubedns.CoreDNSConfigMap, "configmap.yaml") writeTemplate(kubedns.CoreDNSService, "service.yaml") + writeFileRelativeToChartDir(patchedConfigMap(), "templates", "configmap.yaml") writeFileRelativeToChartDir(patchedDeployment(), "templates", "deployment.yaml") } @@ -92,7 +94,25 @@ func valuesYAML() []byte { return data } -// patchedDeployment extracts the CoreDNS deployment from kubeadm and adds necessary tolerations. +// patchedConfigMap renames the CoreDNS ConfigMap such that kubeadm does not find it. +// +// See https://github.com/kubernetes/kubeadm/issues/2846#issuecomment-1899942683. +func patchedConfigMap() []byte { + var cm corev1.ConfigMap + if err := yaml.Unmarshal(parseTemplate(kubedns.CoreDNSConfigMap), &cm); err != nil { + log.Fatalf("Could not parse configmap: %v", err) + } + + cm.Name = configMapName + + out, err := yaml.Marshal(cm) + if err != nil { + log.Fatalf("Could not marshal patched deployment: %v", err) + } + return out +} + +// patchedDeployment extracts the CoreDNS Deployment from kubeadm, adds necessary tolerations and updates the ConfigMap reference. func patchedDeployment() []byte { var d appsv1.Deployment if err := yaml.Unmarshal(parseTemplate(kubedns.CoreDNSDeployment), &d); err != nil { @@ -104,6 +124,14 @@ func patchedDeployment() []byte { {Key: "node.kubernetes.io/unreachable", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoExecute, TolerationSeconds: toPtr(int64(10))}, } d.Spec.Template.Spec.Tolerations = append(d.Spec.Template.Spec.Tolerations, tolerations...) + + for i, vol := range d.Spec.Template.Spec.Volumes { + if vol.ConfigMap != nil { + vol.ConfigMap.Name = configMapName + } + d.Spec.Template.Spec.Volumes[i] = vol + } + out, err := yaml.Marshal(d) if err != nil { log.Fatalf("Could not marshal patched deployment: %v", err) diff --git a/terraform-provider-constellation/internal/provider/cluster_resource.go b/terraform-provider-constellation/internal/provider/cluster_resource.go index 01aa6615b..978771e83 100644 --- a/terraform-provider-constellation/internal/provider/cluster_resource.go +++ b/terraform-provider-constellation/internal/provider/cluster_resource.go @@ -1311,6 +1311,11 @@ func (r *ClusterResource) applyHelmCharts(ctx context.Context, applier *constell diags.AddError("Applying Helm charts", err.Error()) return diags } + + if err := applier.CleanupCoreDNSResources(ctx); err != nil { + diags.AddError("Cleaning up CoreDNS resources", err.Error()) + return diags + } return diags } diff --git a/upgrade-agent/internal/server/server.go b/upgrade-agent/internal/server/server.go index 1c8ff41b8..3221baed5 100644 --- a/upgrade-agent/internal/server/server.go +++ b/upgrade-agent/internal/server/server.go @@ -111,12 +111,18 @@ func (s *Server) ExecuteUpdate(ctx context.Context, updateRequest *upgradeproto. return nil, status.Errorf(codes.Internal, "unable to install the kubeadm binary: %s", err) } - upgradeCmd := exec.CommandContext(ctx, "kubeadm", "upgrade", "plan", updateRequest.WantedKubernetesVersion) + // CoreDNS addon status is checked even though we did not install it. + // TODO(burgerdev): Use kubeadm phases once supported: https://github.com/kubernetes/kubeadm/issues/1318. + commonArgs := []string{"--ignore-preflight-errors", "CoreDNSMigration,CoreDNSUnsupportedPlugins", updateRequest.WantedKubernetesVersion} + planArgs := append([]string{"upgrade", "plan"}, commonArgs...) + applyArgs := append([]string{"upgrade", "apply", "--yes", "--patches", constants.KubeadmPatchDir}, commonArgs...) + + upgradeCmd := exec.CommandContext(ctx, "kubeadm", planArgs...) if out, err := upgradeCmd.CombinedOutput(); err != nil { return nil, status.Errorf(codes.Internal, "unable to execute kubeadm upgrade plan %s: %s: %s", updateRequest.WantedKubernetesVersion, err, string(out)) } - applyCmd := exec.CommandContext(ctx, "kubeadm", "upgrade", "apply", "--yes", "--patches", constants.KubeadmPatchDir, updateRequest.WantedKubernetesVersion) + applyCmd := exec.CommandContext(ctx, "kubeadm", applyArgs...) if out, err := applyCmd.CombinedOutput(); err != nil { return nil, status.Errorf(codes.Internal, "unable to execute kubeadm upgrade apply: %s: %s", err, string(out)) }