Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-10-01 01:36:09 -04:00
Code Issues Packages Projects Releases Wiki Activity

cli: create backups for CRDs and their resources

Browse Source 
These backups could be used in case an upgrade
misbehaves after helm declared it as successful.
The manual backups are required as helm-rollback
won't touch custom resources and changes to CRDs
delete resources of the old version.
This commit is contained in:
Otto Bittner 2022-12-19 08:08:46 +01:00
parent afbd4a3dc1
commit e7c7e35f51
11 changed files with 397 additions and 87 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
bootstrapper/cmd/bootstrapper/main.go
View File

@ -17,7 +17,6 @@ import (
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/initserver"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/kubectl"
kubewaiter "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubeWaiter"
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/logging"
"github.com/edgelesssys/constellation/v2/internal/atls"
@ -36,6 +35,7 @@ import (
"github.com/edgelesssys/constellation/v2/internal/cloud/vmtype"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
"github.com/edgelesssys/constellation/v2/internal/logger"
"github.com/edgelesssys/constellation/v2/internal/oid"
"github.com/spf13/afero"

9
cli/internal/cloudcmd/upgrade.go
View File

@ -18,8 +18,8 @@ import (
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
corev1 "k8s.io/api/core/v1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
@ -55,12 +55,7 @@ func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) {
return nil, fmt.Errorf("setting up custom resource client: %w", err)
}
client, err := apiextensionsclient.NewForConfig(kubeConfig)
if err != nil {
return nil, err
}
helmClient, err := helm.NewClient(constants.AdminConfFilename, constants.HelmNamespace, client, log)
helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log)
if err != nil {
return nil, fmt.Errorf("setting up helm client: %w", err)
}

80
cli/internal/helm/backup.go Normal file
View File

@ -0,0 +1,80 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"fmt"
"path/filepath"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/yaml"
)
const (
crdBackupFolder = "constellation-upgrade/backups/crds/"
backupFolder = "constellation-upgrade/backups/"
)
func (c *Client) backupCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) {
crds, err := c.kubectl.GetCRDs(ctx)
if err != nil {
return nil, fmt.Errorf("getting CRDs: %w", err)
}
if err := c.fs.MkdirAll(crdBackupFolder); err != nil {
return nil, fmt.Errorf("creating backup dir: %w", err)
}
for i := range crds {
path := filepath.Join(crdBackupFolder, crds[i].Name+".yaml")
// We have to manually set kind/apiversion because of a long-standing limitation of the API:
// https://github.com/kubernetes/kubernetes/issues/3030#issuecomment-67543738
// The comment states that kind/version are encoded in the type.
// The package holding the CRD type encodes the version.
crds[i].Kind = "CustomResourceDefinition"
crds[i].APIVersion = "apiextensions.k8s.io/v1"
yamlBytes, err := yaml.Marshal(crds[i])
if err != nil {
return nil, err
}
if err := c.fs.Write(path, yamlBytes); err != nil {
return nil, err
}
c.log.Debugf("Created backup crd: %s", path)
}
return crds, nil
}
func (c *Client) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition) error {
for _, crd := range crds {
for _, version := range crd.Spec.Versions {
gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: version.Name, Resource: crd.Spec.Names.Plural}
crs, err := c.kubectl.GetCRs(ctx, gvr)
if err != nil {
return fmt.Errorf("retrieving CR %s: %w", crd.Name, err)
}
for _, cr := range crs {
path := filepath.Join(backupFolder, cr.GetName()+".yaml")
yamlBytes, err := yaml.Marshal(cr.Object)
if err != nil {
return err
}
if err := c.fs.Write(path, yamlBytes); err != nil {
return err
}
}
}
c.log.Debugf("Created backups for resource type: %s", crd.Name)
}
return nil
}

168
cli/internal/helm/backup_test.go Normal file
View File

@ -0,0 +1,168 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package helm
import (
"context"
"path/filepath"
"testing"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/pkg/errors"
"github.com/spf13/afero"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/yaml"
)
func TestBackupCRDs(t *testing.T) {
testCases := map[string]struct {
crd string
expectedFile string
getCRDsError error
wantError bool
}{
"success": {
crd: "apiVersion: \nkind: \nmetadata:\n name: foobar\n creationTimestamp: null\nspec:\n group: \"\"\n names:\n kind: \"somename\"\n plural: \"somenames\"\n scope: \"\"\n versions: null\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: null\n storedVersions: null\n",
expectedFile: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: foobar\n creationTimestamp: null\nspec:\n group: \"\"\n names:\n kind: \"somename\"\n plural: \"somenames\"\n scope: \"\"\n versions: null\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: null\n storedVersions: null\n",
},
"api request fails": {
getCRDsError: errors.New("api error"),
wantError: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
memFs := afero.NewMemMapFs()
crd := apiextensionsv1.CustomResourceDefinition{}
err := yaml.Unmarshal([]byte(tc.crd), &crd)
require.NoError(err)
client := Client{
config: nil,
kubectl: stubCrdClient{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError},
fs: file.NewHandler(memFs),
log: stubLog{},
}
_, err = client.backupCRDs(context.Background())
if tc.wantError {
assert.Error(err)
return
}
assert.NoError(err)
data, err := afero.ReadFile(memFs, filepath.Join(crdBackupFolder, crd.Name+".yaml"))
require.NoError(err)
assert.YAMLEq(tc.expectedFile, string(data))
})
}
}
func TestBackupCRs(t *testing.T) {
testCases := map[string]struct {
crd apiextensionsv1.CustomResourceDefinition
resource unstructured.Unstructured
expectedFile string
getCRsError error
wantError bool
}{
"success": {
crd: apiextensionsv1.CustomResourceDefinition{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foobars",
},
Group: "some.group",
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "versionZero",
},
},
},
},
resource: unstructured.Unstructured{Object: map[string]any{"metadata": map[string]any{"name": "foobar"}}},
expectedFile: "metadata:\n name: foobar\n",
},
"api request fails": {
crd: apiextensionsv1.CustomResourceDefinition{
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
Names: apiextensionsv1.CustomResourceDefinitionNames{
Plural: "foobars",
},
Group: "some.group",
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
{
Name: "versionZero",
},
},
},
},
getCRsError: errors.New("api error"),
wantError: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
memFs := afero.NewMemMapFs()
client := Client{
config: nil,
kubectl: stubCrdClient{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError},
fs: file.NewHandler(memFs),
log: stubLog{},
}
err := client.backupCRs(context.Background(), []apiextensionsv1.CustomResourceDefinition{tc.crd})
if tc.wantError {
assert.Error(err)
return
}
assert.NoError(err)
data, err := afero.ReadFile(memFs, filepath.Join(backupFolder, tc.resource.GetName()+".yaml"))
require.NoError(err)
assert.YAMLEq(tc.expectedFile, string(data))
})
}
}
type stubLog struct{}
func (s stubLog) Debugf(format string, args ...any) {}
func (s stubLog) Sync() {}
type stubCrdClient struct {
crds []apiextensionsv1.CustomResourceDefinition
getCRDsError error
crs []unstructured.Unstructured
getCRsError error
crdClient
}
func (c stubCrdClient) GetCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) {
if c.getCRDsError != nil {
return nil, c.getCRDsError
}
return c.crds, nil
}
func (c stubCrdClient) GetCRs(ctx context.Context, gvr schema.GroupVersionResource) ([]unstructured.Unstructured, error) {
if c.getCRsError != nil {
return nil, c.getCRsError
}
return c.crs, nil
}

75
cli/internal/helm/client.go
View File

@ -15,28 +15,27 @@ import (
"github.com/edgelesssys/constellation/v2/internal/config"
"github.com/edgelesssys/constellation/v2/internal/constants"
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
"github.com/pkg/errors"
"github.com/edgelesssys/constellation/v2/internal/file"
"github.com/spf13/afero"
"golang.org/x/mod/semver"
"helm.sh/helm/v3/pkg/action"
"helm.sh/helm/v3/pkg/chart"
"helm.sh/helm/v3/pkg/cli"
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/client-go/kubernetes/scheme"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
)
// Client handles interaction with helm.
type Client struct {
config *action.Configuration
crdClient *apiextensionsclient.Clientset
log debugLog
config *action.Configuration
kubectl crdClient
fs file.Handler
log debugLog
}
// NewClient returns a new initializes client for the namespace Client.
func NewClient(kubeConfigPath, helmNamespace string, client *apiextensionsclient.Clientset, log debugLog) (*Client, error) {
func NewClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*Client, error) {
settings := cli.New()
settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename
@ -45,7 +44,18 @@ func NewClient(kubeConfigPath, helmNamespace string, client *apiextensionsclient
return nil, fmt.Errorf("initializing config: %w", err)
}
return &Client{config: actionConfig, crdClient: client, log: log}, nil
fileHandler := file.NewHandler(afero.NewOsFs())
kubeconfig, err := fileHandler.Read(kubeConfigPath)
if err != nil {
return nil, fmt.Errorf("reading gce config: %w", err)
}
if err := client.Initialize(kubeconfig); err != nil {
return nil, fmt.Errorf("initializing kubectl: %w", err)
}
return &Client{config: actionConfig, kubectl: client, fs: fileHandler, log: log}, nil
}
// Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
@ -168,38 +178,6 @@ func (c *Client) GetValues(release string) (map[string]any, error) {
return values, nil
}
// ApplyCRD updates the given CRD by parsing it, querying it's version from the cluster and finally updating it.
func (c *Client) ApplyCRD(ctx context.Context, rawCRD []byte) error {
crd, err := parseCRD(rawCRD)
if err != nil {
return fmt.Errorf("parsing crds: %w", err)
}
clusterCRD, err := c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("getting crd: %w", err)
}
crd.ResourceVersion = clusterCRD.ResourceVersion
_, err = c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{})
return err
}
// parseCRD takes a byte slice of data and tries to create a CustomResourceDefinition object from it.
func parseCRD(crdString []byte) (*v1.CustomResourceDefinition, error) {
sch := runtime.NewScheme()
_ = scheme.AddToScheme(sch)
_ = v1.AddToScheme(sch)
obj, groupVersionKind, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(crdString, nil, nil)
if err != nil {
return nil, fmt.Errorf("decoding crd: %w", err)
}
if groupVersionKind.Kind == "CustomResourceDefinition" {
return obj.(*v1.CustomResourceDefinition), nil
}
return nil, errors.New("parsed []byte, but did not find a CRD")
}
// updateCRDs walks through the dependencies of the given chart and applies
// the files in the dependencie's 'crds' folder.
// This function is NOT recursive!
@ -208,7 +186,7 @@ func (c *Client) updateCRDs(ctx context.Context, chart *chart.Chart) error {
for _, crdFile := range dep.Files {
if strings.HasPrefix(crdFile.Name, "crds/") {
c.log.Debugf("Updating crd: %s", crdFile.Name)
err := c.ApplyCRD(ctx, crdFile.Data)
err := c.kubectl.ApplyCRD(ctx, crdFile.Data)
if err != nil {
return err
}
@ -245,3 +223,10 @@ type debugLog interface {
Debugf(format string, args ...any)
Sync()
}
type crdClient interface {
Initialize(kubeconfig []byte) error
ApplyCRD(ctx context.Context, rawCRD []byte) error
GetCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error)
GetCRs(ctx context.Context, gvr schema.GroupVersionResource) ([]unstructured.Unstructured, error)
}

33
cli/internal/helm/client_test.go
View File

@ -12,39 +12,6 @@ import (
"github.com/stretchr/testify/assert"
)
func TestParseCRDs(t *testing.T) {
testCases := map[string]struct {
data string
wantErr bool
}{
"success": {
data: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: nodeimages.update.edgeless.systems\nspec:\n group: update.edgeless.systems\n names:\n kind: NodeImage\n",
wantErr: false,
},
"wrong kind": {
data: "apiVersion: v1\nkind: Secret\ntype: Opaque\nmetadata:\n name: supersecret\n namespace: testNamespace\ndata:\n data: YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\n",
wantErr: true,
},
"decoding error": {
data: "asdf",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
_, err := parseCRD([]byte(tc.data))
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
})
}
}
func TestIsUpgrade(t *testing.T) {
testCases := map[string]struct {
currentVersion string

2
go.mod
View File

@ -303,5 +303,5 @@ require (
sigs.k8s.io/kustomize/api v0.12.1 // indirect
sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
sigs.k8s.io/yaml v1.3.0 // indirect
sigs.k8s.io/yaml v1.3.0
)

69
bootstrapper/internal/kubernetes/k8sapi/kubectl/kubectl.go → internal/kubernetes/kubectl/kubectl.go
View File

@ -8,13 +8,22 @@ package kubectl
import (
"context"
"errors"
"fmt"
corev1 "k8s.io/api/core/v1"
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/runtime/serializer"
"k8s.io/cli-runtime/pkg/resource"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/scale/scheme"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/util/retry"
)
@ -22,6 +31,7 @@ import (
// Kubectl implements functionality of the Kubernetes "kubectl" tool.
type Kubectl struct {
kubernetes.Interface
dynamicClient dynamic.Interface
apiextensionClient apiextensionsclientv1.ApiextensionsV1Interface
builder *resource.Builder
}
@ -43,6 +53,12 @@ func (k *Kubectl) Initialize(kubeconfig []byte) error {
}
k.Interface = clientset
dynamicClient, err := dynamic.NewForConfig(clientConfig)
if err != nil {
return fmt.Errorf("creating unstructed client: %w", err)
}
k.dynamicClient = dynamicClient
apiextensionClient, err := apiextensionsclientv1.NewForConfig(clientConfig)
if err != nil {
return fmt.Errorf("creating api extension client from kubeconfig: %w", err)
@ -58,6 +74,59 @@ func (k *Kubectl) Initialize(kubeconfig []byte) error {
return nil
}
// ApplyCRD updates the given CRD by parsing it, querying it's version from the cluster and finally updating it.
func (k *Kubectl) ApplyCRD(ctx context.Context, rawCRD []byte) error {
crd, err := parseCRD(rawCRD)
if err != nil {
return fmt.Errorf("parsing crds: %w", err)
}
clusterCRD, err := k.apiextensionClient.CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
if err != nil {
return fmt.Errorf("getting crd: %w", err)
}
crd.ResourceVersion = clusterCRD.ResourceVersion
_, err = k.apiextensionClient.CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{})
return err
}
// parseCRD takes a byte slice of data and tries to create a CustomResourceDefinition object from it.
func parseCRD(crdString []byte) (*v1.CustomResourceDefinition, error) {
sch := runtime.NewScheme()
_ = scheme.AddToScheme(sch)
_ = v1.AddToScheme(sch)
obj, groupVersionKind, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(crdString, nil, nil)
if err != nil {
return nil, fmt.Errorf("decoding crd: %w", err)
}
if groupVersionKind.Kind == "CustomResourceDefinition" {
return obj.(*v1.CustomResourceDefinition), nil
}
return nil, errors.New("parsed []byte, but did not find a CRD")
}
// GetCRDs retrieves all custom resource definitions currently installed in the cluster.
func (k *Kubectl) GetCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) {
crds, err := k.apiextensionClient.CustomResourceDefinitions().List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("listing CRDs: %w", err)
}
return crds.Items, nil
}
// GetCRs retrieves all objects for a given CRD.
func (k *Kubectl) GetCRs(ctx context.Context, gvr schema.GroupVersionResource) ([]unstructured.Unstructured, error) {
crdClient := k.dynamicClient.Resource(gvr)
unstructuredList, err := crdClient.List(ctx, metav1.ListOptions{})
if err != nil {
return nil, fmt.Errorf("listing CRDs for gvr %+v: %w", crdClient, err)
}
return unstructuredList.Items, nil
}
// CreateConfigMap creates the provided configmap.
func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
_, err := k.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Create(ctx, &configMap, metav1.CreateOptions{})

46
internal/kubernetes/kubectl/kubectl_test.go Normal file
View File

@ -0,0 +1,46 @@
/*
Copyright (c) Edgeless Systems GmbH
SPDX-License-Identifier: AGPL-3.0-only
*/
package kubectl
import (
"testing"
"github.com/stretchr/testify/assert"
)
func TestParseCRDs(t *testing.T) {
testCases := map[string]struct {
data string
wantErr bool
}{
"success": {
data: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: nodeimages.update.edgeless.systems\nspec:\n group: update.edgeless.systems\n names:\n kind: NodeImage\n",
wantErr: false,
},
"wrong kind": {
data: "apiVersion: v1\nkind: Secret\ntype: Opaque\nmetadata:\n name: supersecret\n namespace: testNamespace\ndata:\n data: YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\n",
wantErr: true,
},
"decoding error": {
data: "asdf",
wantErr: true,
},
}
for name, tc := range testCases {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
_, err := parseCRD([]byte(tc.data))
if tc.wantErr {
assert.Error(err)
return
}
assert.NoError(err)
})
}
}

0
bootstrapper/internal/kubernetes/k8sapi/kubectl/restclient.go → internal/kubernetes/kubectl/restclient.go
View File

0
bootstrapper/internal/kubernetes/k8sapi/kubectl/restclient_test.go → internal/kubernetes/kubectl/restclient_test.go
View File