mirror of
https://github.com/edgelesssys/constellation.git
synced 2024-10-01 01:36:09 -04:00
cli: create backups for CRDs and their resources
These backups could be used in case an upgrade misbehaves after helm declared it as successful. The manual backups are required as helm-rollback won't touch custom resources and changes to CRDs delete resources of the old version.
This commit is contained in:
parent
afbd4a3dc1
commit
e7c7e35f51
@ -17,7 +17,6 @@ import (
|
|||||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/initserver"
|
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/initserver"
|
||||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes"
|
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes"
|
||||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
|
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi"
|
||||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/k8sapi/kubectl"
|
|
||||||
kubewaiter "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubeWaiter"
|
kubewaiter "github.com/edgelesssys/constellation/v2/bootstrapper/internal/kubernetes/kubeWaiter"
|
||||||
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/logging"
|
"github.com/edgelesssys/constellation/v2/bootstrapper/internal/logging"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/atls"
|
"github.com/edgelesssys/constellation/v2/internal/atls"
|
||||||
@ -36,6 +35,7 @@ import (
|
|||||||
"github.com/edgelesssys/constellation/v2/internal/cloud/vmtype"
|
"github.com/edgelesssys/constellation/v2/internal/cloud/vmtype"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/file"
|
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/logger"
|
"github.com/edgelesssys/constellation/v2/internal/logger"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/oid"
|
"github.com/edgelesssys/constellation/v2/internal/oid"
|
||||||
"github.com/spf13/afero"
|
"github.com/spf13/afero"
|
||||||
|
@ -18,8 +18,8 @@ import (
|
|||||||
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
|
"github.com/edgelesssys/constellation/v2/internal/attestation/measurements"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/config"
|
"github.com/edgelesssys/constellation/v2/internal/config"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/kubernetes/kubectl"
|
||||||
corev1 "k8s.io/api/core/v1"
|
corev1 "k8s.io/api/core/v1"
|
||||||
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
@ -55,12 +55,7 @@ func NewUpgrader(outWriter io.Writer, log debugLog) (*Upgrader, error) {
|
|||||||
return nil, fmt.Errorf("setting up custom resource client: %w", err)
|
return nil, fmt.Errorf("setting up custom resource client: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
client, err := apiextensionsclient.NewForConfig(kubeConfig)
|
helmClient, err := helm.NewClient(kubectl.New(), constants.AdminConfFilename, constants.HelmNamespace, log)
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
helmClient, err := helm.NewClient(constants.AdminConfFilename, constants.HelmNamespace, client, log)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("setting up helm client: %w", err)
|
return nil, fmt.Errorf("setting up helm client: %w", err)
|
||||||
}
|
}
|
||||||
|
80
cli/internal/helm/backup.go
Normal file
80
cli/internal/helm/backup.go
Normal file
@ -0,0 +1,80 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) Edgeless Systems GmbH
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
package helm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
"sigs.k8s.io/yaml"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
crdBackupFolder = "constellation-upgrade/backups/crds/"
|
||||||
|
backupFolder = "constellation-upgrade/backups/"
|
||||||
|
)
|
||||||
|
|
||||||
|
func (c *Client) backupCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) {
|
||||||
|
crds, err := c.kubectl.GetCRDs(ctx)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("getting CRDs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := c.fs.MkdirAll(crdBackupFolder); err != nil {
|
||||||
|
return nil, fmt.Errorf("creating backup dir: %w", err)
|
||||||
|
}
|
||||||
|
for i := range crds {
|
||||||
|
path := filepath.Join(crdBackupFolder, crds[i].Name+".yaml")
|
||||||
|
|
||||||
|
// We have to manually set kind/apiversion because of a long-standing limitation of the API:
|
||||||
|
// https://github.com/kubernetes/kubernetes/issues/3030#issuecomment-67543738
|
||||||
|
// The comment states that kind/version are encoded in the type.
|
||||||
|
// The package holding the CRD type encodes the version.
|
||||||
|
crds[i].Kind = "CustomResourceDefinition"
|
||||||
|
crds[i].APIVersion = "apiextensions.k8s.io/v1"
|
||||||
|
|
||||||
|
yamlBytes, err := yaml.Marshal(crds[i])
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := c.fs.Write(path, yamlBytes); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
c.log.Debugf("Created backup crd: %s", path)
|
||||||
|
}
|
||||||
|
return crds, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *Client) backupCRs(ctx context.Context, crds []apiextensionsv1.CustomResourceDefinition) error {
|
||||||
|
for _, crd := range crds {
|
||||||
|
for _, version := range crd.Spec.Versions {
|
||||||
|
gvr := schema.GroupVersionResource{Group: crd.Spec.Group, Version: version.Name, Resource: crd.Spec.Names.Plural}
|
||||||
|
crs, err := c.kubectl.GetCRs(ctx, gvr)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("retrieving CR %s: %w", crd.Name, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, cr := range crs {
|
||||||
|
path := filepath.Join(backupFolder, cr.GetName()+".yaml")
|
||||||
|
yamlBytes, err := yaml.Marshal(cr.Object)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := c.fs.Write(path, yamlBytes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
c.log.Debugf("Created backups for resource type: %s", crd.Name)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
168
cli/internal/helm/backup_test.go
Normal file
168
cli/internal/helm/backup_test.go
Normal file
@ -0,0 +1,168 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) Edgeless Systems GmbH
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
package helm
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"path/filepath"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/spf13/afero"
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
"sigs.k8s.io/yaml"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestBackupCRDs(t *testing.T) {
|
||||||
|
testCases := map[string]struct {
|
||||||
|
crd string
|
||||||
|
expectedFile string
|
||||||
|
getCRDsError error
|
||||||
|
wantError bool
|
||||||
|
}{
|
||||||
|
"success": {
|
||||||
|
crd: "apiVersion: \nkind: \nmetadata:\n name: foobar\n creationTimestamp: null\nspec:\n group: \"\"\n names:\n kind: \"somename\"\n plural: \"somenames\"\n scope: \"\"\n versions: null\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: null\n storedVersions: null\n",
|
||||||
|
expectedFile: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: foobar\n creationTimestamp: null\nspec:\n group: \"\"\n names:\n kind: \"somename\"\n plural: \"somenames\"\n scope: \"\"\n versions: null\nstatus:\n acceptedNames:\n kind: \"\"\n plural: \"\"\n conditions: null\n storedVersions: null\n",
|
||||||
|
},
|
||||||
|
"api request fails": {
|
||||||
|
getCRDsError: errors.New("api error"),
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range testCases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
require := require.New(t)
|
||||||
|
memFs := afero.NewMemMapFs()
|
||||||
|
|
||||||
|
crd := apiextensionsv1.CustomResourceDefinition{}
|
||||||
|
err := yaml.Unmarshal([]byte(tc.crd), &crd)
|
||||||
|
require.NoError(err)
|
||||||
|
client := Client{
|
||||||
|
config: nil,
|
||||||
|
kubectl: stubCrdClient{crds: []apiextensionsv1.CustomResourceDefinition{crd}, getCRDsError: tc.getCRDsError},
|
||||||
|
fs: file.NewHandler(memFs),
|
||||||
|
log: stubLog{},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = client.backupCRDs(context.Background())
|
||||||
|
if tc.wantError {
|
||||||
|
assert.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
data, err := afero.ReadFile(memFs, filepath.Join(crdBackupFolder, crd.Name+".yaml"))
|
||||||
|
require.NoError(err)
|
||||||
|
assert.YAMLEq(tc.expectedFile, string(data))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackupCRs(t *testing.T) {
|
||||||
|
testCases := map[string]struct {
|
||||||
|
crd apiextensionsv1.CustomResourceDefinition
|
||||||
|
resource unstructured.Unstructured
|
||||||
|
expectedFile string
|
||||||
|
getCRsError error
|
||||||
|
wantError bool
|
||||||
|
}{
|
||||||
|
"success": {
|
||||||
|
crd: apiextensionsv1.CustomResourceDefinition{
|
||||||
|
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
|
||||||
|
Names: apiextensionsv1.CustomResourceDefinitionNames{
|
||||||
|
Plural: "foobars",
|
||||||
|
},
|
||||||
|
Group: "some.group",
|
||||||
|
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
|
||||||
|
{
|
||||||
|
Name: "versionZero",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
resource: unstructured.Unstructured{Object: map[string]any{"metadata": map[string]any{"name": "foobar"}}},
|
||||||
|
expectedFile: "metadata:\n name: foobar\n",
|
||||||
|
},
|
||||||
|
"api request fails": {
|
||||||
|
crd: apiextensionsv1.CustomResourceDefinition{
|
||||||
|
Spec: apiextensionsv1.CustomResourceDefinitionSpec{
|
||||||
|
Names: apiextensionsv1.CustomResourceDefinitionNames{
|
||||||
|
Plural: "foobars",
|
||||||
|
},
|
||||||
|
Group: "some.group",
|
||||||
|
Versions: []apiextensionsv1.CustomResourceDefinitionVersion{
|
||||||
|
{
|
||||||
|
Name: "versionZero",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
getCRsError: errors.New("api error"),
|
||||||
|
wantError: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range testCases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
require := require.New(t)
|
||||||
|
memFs := afero.NewMemMapFs()
|
||||||
|
|
||||||
|
client := Client{
|
||||||
|
config: nil,
|
||||||
|
kubectl: stubCrdClient{crs: []unstructured.Unstructured{tc.resource}, getCRsError: tc.getCRsError},
|
||||||
|
fs: file.NewHandler(memFs),
|
||||||
|
log: stubLog{},
|
||||||
|
}
|
||||||
|
|
||||||
|
err := client.backupCRs(context.Background(), []apiextensionsv1.CustomResourceDefinition{tc.crd})
|
||||||
|
if tc.wantError {
|
||||||
|
assert.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assert.NoError(err)
|
||||||
|
|
||||||
|
data, err := afero.ReadFile(memFs, filepath.Join(backupFolder, tc.resource.GetName()+".yaml"))
|
||||||
|
require.NoError(err)
|
||||||
|
assert.YAMLEq(tc.expectedFile, string(data))
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type stubLog struct{}
|
||||||
|
|
||||||
|
func (s stubLog) Debugf(format string, args ...any) {}
|
||||||
|
func (s stubLog) Sync() {}
|
||||||
|
|
||||||
|
type stubCrdClient struct {
|
||||||
|
crds []apiextensionsv1.CustomResourceDefinition
|
||||||
|
getCRDsError error
|
||||||
|
crs []unstructured.Unstructured
|
||||||
|
getCRsError error
|
||||||
|
crdClient
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c stubCrdClient) GetCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) {
|
||||||
|
if c.getCRDsError != nil {
|
||||||
|
return nil, c.getCRDsError
|
||||||
|
}
|
||||||
|
return c.crds, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c stubCrdClient) GetCRs(ctx context.Context, gvr schema.GroupVersionResource) ([]unstructured.Unstructured, error) {
|
||||||
|
if c.getCRsError != nil {
|
||||||
|
return nil, c.getCRsError
|
||||||
|
}
|
||||||
|
return c.crs, nil
|
||||||
|
}
|
@ -15,28 +15,27 @@ import (
|
|||||||
"github.com/edgelesssys/constellation/v2/internal/config"
|
"github.com/edgelesssys/constellation/v2/internal/config"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/constants"
|
"github.com/edgelesssys/constellation/v2/internal/constants"
|
||||||
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
"github.com/edgelesssys/constellation/v2/internal/deploy/helm"
|
||||||
"github.com/pkg/errors"
|
"github.com/edgelesssys/constellation/v2/internal/file"
|
||||||
|
"github.com/spf13/afero"
|
||||||
"golang.org/x/mod/semver"
|
"golang.org/x/mod/semver"
|
||||||
"helm.sh/helm/v3/pkg/action"
|
"helm.sh/helm/v3/pkg/action"
|
||||||
"helm.sh/helm/v3/pkg/chart"
|
"helm.sh/helm/v3/pkg/chart"
|
||||||
"helm.sh/helm/v3/pkg/cli"
|
"helm.sh/helm/v3/pkg/cli"
|
||||||
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||||
apiextensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
"k8s.io/apimachinery/pkg/runtime"
|
|
||||||
"k8s.io/apimachinery/pkg/runtime/serializer"
|
|
||||||
"k8s.io/client-go/kubernetes/scheme"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
// Client handles interaction with helm.
|
// Client handles interaction with helm.
|
||||||
type Client struct {
|
type Client struct {
|
||||||
config *action.Configuration
|
config *action.Configuration
|
||||||
crdClient *apiextensionsclient.Clientset
|
kubectl crdClient
|
||||||
|
fs file.Handler
|
||||||
log debugLog
|
log debugLog
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewClient returns a new initializes client for the namespace Client.
|
// NewClient returns a new initializes client for the namespace Client.
|
||||||
func NewClient(kubeConfigPath, helmNamespace string, client *apiextensionsclient.Clientset, log debugLog) (*Client, error) {
|
func NewClient(client crdClient, kubeConfigPath, helmNamespace string, log debugLog) (*Client, error) {
|
||||||
settings := cli.New()
|
settings := cli.New()
|
||||||
settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename
|
settings.KubeConfig = kubeConfigPath // constants.AdminConfFilename
|
||||||
|
|
||||||
@ -45,7 +44,18 @@ func NewClient(kubeConfigPath, helmNamespace string, client *apiextensionsclient
|
|||||||
return nil, fmt.Errorf("initializing config: %w", err)
|
return nil, fmt.Errorf("initializing config: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &Client{config: actionConfig, crdClient: client, log: log}, nil
|
fileHandler := file.NewHandler(afero.NewOsFs())
|
||||||
|
|
||||||
|
kubeconfig, err := fileHandler.Read(kubeConfigPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("reading gce config: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := client.Initialize(kubeconfig); err != nil {
|
||||||
|
return nil, fmt.Errorf("initializing kubectl: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &Client{config: actionConfig, kubectl: client, fs: fileHandler, log: log}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
|
// Upgrade runs a helm-upgrade on all deployments that are managed via Helm.
|
||||||
@ -168,38 +178,6 @@ func (c *Client) GetValues(release string) (map[string]any, error) {
|
|||||||
return values, nil
|
return values, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ApplyCRD updates the given CRD by parsing it, querying it's version from the cluster and finally updating it.
|
|
||||||
func (c *Client) ApplyCRD(ctx context.Context, rawCRD []byte) error {
|
|
||||||
crd, err := parseCRD(rawCRD)
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("parsing crds: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
clusterCRD, err := c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
|
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("getting crd: %w", err)
|
|
||||||
}
|
|
||||||
crd.ResourceVersion = clusterCRD.ResourceVersion
|
|
||||||
_, err = c.crdClient.ApiextensionsV1().CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{})
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
|
|
||||||
// parseCRD takes a byte slice of data and tries to create a CustomResourceDefinition object from it.
|
|
||||||
func parseCRD(crdString []byte) (*v1.CustomResourceDefinition, error) {
|
|
||||||
sch := runtime.NewScheme()
|
|
||||||
_ = scheme.AddToScheme(sch)
|
|
||||||
_ = v1.AddToScheme(sch)
|
|
||||||
obj, groupVersionKind, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(crdString, nil, nil)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("decoding crd: %w", err)
|
|
||||||
}
|
|
||||||
if groupVersionKind.Kind == "CustomResourceDefinition" {
|
|
||||||
return obj.(*v1.CustomResourceDefinition), nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return nil, errors.New("parsed []byte, but did not find a CRD")
|
|
||||||
}
|
|
||||||
|
|
||||||
// updateCRDs walks through the dependencies of the given chart and applies
|
// updateCRDs walks through the dependencies of the given chart and applies
|
||||||
// the files in the dependencie's 'crds' folder.
|
// the files in the dependencie's 'crds' folder.
|
||||||
// This function is NOT recursive!
|
// This function is NOT recursive!
|
||||||
@ -208,7 +186,7 @@ func (c *Client) updateCRDs(ctx context.Context, chart *chart.Chart) error {
|
|||||||
for _, crdFile := range dep.Files {
|
for _, crdFile := range dep.Files {
|
||||||
if strings.HasPrefix(crdFile.Name, "crds/") {
|
if strings.HasPrefix(crdFile.Name, "crds/") {
|
||||||
c.log.Debugf("Updating crd: %s", crdFile.Name)
|
c.log.Debugf("Updating crd: %s", crdFile.Name)
|
||||||
err := c.ApplyCRD(ctx, crdFile.Data)
|
err := c.kubectl.ApplyCRD(ctx, crdFile.Data)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -245,3 +223,10 @@ type debugLog interface {
|
|||||||
Debugf(format string, args ...any)
|
Debugf(format string, args ...any)
|
||||||
Sync()
|
Sync()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type crdClient interface {
|
||||||
|
Initialize(kubeconfig []byte) error
|
||||||
|
ApplyCRD(ctx context.Context, rawCRD []byte) error
|
||||||
|
GetCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error)
|
||||||
|
GetCRs(ctx context.Context, gvr schema.GroupVersionResource) ([]unstructured.Unstructured, error)
|
||||||
|
}
|
||||||
|
@ -12,39 +12,6 @@ import (
|
|||||||
"github.com/stretchr/testify/assert"
|
"github.com/stretchr/testify/assert"
|
||||||
)
|
)
|
||||||
|
|
||||||
func TestParseCRDs(t *testing.T) {
|
|
||||||
testCases := map[string]struct {
|
|
||||||
data string
|
|
||||||
wantErr bool
|
|
||||||
}{
|
|
||||||
"success": {
|
|
||||||
data: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: nodeimages.update.edgeless.systems\nspec:\n group: update.edgeless.systems\n names:\n kind: NodeImage\n",
|
|
||||||
wantErr: false,
|
|
||||||
},
|
|
||||||
"wrong kind": {
|
|
||||||
data: "apiVersion: v1\nkind: Secret\ntype: Opaque\nmetadata:\n name: supersecret\n namespace: testNamespace\ndata:\n data: YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\n",
|
|
||||||
wantErr: true,
|
|
||||||
},
|
|
||||||
"decoding error": {
|
|
||||||
data: "asdf",
|
|
||||||
wantErr: true,
|
|
||||||
},
|
|
||||||
}
|
|
||||||
|
|
||||||
for name, tc := range testCases {
|
|
||||||
t.Run(name, func(t *testing.T) {
|
|
||||||
assert := assert.New(t)
|
|
||||||
|
|
||||||
_, err := parseCRD([]byte(tc.data))
|
|
||||||
if tc.wantErr {
|
|
||||||
assert.Error(err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
assert.NoError(err)
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestIsUpgrade(t *testing.T) {
|
func TestIsUpgrade(t *testing.T) {
|
||||||
testCases := map[string]struct {
|
testCases := map[string]struct {
|
||||||
currentVersion string
|
currentVersion string
|
||||||
|
2
go.mod
2
go.mod
@ -303,5 +303,5 @@ require (
|
|||||||
sigs.k8s.io/kustomize/api v0.12.1 // indirect
|
sigs.k8s.io/kustomize/api v0.12.1 // indirect
|
||||||
sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
|
sigs.k8s.io/kustomize/kyaml v0.13.9 // indirect
|
||||||
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
|
sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
|
||||||
sigs.k8s.io/yaml v1.3.0 // indirect
|
sigs.k8s.io/yaml v1.3.0
|
||||||
)
|
)
|
||||||
|
@ -8,13 +8,22 @@ package kubectl
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
|
||||||
corev1 "k8s.io/api/core/v1"
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||||
|
v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
|
||||||
apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
|
apiextensionsclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1"
|
||||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||||
|
"k8s.io/apimachinery/pkg/runtime/serializer"
|
||||||
"k8s.io/cli-runtime/pkg/resource"
|
"k8s.io/cli-runtime/pkg/resource"
|
||||||
|
"k8s.io/client-go/dynamic"
|
||||||
"k8s.io/client-go/kubernetes"
|
"k8s.io/client-go/kubernetes"
|
||||||
|
"k8s.io/client-go/scale/scheme"
|
||||||
"k8s.io/client-go/tools/clientcmd"
|
"k8s.io/client-go/tools/clientcmd"
|
||||||
"k8s.io/client-go/util/retry"
|
"k8s.io/client-go/util/retry"
|
||||||
)
|
)
|
||||||
@ -22,6 +31,7 @@ import (
|
|||||||
// Kubectl implements functionality of the Kubernetes "kubectl" tool.
|
// Kubectl implements functionality of the Kubernetes "kubectl" tool.
|
||||||
type Kubectl struct {
|
type Kubectl struct {
|
||||||
kubernetes.Interface
|
kubernetes.Interface
|
||||||
|
dynamicClient dynamic.Interface
|
||||||
apiextensionClient apiextensionsclientv1.ApiextensionsV1Interface
|
apiextensionClient apiextensionsclientv1.ApiextensionsV1Interface
|
||||||
builder *resource.Builder
|
builder *resource.Builder
|
||||||
}
|
}
|
||||||
@ -43,6 +53,12 @@ func (k *Kubectl) Initialize(kubeconfig []byte) error {
|
|||||||
}
|
}
|
||||||
k.Interface = clientset
|
k.Interface = clientset
|
||||||
|
|
||||||
|
dynamicClient, err := dynamic.NewForConfig(clientConfig)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("creating unstructed client: %w", err)
|
||||||
|
}
|
||||||
|
k.dynamicClient = dynamicClient
|
||||||
|
|
||||||
apiextensionClient, err := apiextensionsclientv1.NewForConfig(clientConfig)
|
apiextensionClient, err := apiextensionsclientv1.NewForConfig(clientConfig)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("creating api extension client from kubeconfig: %w", err)
|
return fmt.Errorf("creating api extension client from kubeconfig: %w", err)
|
||||||
@ -58,6 +74,59 @@ func (k *Kubectl) Initialize(kubeconfig []byte) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ApplyCRD updates the given CRD by parsing it, querying it's version from the cluster and finally updating it.
|
||||||
|
func (k *Kubectl) ApplyCRD(ctx context.Context, rawCRD []byte) error {
|
||||||
|
crd, err := parseCRD(rawCRD)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("parsing crds: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
clusterCRD, err := k.apiextensionClient.CustomResourceDefinitions().Get(ctx, crd.Name, metav1.GetOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("getting crd: %w", err)
|
||||||
|
}
|
||||||
|
crd.ResourceVersion = clusterCRD.ResourceVersion
|
||||||
|
_, err = k.apiextensionClient.CustomResourceDefinitions().Update(ctx, crd, metav1.UpdateOptions{})
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseCRD takes a byte slice of data and tries to create a CustomResourceDefinition object from it.
|
||||||
|
func parseCRD(crdString []byte) (*v1.CustomResourceDefinition, error) {
|
||||||
|
sch := runtime.NewScheme()
|
||||||
|
_ = scheme.AddToScheme(sch)
|
||||||
|
_ = v1.AddToScheme(sch)
|
||||||
|
obj, groupVersionKind, err := serializer.NewCodecFactory(sch).UniversalDeserializer().Decode(crdString, nil, nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decoding crd: %w", err)
|
||||||
|
}
|
||||||
|
if groupVersionKind.Kind == "CustomResourceDefinition" {
|
||||||
|
return obj.(*v1.CustomResourceDefinition), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil, errors.New("parsed []byte, but did not find a CRD")
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCRDs retrieves all custom resource definitions currently installed in the cluster.
|
||||||
|
func (k *Kubectl) GetCRDs(ctx context.Context) ([]apiextensionsv1.CustomResourceDefinition, error) {
|
||||||
|
crds, err := k.apiextensionClient.CustomResourceDefinitions().List(ctx, metav1.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listing CRDs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return crds.Items, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetCRs retrieves all objects for a given CRD.
|
||||||
|
func (k *Kubectl) GetCRs(ctx context.Context, gvr schema.GroupVersionResource) ([]unstructured.Unstructured, error) {
|
||||||
|
crdClient := k.dynamicClient.Resource(gvr)
|
||||||
|
unstructuredList, err := crdClient.List(ctx, metav1.ListOptions{})
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("listing CRDs for gvr %+v: %w", crdClient, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return unstructuredList.Items, nil
|
||||||
|
}
|
||||||
|
|
||||||
// CreateConfigMap creates the provided configmap.
|
// CreateConfigMap creates the provided configmap.
|
||||||
func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
|
func (k *Kubectl) CreateConfigMap(ctx context.Context, configMap corev1.ConfigMap) error {
|
||||||
_, err := k.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Create(ctx, &configMap, metav1.CreateOptions{})
|
_, err := k.CoreV1().ConfigMaps(configMap.ObjectMeta.Namespace).Create(ctx, &configMap, metav1.CreateOptions{})
|
46
internal/kubernetes/kubectl/kubectl_test.go
Normal file
46
internal/kubernetes/kubectl/kubectl_test.go
Normal file
@ -0,0 +1,46 @@
|
|||||||
|
/*
|
||||||
|
Copyright (c) Edgeless Systems GmbH
|
||||||
|
|
||||||
|
SPDX-License-Identifier: AGPL-3.0-only
|
||||||
|
*/
|
||||||
|
|
||||||
|
package kubectl
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/assert"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestParseCRDs(t *testing.T) {
|
||||||
|
testCases := map[string]struct {
|
||||||
|
data string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
"success": {
|
||||||
|
data: "apiVersion: apiextensions.k8s.io/v1\nkind: CustomResourceDefinition\nmetadata:\n name: nodeimages.update.edgeless.systems\nspec:\n group: update.edgeless.systems\n names:\n kind: NodeImage\n",
|
||||||
|
wantErr: false,
|
||||||
|
},
|
||||||
|
"wrong kind": {
|
||||||
|
data: "apiVersion: v1\nkind: Secret\ntype: Opaque\nmetadata:\n name: supersecret\n namespace: testNamespace\ndata:\n data: YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWE=\n",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
"decoding error": {
|
||||||
|
data: "asdf",
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for name, tc := range testCases {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
assert := assert.New(t)
|
||||||
|
|
||||||
|
_, err := parseCRD([]byte(tc.data))
|
||||||
|
if tc.wantErr {
|
||||||
|
assert.Error(err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
assert.NoError(err)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue
Block a user