upgrade: support Kubernetes components (#839)

* upgrade: add Kubernetes components to NodeVersion

* update rfc
This commit is contained in:
3u13r 2023-01-03 12:09:53 +01:00 committed by GitHub
parent 4b43311fbd
commit f14af0c3eb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
56 changed files with 897 additions and 738 deletions

View file

@ -12,7 +12,7 @@ resources:
controller: true
domain: edgeless.systems
group: update
kind: NodeImage
kind: NodeVersion
path: github.com/edgelesssys/constellation/operators/constellation-node-operator/api/v1alpha1
version: v1alpha1
- api:

View file

@ -15,14 +15,14 @@ In particular, it is responsible for updating the OS images of nodes by replacin
The operator has multiple controllers with corresponding custom resource definitions (CRDs) that are responsible for the following high level tasks:
### NodeImage
### NodeVersion
`NodeImage` is the only user controlled CRD. The spec allows an administrator to update the desired image and trigger a rolling update.
`NodeVersion` is the only user controlled CRD. The spec allows an administrator to update the desired image and trigger a rolling update.
Example for GCP:
```yaml
apiVersion: update.edgeless.systems/v1alpha1
kind: NodeImage
kind: NodeVersion
metadata:
name: constellation-os
spec:
@ -32,7 +32,7 @@ spec:
Example for Azure:
```yaml
apiVersion: update.edgeless.systems/v1alpha1
kind: NodeImage
kind: NodeVersion
metadata:
name: constellation-os
spec:
@ -42,7 +42,7 @@ spec:
### AutoscalingStrategy
`AutoscalingStrategy` is used and modified by the `NodeImage` controller to pause the `cluster-autoscaler` while an image update is in progress.
`AutoscalingStrategy` is used and modified by the `NodeVersion` controller to pause the `cluster-autoscaler` while an image update is in progress.
Example:
@ -60,7 +60,7 @@ spec:
### ScalingGroup
`ScalingGroup` represents one scaling group at the CSP. Constellation uses one scaling group for worker nodes and one for control-plane nodes.
The scaling group controller will automatically set the image used for newly created nodes to be the image set in the `NodeImage` Spec. On cluster creation, one instance of the `ScalingGroup` resource per scaling group at the CSP is created. It does not need to be updated manually.
The scaling group controller will automatically set the image used for newly created nodes to be the image set in the `NodeVersion` Spec. On cluster creation, one instance of the `ScalingGroup` resource per scaling group at the CSP is created. It does not need to be updated manually.
Example for GCP:

View file

@ -14,8 +14,8 @@ import (
type JoiningNodeSpec struct {
// Name of the node expected to join.
Name string `json:"name,omitempty"`
// ComponentsHash is the hash of the components that were sent to the node by the join service.
ComponentsHash string `json:"componentshash,omitempty"`
// ComponentsReference is the reference to the ConfigMap containing the components.
ComponentsReference string `json:"componentsreference,omitempty"`
// IsControlPlane is true if the node is a control plane node.
IsControlPlane bool `json:"iscontrolplane,omitempty"`
// Deadline is the time after which the joining node is considered to have failed.

View file

@ -11,16 +11,18 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// NodeImageSpec defines the desired state of NodeImage.
type NodeImageSpec struct {
// NodeVersionSpec defines the desired state of NodeVersion.
type NodeVersionSpec struct {
// ImageReference is the image to use for all nodes.
ImageReference string `json:"image,omitempty"`
// ImageVersion is the CSP independent version of the image to use for all nodes.
ImageVersion string `json:"imageVersion,omitempty"`
// KubernetesComponentsReference is a reference to the ConfigMap containing the Kubernetes components to use for all nodes.
KubernetesComponentsReference string `json:"kubernetesComponentsReference,omitempty"`
}
// NodeImageStatus defines the observed state of NodeImage.
type NodeImageStatus struct {
// NodeVersionStatus defines the observed state of NodeVersion.
type NodeVersionStatus struct {
// Outdated is a list of nodes that are using an outdated image.
Outdated []corev1.ObjectReference `json:"outdated,omitempty"`
// UpToDate is a list of nodes that are using the latest image and labels.
@ -47,24 +49,24 @@ type NodeImageStatus struct {
//+kubebuilder:subresource:status
//+kubebuilder:resource:scope=Cluster
// NodeImage is the Schema for the nodeimages API.
type NodeImage struct {
// NodeVersion is the Schema for the nodeversions API.
type NodeVersion struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec NodeImageSpec `json:"spec,omitempty"`
Status NodeImageStatus `json:"status,omitempty"`
Spec NodeVersionSpec `json:"spec,omitempty"`
Status NodeVersionStatus `json:"status,omitempty"`
}
//+kubebuilder:object:root=true
// NodeImageList contains a list of NodeImage.
type NodeImageList struct {
// NodeVersionList contains a list of NodeVersion.
type NodeVersionList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []NodeImage `json:"items"`
Items []NodeVersion `json:"items"`
}
func init() {
SchemeBuilder.Register(&NodeImage{}, &NodeImageList{})
SchemeBuilder.Register(&NodeVersion{}, &NodeVersionList{})
}

View file

@ -22,8 +22,8 @@ const (
// ScalingGroupSpec defines the desired state of ScalingGroup.
type ScalingGroupSpec struct {
// NodeImage is the name of the NodeImage resource.
NodeImage string `json:"nodeImage,omitempty"`
// NodeVersion is the name of the NodeVersion resource.
NodeVersion string `json:"nodeImage,omitempty"`
// GroupID is the CSP specific, canonical identifier of a scaling group.
GroupID string `json:"groupId,omitempty"`
// AutoscalerGroupName is name that is expected by the autoscaler.

View file

@ -201,7 +201,7 @@ func (in *JoiningNodeStatus) DeepCopy() *JoiningNodeStatus {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeImage) DeepCopyInto(out *NodeImage) {
func (in *NodeVersion) DeepCopyInto(out *NodeVersion) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
@ -209,18 +209,18 @@ func (in *NodeImage) DeepCopyInto(out *NodeImage) {
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeImage.
func (in *NodeImage) DeepCopy() *NodeImage {
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeVersion.
func (in *NodeVersion) DeepCopy() *NodeVersion {
if in == nil {
return nil
}
out := new(NodeImage)
out := new(NodeVersion)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *NodeImage) DeepCopyObject() runtime.Object {
func (in *NodeVersion) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
@ -228,31 +228,31 @@ func (in *NodeImage) DeepCopyObject() runtime.Object {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeImageList) DeepCopyInto(out *NodeImageList) {
func (in *NodeVersionList) DeepCopyInto(out *NodeVersionList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]NodeImage, len(*in))
*out = make([]NodeVersion, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeImageList.
func (in *NodeImageList) DeepCopy() *NodeImageList {
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeVersionList.
func (in *NodeVersionList) DeepCopy() *NodeVersionList {
if in == nil {
return nil
}
out := new(NodeImageList)
out := new(NodeVersionList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *NodeImageList) DeepCopyObject() runtime.Object {
func (in *NodeVersionList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
@ -260,22 +260,22 @@ func (in *NodeImageList) DeepCopyObject() runtime.Object {
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeImageSpec) DeepCopyInto(out *NodeImageSpec) {
func (in *NodeVersionSpec) DeepCopyInto(out *NodeVersionSpec) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeImageSpec.
func (in *NodeImageSpec) DeepCopy() *NodeImageSpec {
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeVersionSpec.
func (in *NodeVersionSpec) DeepCopy() *NodeVersionSpec {
if in == nil {
return nil
}
out := new(NodeImageSpec)
out := new(NodeVersionSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeImageStatus) DeepCopyInto(out *NodeImageStatus) {
func (in *NodeVersionStatus) DeepCopyInto(out *NodeVersionStatus) {
*out = *in
if in.Outdated != nil {
in, out := &in.Outdated, &out.Outdated
@ -326,12 +326,12 @@ func (in *NodeImageStatus) DeepCopyInto(out *NodeImageStatus) {
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeImageStatus.
func (in *NodeImageStatus) DeepCopy() *NodeImageStatus {
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeVersionStatus.
func (in *NodeVersionStatus) DeepCopy() *NodeVersionStatus {
if in == nil {
return nil
}
out := new(NodeImageStatus)
out := new(NodeVersionStatus)
in.DeepCopyInto(out)
return out
}

View file

@ -36,9 +36,9 @@ spec:
description: JoiningNodeSpec defines the components hash which the node
should be annotated with.
properties:
componentshash:
description: ComponentsHash is the hash of the components that were
sent to the node by the join service.
componentsreference:
description: ComponentsReference is the reference to the ConfigMap
containing the components.
type: string
deadline:
description: Deadline is the time after which the joining node is

View file

@ -5,20 +5,20 @@ metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.9.0
creationTimestamp: null
name: nodeimages.update.edgeless.systems
name: nodeversions.update.edgeless.systems
spec:
group: update.edgeless.systems
names:
kind: NodeImage
listKind: NodeImageList
plural: nodeimages
singular: nodeimage
kind: NodeVersion
listKind: NodeVersionList
plural: nodeversions
singular: nodeversion
scope: Cluster
versions:
- name: v1alpha1
schema:
openAPIV3Schema:
description: NodeImage is the Schema for the nodeimages API.
description: NodeVersion is the Schema for the nodeversions API.
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
@ -33,7 +33,7 @@ spec:
metadata:
type: object
spec:
description: NodeImageSpec defines the desired state of NodeImage.
description: NodeVersionSpec defines the desired state of NodeVersion.
properties:
image:
description: ImageReference is the image to use for all nodes.
@ -42,9 +42,13 @@ spec:
description: ImageVersion is the CSP independent version of the image
to use for all nodes.
type: string
kubernetesComponentsReference:
description: KubernetesComponentsReference is a reference to the ConfigMap
containing the Kubernetes components to use for all nodes.
type: string
type: object
status:
description: NodeImageStatus defines the observed state of NodeImage.
description: NodeVersionStatus defines the observed state of NodeVersion.
properties:
budget:
description: Budget is the amount of extra nodes that can be created

View file

@ -57,7 +57,7 @@ spec:
format: int32
type: integer
nodeImage:
description: NodeImage is the name of the NodeImage resource.
description: NodeVersion is the name of the NodeVersion resource.
type: string
role:
description: Role is the role of the nodes in the scaling group.

View file

@ -2,7 +2,7 @@
# since it depends on service name and namespace that are out of this kustomize package.
# It should be run by config/default
resources:
- bases/update.edgeless.systems_nodeimages.yaml
- bases/update.edgeless.systems_nodeversions.yaml
- bases/update.edgeless.systems_joiningnodes.yaml
- bases/update.edgeless.systems_autoscalingstrategies.yaml
- bases/update.edgeless.systems_scalinggroups.yaml
@ -12,7 +12,7 @@ resources:
patchesStrategicMerge:
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
# patches here are for enabling the conversion webhook for each CRD
#- patches/webhook_in_nodeimages.yaml
#- patches/webhook_in_nodeversions.yaml
#- patches/webhook_in_joiningnodes.yaml
#- patches/webhook_in_autoscalingstrategies.yaml
#- patches/webhook_in_scalinggroups.yaml
@ -21,7 +21,7 @@ patchesStrategicMerge:
# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
# patches here are for enabling the CA injection for each CRD
#- patches/cainjection_in_nodeimages.yaml
#- patches/cainjection_in_nodeversions.yaml
#- patches/cainjection_in_joiningnodes.yaml
#- patches/cainjection_in_autoscalingstrategies.yaml
#- patches/cainjection_in_scalinggroups.yaml

View file

@ -4,4 +4,4 @@ kind: CustomResourceDefinition
metadata:
annotations:
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
name: nodeimages.update.edgeless.systems
name: nodeversions.update.edgeless.systems

View file

@ -2,7 +2,7 @@
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
name: nodeimages.update.edgeless.systems
name: nodeversions.update.edgeless.systems
spec:
conversion:
strategy: Webhook

View file

@ -5,6 +5,12 @@ generatorOptions:
disableNameSuffixHash: true
configMapGenerator:
- name: manager-config
files:
- files:
- controller_manager_config.yaml
name: manager-config
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
images:
- name: controller
newName: ghcr.io/edgelesssys/constellation/node-operator
newTag: v0.0.1

View file

@ -16,10 +16,10 @@ spec:
kind: AutoscalingStrategy
name: autoscalingstrategies.update.edgeless.systems
version: v1alpha1
- description: NodeImage is the Schema for the nodeimages API.
displayName: Node Image
kind: NodeImage
name: nodeimages.update.edgeless.systems
- description: NodeVersion is the Schema for the nodeversions API.
displayName: Node Version
kind: NodeVersion
name: nodeversions.update.edgeless.systems
version: v1alpha1
- description: PendingNode is the Schema for the pendingnodes API.
displayName: Pending Node

View file

@ -1,13 +1,13 @@
# permissions for end users to edit nodeimages.
# permissions for end users to edit nodeversions.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: nodeimage-editor-role
name: nodeversion-editor-role
rules:
- apiGroups:
- update.edgeless.systems
resources:
- nodeimages
- nodeversions
verbs:
- create
- delete
@ -19,6 +19,6 @@ rules:
- apiGroups:
- update.edgeless.systems
resources:
- nodeimages/status
- nodeversions/status
verbs:
- get

View file

@ -1,13 +1,13 @@
# permissions for end users to view nodeimages.
# permissions for end users to view nodeversions.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: nodeimage-viewer-role
name: nodeversion-viewer-role
rules:
- apiGroups:
- update.edgeless.systems
resources:
- nodeimages
- nodeversions
verbs:
- get
- list
@ -15,6 +15,6 @@ rules:
- apiGroups:
- update.edgeless.systems
resources:
- nodeimages/status
- nodeversions/status
verbs:
- get

View file

@ -101,7 +101,7 @@ rules:
- apiGroups:
- update.edgeless.systems
resources:
- nodeimage
- nodeversion
verbs:
- get
- list
@ -109,7 +109,13 @@ rules:
- apiGroups:
- update.edgeless.systems
resources:
- nodeimages
- nodeversion/status
verbs:
- get
- apiGroups:
- update.edgeless.systems
resources:
- nodeversions
verbs:
- create
- delete
@ -121,13 +127,13 @@ rules:
- apiGroups:
- update.edgeless.systems
resources:
- nodeimages/finalizers
- nodeversions/finalizers
verbs:
- update
- apiGroups:
- update.edgeless.systems
resources:
- nodeimages/status
- nodeversions/status
verbs:
- get
- patch

View file

@ -1,6 +1,6 @@
## Append samples you want in your CSV to this file as resources ##
resources:
- update_v1alpha1_nodeimage.yaml
- update_v1alpha1_nodeversion.yaml
- update_v1alpha1_autoscalingstrategy.yaml
- update_v1alpha1_scalinggroup.yaml
- update_v1alpha1_pendingnode.yaml

View file

@ -1,5 +1,5 @@
apiVersion: update.edgeless.systems/v1alpha1
kind: NodeImage
kind: NodeVersion
metadata:
name: constellation-os-azure
namespace: kube-system
@ -7,7 +7,7 @@ spec:
image: "/subscriptions/<subscription-id>/resourceGroups/<resource-group>/providers/Microsoft.Compute/galleries/<gallery-name>/images/<image-definition-name>/versions/<version>"
---
apiVersion: update.edgeless.systems/v1alpha1
kind: NodeImage
kind: NodeVersion
metadata:
name: constellation-os-gcp
namespace: kube-system

View file

@ -26,8 +26,8 @@ import (
)
const (
// NodeKubernetesComponentsHashAnnotationKey is the name of the annotation holding the hash of the installed components of this node.
NodeKubernetesComponentsHashAnnotationKey = "updates.edgeless.systems/kubernetes-components-hash"
// NodeKubernetesComponentsReferenceAnnotationKey is the name of the annotation holding the reference to the ConfigMap listing all K8s components.
NodeKubernetesComponentsReferenceAnnotationKey = "constellation.edgeless.systems/kubernetes-components"
joiningNodeNameKey = ".spec.name"
)
@ -76,7 +76,7 @@ func (r *JoiningNodesReconciler) Reconcile(ctx context.Context, req ctrl.Request
if node.Annotations == nil {
node.Annotations = map[string]string{}
}
node.Annotations[NodeKubernetesComponentsHashAnnotationKey] = joiningNode.Spec.ComponentsHash
node.Annotations[NodeKubernetesComponentsReferenceAnnotationKey] = joiningNode.Spec.ComponentsReference
return r.Update(ctx, &node)
})
if err != nil {

View file

@ -23,12 +23,12 @@ import (
var _ = Describe("JoiningNode controller", func() {
const (
nodeName1 = "node-name-1"
nodeName2 = "node-name-2"
nodeName3 = "node-name-3"
componentsHash1 = "test-hash-1"
componentsHash2 = "test-hash-2"
componentsHash3 = "test-hash-3"
nodeName1 = "node-name-1"
nodeName2 = "node-name-2"
nodeName3 = "node-name-3"
ComponentsReference1 = "test-ref-1"
ComponentsReference2 = "test-ref-2"
ComponentsReference3 = "test-ref-3"
timeout = time.Second * 20
duration = time.Second * 2
@ -47,8 +47,8 @@ var _ = Describe("JoiningNode controller", func() {
Name: nodeName1,
},
Spec: updatev1alpha1.JoiningNodeSpec{
Name: nodeName1,
ComponentsHash: componentsHash1,
Name: nodeName1,
ComponentsReference: ComponentsReference1,
},
}
Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed())
@ -57,7 +57,7 @@ var _ = Describe("JoiningNode controller", func() {
return k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdJoiningNode)
}, timeout, interval).Should(Succeed())
Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName1))
Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash1))
Expect(createdJoiningNode.Spec.ComponentsReference).Should(Equal(ComponentsReference1))
By("creating a node")
node := &corev1.Node{
@ -80,8 +80,8 @@ var _ = Describe("JoiningNode controller", func() {
By("annotating the node")
Eventually(func() string {
_ = k8sClient.Get(ctx, types.NamespacedName{Name: nodeName1}, createdNode)
return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey]
}, timeout, interval).Should(Equal(componentsHash1))
return createdNode.Annotations[NodeKubernetesComponentsReferenceAnnotationKey]
}, timeout, interval).Should(Equal(ComponentsReference1))
By("deleting the joining node resource")
Eventually(func() error {
@ -119,8 +119,8 @@ var _ = Describe("JoiningNode controller", func() {
Name: nodeName2,
},
Spec: updatev1alpha1.JoiningNodeSpec{
Name: nodeName2,
ComponentsHash: componentsHash2,
Name: nodeName2,
ComponentsReference: ComponentsReference2,
},
}
Expect(k8sClient.Create(ctx, joiningNode)).Should(Succeed())
@ -129,13 +129,13 @@ var _ = Describe("JoiningNode controller", func() {
return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
}, timeout, interval).Should(Succeed())
Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName2))
Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash2))
Expect(createdJoiningNode.Spec.ComponentsReference).Should(Equal(ComponentsReference2))
By("annotating the node")
Eventually(func() string {
_ = k8sClient.Get(ctx, types.NamespacedName{Name: createdNode.Name}, createdNode)
return createdNode.Annotations[NodeKubernetesComponentsHashAnnotationKey]
}, timeout, interval).Should(Equal(componentsHash2))
return createdNode.Annotations[NodeKubernetesComponentsReferenceAnnotationKey]
}, timeout, interval).Should(Equal(ComponentsReference2))
By("deleting the joining node resource")
Eventually(func() error {
@ -154,8 +154,8 @@ var _ = Describe("JoiningNode controller", func() {
Name: nodeName3,
},
Spec: updatev1alpha1.JoiningNodeSpec{
Name: nodeName3,
ComponentsHash: componentsHash3,
Name: nodeName3,
ComponentsReference: ComponentsReference3,
// create without deadline first
},
}
@ -165,7 +165,7 @@ var _ = Describe("JoiningNode controller", func() {
return k8sClient.Get(ctx, types.NamespacedName{Name: joiningNode.Name}, createdJoiningNode)
}, timeout, interval).Should(Succeed())
Expect(createdJoiningNode.Spec.Name).Should(Equal(nodeName3))
Expect(createdJoiningNode.Spec.ComponentsHash).Should(Equal(componentsHash3))
Expect(createdJoiningNode.Spec.ComponentsReference).Should(Equal(ComponentsReference3))
By("setting the deadline to the past")
createdJoiningNode.Spec.Deadline = &metav1.Time{Time: fakes.clock.Now().Add(-time.Second)}

View file

@ -38,29 +38,29 @@ const (
// nodeJoinTimeout is the time limit pending nodes have to join the cluster before being terminated.
nodeJoinTimeout = time.Minute * 30
// nodeLeaveTimeout is the time limit pending nodes have to leave the cluster and being terminated.
nodeLeaveTimeout = time.Minute
donorAnnotation = "constellation.edgeless.systems/donor"
heirAnnotation = "constellation.edgeless.systems/heir"
scalingGroupAnnotation = "constellation.edgeless.systems/scaling-group-id"
nodeImageAnnotation = "constellation.edgeless.systems/node-image"
obsoleteAnnotation = "constellation.edgeless.systems/obsolete"
conditionNodeImageUpToDateReason = "NodeImagesUpToDate"
conditionNodeImageUpToDateMessage = "Node image of every node is up to date"
conditionNodeImageOutOfDateReason = "NodeImagesOutOfDate"
conditionNodeImageOutOfDateMessage = "Some node images are out of date"
nodeLeaveTimeout = time.Minute
donorAnnotation = "constellation.edgeless.systems/donor"
heirAnnotation = "constellation.edgeless.systems/heir"
scalingGroupAnnotation = "constellation.edgeless.systems/scaling-group-id"
nodeImageAnnotation = "constellation.edgeless.systems/node-image"
obsoleteAnnotation = "constellation.edgeless.systems/obsolete"
conditionNodeVersionUpToDateReason = "NodeVersionsUpToDate"
conditionNodeVersionUpToDateMessage = "Node version of every node is up to date"
conditionNodeVersionOutOfDateReason = "NodeVersionsOutOfDate"
conditionNodeVersionOutOfDateMessage = "Some node versions are out of date"
)
// NodeImageReconciler reconciles a NodeImage object.
type NodeImageReconciler struct {
// NodeVersionReconciler reconciles a NodeVersion object.
type NodeVersionReconciler struct {
nodeReplacer
etcdRemover
client.Client
Scheme *runtime.Scheme
}
// NewNodeImageReconciler creates a new NodeImageReconciler.
func NewNodeImageReconciler(nodeReplacer nodeReplacer, etcdRemover etcdRemover, client client.Client, scheme *runtime.Scheme) *NodeImageReconciler {
return &NodeImageReconciler{
// NewNodeVersionReconciler creates a new NodeVersionReconciler.
func NewNodeVersionReconciler(nodeReplacer nodeReplacer, etcdRemover etcdRemover, client client.Client, scheme *runtime.Scheme) *NodeVersionReconciler {
return &NodeVersionReconciler{
nodeReplacer: nodeReplacer,
etcdRemover: etcdRemover,
Client: client,
@ -68,20 +68,20 @@ func NewNodeImageReconciler(nodeReplacer nodeReplacer, etcdRemover etcdRemover,
}
}
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeimages,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeimages/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeimages/finalizers,verbs=update
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeversions,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeversions/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeversions/finalizers,verbs=update
//+kubebuilder:rbac:groups=nodemaintenance.medik8s.io,resources=nodemaintenances,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups="",resources=nodes,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups="",resources=nodes/status,verbs=get
// Reconcile replaces outdated nodes (using an old image) with new nodes (using a new image) as specified in the NodeImage spec.
func (r *NodeImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
// Reconcile replaces outdated nodes (using an old image) with new nodes (using a new image) as specified in the NodeVersion spec.
func (r *NodeVersionReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logr := log.FromContext(ctx)
logr.Info("Reconciling NodeImage")
logr.Info("Reconciling NodeVersion")
var desiredNodeImage updatev1alpha1.NodeImage
if err := r.Get(ctx, req.NamespacedName, &desiredNodeImage); err != nil {
var desiredNodeVersion updatev1alpha1.NodeVersion
if err := r.Get(ctx, req.NamespacedName, &desiredNodeVersion); err != nil {
return ctrl.Result{}, client.IgnoreNotFound(err)
}
// get list of autoscaling strategies
@ -122,7 +122,7 @@ func (r *NodeImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
scalingGroupByID[strings.ToLower(scalingGroup.Spec.GroupID)] = scalingGroup
}
annotatedNodes, invalidNodes := r.annotateNodes(ctx, nodeList.Items)
groups := groupNodes(annotatedNodes, pendingNodeList.Items, desiredNodeImage.Spec.ImageReference)
groups := groupNodes(annotatedNodes, pendingNodeList.Items, desiredNodeVersion.Spec.ImageReference, desiredNodeVersion.Spec.KubernetesComponentsReference)
logr.Info("Grouped nodes",
"outdatedNodes", len(groups.Outdated),
@ -147,7 +147,7 @@ func (r *NodeImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
logr.Info("Budget for new nodes", "newNodesBudget", newNodesBudget)
status := nodeImageStatus(r.Scheme, groups, pendingNodeList.Items, invalidNodes, newNodesBudget)
status := nodeVersionStatus(r.Scheme, groups, pendingNodeList.Items, invalidNodes, newNodesBudget)
if err := r.tryUpdateStatus(ctx, req.NamespacedName, status); err != nil {
logr.Error(err, "Updating status")
}
@ -159,20 +159,20 @@ func (r *NodeImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
if allNodesUpToDate {
logr.Info("All node images up to date")
logr.Info("All node versions up to date")
return ctrl.Result{}, nil
}
// should requeue is set if a node is deleted
var shouldRequeue bool
// find pairs of mint nodes and outdated nodes in the same scaling group to become donor & heir
replacementPairs := r.pairDonorsAndHeirs(ctx, &desiredNodeImage, groups.Outdated, groups.Mint)
replacementPairs := r.pairDonorsAndHeirs(ctx, &desiredNodeVersion, groups.Outdated, groups.Mint)
// extend replacement pairs to include existing pairs of donors and heirs
replacementPairs = r.matchDonorsAndHeirs(ctx, replacementPairs, groups.Donors, groups.Heirs)
// replace donor nodes by heirs
for _, pair := range replacementPairs {
logr.Info("Replacing node", "donorNode", pair.donor.Name, "heirNode", pair.heir.Name)
done, err := r.replaceNode(ctx, &desiredNodeImage, pair)
done, err := r.replaceNode(ctx, &desiredNodeVersion, pair)
if err != nil {
logr.Error(err, "Replacing node")
return ctrl.Result{}, err
@ -192,13 +192,13 @@ func (r *NodeImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
return ctrl.Result{Requeue: shouldRequeue}, nil
}
newNodeConfig := newNodeConfig{desiredNodeImage, groups.Outdated, pendingNodeList.Items, scalingGroupByID, newNodesBudget}
newNodeConfig := newNodeConfig{desiredNodeVersion, groups.Outdated, pendingNodeList.Items, scalingGroupByID, newNodesBudget}
if err := r.createNewNodes(ctx, newNodeConfig); err != nil {
return ctrl.Result{Requeue: shouldRequeue}, nil
}
// cleanup obsolete nodes
for _, node := range groups.Obsolete {
done, err := r.deleteNode(ctx, &desiredNodeImage, node)
done, err := r.deleteNode(ctx, &desiredNodeVersion, node)
if err != nil {
logr.Error(err, "Unable to remove obsolete node")
}
@ -211,9 +211,9 @@ func (r *NodeImageReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
}
// SetupWithManager sets up the controller with the Manager.
func (r *NodeImageReconciler) SetupWithManager(mgr ctrl.Manager) error {
func (r *NodeVersionReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&updatev1alpha1.NodeImage{}).
For(&updatev1alpha1.NodeVersion{}).
Watches(
&source.Kind{Type: &updatev1alpha1.ScalingGroup{}},
handler.EnqueueRequestsFromMapFunc(r.findObjectsForScalingGroup),
@ -221,17 +221,17 @@ func (r *NodeImageReconciler) SetupWithManager(mgr ctrl.Manager) error {
).
Watches(
&source.Kind{Type: &updatev1alpha1.AutoscalingStrategy{}},
handler.EnqueueRequestsFromMapFunc(r.findAllNodeImages),
handler.EnqueueRequestsFromMapFunc(r.findAllNodeVersions),
builder.WithPredicates(autoscalerEnabledStatusChangedPredicate()),
).
Watches(
&source.Kind{Type: &corev1.Node{}},
handler.EnqueueRequestsFromMapFunc(r.findAllNodeImages),
handler.EnqueueRequestsFromMapFunc(r.findAllNodeVersions),
builder.WithPredicates(nodeReadyPredicate()),
).
Watches(
&source.Kind{Type: &nodemaintenancev1beta1.NodeMaintenance{}},
handler.EnqueueRequestsFromMapFunc(r.findAllNodeImages),
handler.EnqueueRequestsFromMapFunc(r.findAllNodeVersions),
builder.WithPredicates(nodeMaintenanceSucceededPredicate()),
).
Owns(&updatev1alpha1.PendingNode{}).
@ -239,7 +239,7 @@ func (r *NodeImageReconciler) SetupWithManager(mgr ctrl.Manager) error {
}
// annotateNodes takes all nodes of the cluster and annotates them with the scaling group they are in and the image they are using.
func (r *NodeImageReconciler) annotateNodes(ctx context.Context, nodes []corev1.Node) (annotatedNodes, invalidNodes []corev1.Node) {
func (r *NodeVersionReconciler) annotateNodes(ctx context.Context, nodes []corev1.Node) (annotatedNodes, invalidNodes []corev1.Node) {
logr := log.FromContext(ctx)
for _, node := range nodes {
annotations := make(map[string]string)
@ -285,7 +285,7 @@ func (r *NodeImageReconciler) annotateNodes(ctx context.Context, nodes []corev1.
// pairDonorsAndHeirs takes a list of outdated nodes (that do not yet have a heir node) and a list of mint nodes (nodes using the latest image) and pairs matching nodes to become donor and heir.
// outdatedNodes is also updated with heir annotations.
func (r *NodeImageReconciler) pairDonorsAndHeirs(ctx context.Context, controller metav1.Object, outdatedNodes []corev1.Node, mintNodes []mintNode) []replacementPair {
func (r *NodeVersionReconciler) pairDonorsAndHeirs(ctx context.Context, controller metav1.Object, outdatedNodes []corev1.Node, mintNodes []mintNode) []replacementPair {
logr := log.FromContext(ctx)
var pairs []replacementPair
for _, mintNode := range mintNodes {
@ -345,7 +345,7 @@ func (r *NodeImageReconciler) pairDonorsAndHeirs(ctx context.Context, controller
// matchDonorsAndHeirs takes separate lists of donors and heirs and matches each heir to its previously chosen donor.
// a list of replacement pairs is returned.
// donors and heirs with invalid pair references are cleaned up (the donor/heir annotations gets removed).
func (r *NodeImageReconciler) matchDonorsAndHeirs(ctx context.Context, pairs []replacementPair, donors, heirs []corev1.Node) []replacementPair {
func (r *NodeVersionReconciler) matchDonorsAndHeirs(ctx context.Context, pairs []replacementPair, donors, heirs []corev1.Node) []replacementPair {
logr := log.FromContext(ctx)
for _, heir := range heirs {
var foundPair bool
@ -389,7 +389,7 @@ func (r *NodeImageReconciler) matchDonorsAndHeirs(ctx context.Context, pairs []r
}
// ensureAutoscaling will ensure that the autoscaling is enabled or disabled as needed.
func (r *NodeImageReconciler) ensureAutoscaling(ctx context.Context, autoscalingEnabled bool, wantAutoscalingEnabled bool) error {
func (r *NodeVersionReconciler) ensureAutoscaling(ctx context.Context, autoscalingEnabled bool, wantAutoscalingEnabled bool) error {
if autoscalingEnabled == wantAutoscalingEnabled {
return nil
}
@ -418,7 +418,7 @@ func (r *NodeImageReconciler) ensureAutoscaling(ctx context.Context, autoscaling
// Labels are copied from the donor node to the heir node.
// Readiness of the heir node is awaited.
// Deletion of the donor node is scheduled.
func (r *NodeImageReconciler) replaceNode(ctx context.Context, controller metav1.Object, pair replacementPair) (bool, error) {
func (r *NodeVersionReconciler) replaceNode(ctx context.Context, controller metav1.Object, pair replacementPair) (bool, error) {
logr := log.FromContext(ctx)
if !reflect.DeepEqual(nodeutil.FilterLabels(pair.donor.Labels), nodeutil.FilterLabels(pair.heir.Labels)) {
if err := r.copyNodeLabels(ctx, pair.donor.Name, pair.heir.Name); err != nil {
@ -434,7 +434,7 @@ func (r *NodeImageReconciler) replaceNode(ctx context.Context, controller metav1
}
// deleteNode safely removes a node from the cluster and issues termination of the node by the CSP.
func (r *NodeImageReconciler) deleteNode(ctx context.Context, controller metav1.Object, node corev1.Node) (bool, error) {
func (r *NodeVersionReconciler) deleteNode(ctx context.Context, controller metav1.Object, node corev1.Node) (bool, error) {
logr := log.FromContext(ctx)
// cordon & drain node using node-maintenance-operator
var foundNodeMaintenance nodemaintenancev1beta1.NodeMaintenance
@ -509,7 +509,7 @@ func (r *NodeImageReconciler) deleteNode(ctx context.Context, controller metav1.
}
// createNewNodes creates new nodes using up to date images as replacement for outdated nodes.
func (r *NodeImageReconciler) createNewNodes(ctx context.Context, config newNodeConfig) error {
func (r *NodeVersionReconciler) createNewNodes(ctx context.Context, config newNodeConfig) error {
logr := log.FromContext(ctx)
if config.newNodesBudget < 1 || len(config.outdatedNodes) == 0 {
return nil
@ -543,8 +543,8 @@ func (r *NodeImageReconciler) createNewNodes(ctx context.Context, config newNode
logr.Info("Scaling group does not have matching resource", "scalingGroup", scalingGroupID, "scalingGroups", config.scalingGroupByID)
continue
}
if !strings.EqualFold(scalingGroup.Status.ImageReference, config.desiredNodeImage.Spec.ImageReference) {
logr.Info("Scaling group does not use latest image", "scalingGroup", scalingGroupID, "usedImage", scalingGroup.Status.ImageReference, "wantedImage", config.desiredNodeImage.Spec.ImageReference)
if !strings.EqualFold(scalingGroup.Status.ImageReference, config.desiredNodeVersion.Spec.ImageReference) {
logr.Info("Scaling group does not use latest image", "scalingGroup", scalingGroupID, "usedImage", scalingGroup.Status.ImageReference, "wantedImage", config.desiredNodeVersion.Spec.ImageReference)
continue
}
if requiredNodesPerScalingGroup[scalingGroupID] == 0 {
@ -573,7 +573,7 @@ func (r *NodeImageReconciler) createNewNodes(ctx context.Context, config newNode
Deadline: &deadline,
},
}
if err := ctrl.SetControllerReference(&config.desiredNodeImage, pendingNode, r.Scheme); err != nil {
if err := ctrl.SetControllerReference(&config.desiredNodeVersion, pendingNode, r.Scheme); err != nil {
return err
}
if err := r.Create(ctx, pendingNode); err != nil {
@ -588,7 +588,7 @@ func (r *NodeImageReconciler) createNewNodes(ctx context.Context, config newNode
}
// patchNodeAnnotations attempts to patch node annotations in a retry loop.
func (r *NodeImageReconciler) patchNodeAnnotations(ctx context.Context, nodeName string, annotations map[string]string) error {
func (r *NodeVersionReconciler) patchNodeAnnotations(ctx context.Context, nodeName string, annotations map[string]string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
var node corev1.Node
if err := r.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil {
@ -601,7 +601,7 @@ func (r *NodeImageReconciler) patchNodeAnnotations(ctx context.Context, nodeName
}
// patchNodeAnnotations attempts to remove node annotations using a patch in a retry loop.
func (r *NodeImageReconciler) patchUnsetNodeAnnotations(ctx context.Context, nodeName string, annotationKeys []string) error {
func (r *NodeVersionReconciler) patchUnsetNodeAnnotations(ctx context.Context, nodeName string, annotationKeys []string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
var node corev1.Node
if err := r.Get(ctx, types.NamespacedName{Name: nodeName}, &node); err != nil {
@ -614,7 +614,7 @@ func (r *NodeImageReconciler) patchUnsetNodeAnnotations(ctx context.Context, nod
}
// copyNodeLabels attempts to copy all node labels (except for reserved labels) from one node to another in a retry loop.
func (r *NodeImageReconciler) copyNodeLabels(ctx context.Context, oldNodeName, newNodeName string) error {
func (r *NodeVersionReconciler) copyNodeLabels(ctx context.Context, oldNodeName, newNodeName string) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
var oldNode corev1.Node
if err := r.Get(ctx, types.NamespacedName{Name: oldNodeName}, &oldNode); err != nil {
@ -630,35 +630,35 @@ func (r *NodeImageReconciler) copyNodeLabels(ctx context.Context, oldNodeName, n
})
}
// tryUpdateStatus attempts to update the NodeImage status field in a retry loop.
func (r *NodeImageReconciler) tryUpdateStatus(ctx context.Context, name types.NamespacedName, status updatev1alpha1.NodeImageStatus) error {
// tryUpdateStatus attempts to update the NodeVersion status field in a retry loop.
func (r *NodeVersionReconciler) tryUpdateStatus(ctx context.Context, name types.NamespacedName, status updatev1alpha1.NodeVersionStatus) error {
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
var nodeImage updatev1alpha1.NodeImage
if err := r.Get(ctx, name, &nodeImage); err != nil {
var nodeVersion updatev1alpha1.NodeVersion
if err := r.Get(ctx, name, &nodeVersion); err != nil {
return err
}
nodeImage.Status = *status.DeepCopy()
if err := r.Status().Update(ctx, &nodeImage); err != nil {
nodeVersion.Status = *status.DeepCopy()
if err := r.Status().Update(ctx, &nodeVersion); err != nil {
return err
}
return nil
})
}
// nodeImageStatus generates the NodeImage.Status field given node groups and the budget for new nodes.
func nodeImageStatus(scheme *runtime.Scheme, groups nodeGroups, pendingNodes []updatev1alpha1.PendingNode, invalidNodes []corev1.Node, newNodesBudget int) updatev1alpha1.NodeImageStatus {
var status updatev1alpha1.NodeImageStatus
// nodeVersionStatus generates the NodeVersion.Status field given node groups and the budget for new nodes.
func nodeVersionStatus(scheme *runtime.Scheme, groups nodeGroups, pendingNodes []updatev1alpha1.PendingNode, invalidNodes []corev1.Node, newNodesBudget int) updatev1alpha1.NodeVersionStatus {
var status updatev1alpha1.NodeVersionStatus
outdatedCondition := metav1.Condition{
Type: updatev1alpha1.ConditionOutdated,
}
if len(groups.Outdated)+len(groups.Heirs)+len(pendingNodes)+len(groups.Obsolete) == 0 {
outdatedCondition.Status = metav1.ConditionFalse
outdatedCondition.Reason = conditionNodeImageUpToDateReason
outdatedCondition.Message = conditionNodeImageUpToDateMessage
outdatedCondition.Reason = conditionNodeVersionUpToDateReason
outdatedCondition.Message = conditionNodeVersionUpToDateMessage
} else {
outdatedCondition.Status = metav1.ConditionTrue
outdatedCondition.Reason = conditionNodeImageOutOfDateReason
outdatedCondition.Message = conditionNodeImageOutOfDateMessage
outdatedCondition.Reason = conditionNodeVersionOutOfDateReason
outdatedCondition.Message = conditionNodeVersionOutOfDateMessage
}
meta.SetStatusCondition(&status.Conditions, outdatedCondition)
for _, node := range groups.Outdated {
@ -739,20 +739,20 @@ type replacementPair struct {
// every properly annotated kubernetes node can be placed in exactly one of the sets.
type nodeGroups struct {
// Outdated nodes are nodes that
// do not use the most recent image AND
// do not use the most recent version AND
// are not yet a donor to an up to date heir node
Outdated,
// UpToDate nodes are nodes that
// use the most recent image,
// use the most recent version,
// are not an heir to an outdated donor node AND
// are not mint nodes
UpToDate,
// Donors are nodes that
// do not use the most recent image AND
// do not use the most recent version AND
// are paired up with an up to date heir node
Donors,
// Heirs are nodes that
// use the most recent image AND
// use the most recent version AND
// are paired up with an outdated donor node
Heirs,
// Obsolete nodes are nodes that
@ -761,21 +761,22 @@ type nodeGroups struct {
// They will be cleaned up by the operator.
Obsolete []corev1.Node
// Mint nodes are nodes that
// use the most recent image AND
// use the most recent version AND
// were created by the operator as replacements (heirs)
// and are awaiting pairing up with a donor node.
Mint []mintNode
}
// groupNodes classifies nodes by placing each into exactly one group.
func groupNodes(nodes []corev1.Node, pendingNodes []updatev1alpha1.PendingNode, latestImageReference string) nodeGroups {
func groupNodes(nodes []corev1.Node, pendingNodes []updatev1alpha1.PendingNode, latestImageReference string, latestK8sComponentsReference string) nodeGroups {
groups := nodeGroups{}
for _, node := range nodes {
if node.Annotations[obsoleteAnnotation] == "true" {
groups.Obsolete = append(groups.Obsolete, node)
continue
}
if !strings.EqualFold(node.Annotations[nodeImageAnnotation], latestImageReference) {
if !strings.EqualFold(node.Annotations[nodeImageAnnotation], latestImageReference) ||
!strings.EqualFold(node.Annotations[NodeKubernetesComponentsReferenceAnnotationKey], latestK8sComponentsReference) {
if heir := node.Annotations[heirAnnotation]; heir != "" {
groups.Donors = append(groups.Donors, node)
} else {
@ -816,9 +817,9 @@ type etcdRemover interface {
}
type newNodeConfig struct {
desiredNodeImage updatev1alpha1.NodeImage
outdatedNodes []corev1.Node
pendingNodes []updatev1alpha1.PendingNode
scalingGroupByID map[string]updatev1alpha1.ScalingGroup
newNodesBudget int
desiredNodeVersion updatev1alpha1.NodeVersion
outdatedNodes []corev1.Node
pendingNodes []updatev1alpha1.PendingNode
scalingGroupByID map[string]updatev1alpha1.ScalingGroup
newNodesBudget int
}

View file

@ -23,15 +23,15 @@ import (
nodemaintenancev1beta1 "github.com/medik8s/node-maintenance-operator/api/v1beta1"
)
var _ = Describe("NodeImage controller", func() {
var _ = Describe("NodeVersion controller", func() {
// Define utility constants for object names and testing timeouts/durations and intervals.
const (
nodeImageResourceName = "nodeimage"
firstNodeName = "node-1"
secondNodeName = "node-2"
firstImage = "image-1"
secondImage = "image-2"
scalingGroupID = "scaling-group"
nodeVersionResourceName = "nodeversion"
firstNodeName = "node-1"
secondNodeName = "node-2"
firstVersion = "version-1"
secondVersion = "version-2"
scalingGroupID = "scaling-group"
timeout = time.Second * 20
duration = time.Second * 2
@ -40,29 +40,29 @@ var _ = Describe("NodeImage controller", func() {
firstNodeLookupKey := types.NamespacedName{Name: firstNodeName}
secondNodeLookupKey := types.NamespacedName{Name: secondNodeName}
nodeImageLookupKey := types.NamespacedName{Name: nodeImageResourceName}
nodeVersionLookupKey := types.NamespacedName{Name: nodeVersionResourceName}
scalingGroupLookupKey := types.NamespacedName{Name: scalingGroupID}
joiningPendingNodeLookupKey := types.NamespacedName{Name: secondNodeName}
nodeMaintenanceLookupKey := types.NamespacedName{Name: firstNodeName}
Context("When updating the cluster-wide node image", func() {
Context("When updating the cluster-wide node version", func() {
It("Should update every node in the cluster", func() {
By("creating a node image resource specifying the first node image")
Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupID, firstImage)).Should(Succeed())
nodeImage := &updatev1alpha1.NodeImage{
By("creating a node version resource specifying the first node version")
Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupID, firstVersion)).Should(Succeed())
nodeVersion := &updatev1alpha1.NodeVersion{
TypeMeta: metav1.TypeMeta{
APIVersion: "update.edgeless.systems/v1alpha1",
Kind: "NodeImage",
Kind: "NodeVersion",
},
ObjectMeta: metav1.ObjectMeta{
Name: nodeImageResourceName,
Name: nodeVersionResourceName,
},
Spec: updatev1alpha1.NodeImageSpec{ImageReference: firstImage},
Spec: updatev1alpha1.NodeVersionSpec{ImageReference: firstVersion},
}
Expect(k8sClient.Create(ctx, nodeImage)).Should(Succeed())
Expect(k8sClient.Create(ctx, nodeVersion)).Should(Succeed())
By("creating a node resource using the first node image")
fakes.nodeReplacer.setNodeImage(firstNodeName, firstImage)
fakes.nodeReplacer.setNodeImage(firstNodeName, firstVersion)
fakes.nodeReplacer.setScalingGroupID(firstNodeName, scalingGroupID)
firstNode := &corev1.Node{
TypeMeta: metav1.TypeMeta{
@ -82,13 +82,13 @@ var _ = Describe("NodeImage controller", func() {
Expect(k8sClient.Create(ctx, firstNode)).Should(Succeed())
By("creating a scaling group resource using the first node image")
Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupID, firstImage)).Should(Succeed())
Expect(fakes.scalingGroupUpdater.SetScalingGroupImage(ctx, scalingGroupID, firstVersion)).Should(Succeed())
scalingGroup := &updatev1alpha1.ScalingGroup{
ObjectMeta: metav1.ObjectMeta{
Name: scalingGroupID,
},
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: nodeImageResourceName,
NodeVersion: nodeVersionResourceName,
GroupID: scalingGroupID,
Autoscaling: true,
},
@ -146,24 +146,24 @@ var _ = Describe("NodeImage controller", func() {
By("checking that all nodes are up-to-date")
Eventually(func() int {
if err := k8sClient.Get(ctx, nodeImageLookupKey, nodeImage); err != nil {
if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil {
return 0
}
return len(nodeImage.Status.UpToDate)
return len(nodeVersion.Status.UpToDate)
}, timeout, interval).Should(Equal(1))
By("updating the node image to the second image")
fakes.nodeStateGetter.setNodeState(updatev1alpha1.NodeStateReady)
fakes.nodeReplacer.setCreatedNode(secondNodeName, secondNodeName, nil)
nodeImage.Spec.ImageReference = secondImage
Expect(k8sClient.Update(ctx, nodeImage)).Should(Succeed())
nodeVersion.Spec.ImageReference = secondVersion
Expect(k8sClient.Update(ctx, nodeVersion)).Should(Succeed())
By("checking that there is an outdated node in the status")
Eventually(func() int {
if err := k8sClient.Get(ctx, nodeImageLookupKey, nodeImage); err != nil {
if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil {
return 0
}
return len(nodeImage.Status.Outdated)
return len(nodeVersion.Status.Outdated)
}, timeout, interval).Should(Equal(1))
By("checking that the scaling group is up to date")
@ -172,7 +172,7 @@ var _ = Describe("NodeImage controller", func() {
return ""
}
return scalingGroup.Status.ImageReference
}, timeout, interval).Should(Equal(secondImage))
}, timeout, interval).Should(Equal(secondVersion))
By("checking that a pending node is created")
pendingNode := &updatev1alpha1.PendingNode{}
@ -184,14 +184,14 @@ var _ = Describe("NodeImage controller", func() {
return pendingNode.Status.CSPNodeState
}).Should(Equal(updatev1alpha1.NodeStateReady))
Eventually(func() int {
if err := k8sClient.Get(ctx, nodeImageLookupKey, nodeImage); err != nil {
if err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion); err != nil {
return 0
}
return len(nodeImage.Status.Pending)
return len(nodeVersion.Status.Pending)
}, timeout, interval).Should(Equal(1))
By("creating a new node resource using the second node image")
fakes.nodeReplacer.setNodeImage(secondNodeName, secondImage)
fakes.nodeReplacer.setNodeImage(secondNodeName, secondVersion)
fakes.nodeReplacer.setScalingGroupID(secondNodeName, scalingGroupID)
secondNode := &corev1.Node{
TypeMeta: metav1.TypeMeta{
@ -214,7 +214,7 @@ var _ = Describe("NodeImage controller", func() {
}
return secondNode.Annotations
}, timeout, interval).Should(HaveKeyWithValue(scalingGroupAnnotation, scalingGroupID))
Expect(secondNode.Annotations).Should(HaveKeyWithValue(nodeImageAnnotation, secondImage))
Expect(secondNode.Annotations).Should(HaveKeyWithValue(nodeImageAnnotation, secondVersion))
By("checking that the nodes are paired as donor and heir")
Eventually(func() map[string]string {
@ -225,9 +225,9 @@ var _ = Describe("NodeImage controller", func() {
}, timeout, interval).Should(HaveKeyWithValue(heirAnnotation, secondNodeName))
Expect(k8sClient.Get(ctx, secondNodeLookupKey, secondNode)).Should(Succeed())
Expect(secondNode.Annotations).Should(HaveKeyWithValue(donorAnnotation, firstNodeName))
Expect(k8sClient.Get(ctx, nodeImageLookupKey, nodeImage)).Should(Succeed())
Expect(nodeImage.Status.Donors).Should(HaveLen(1))
Expect(nodeImage.Status.Heirs).Should(HaveLen(1))
Expect(k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion)).Should(Succeed())
Expect(nodeVersion.Status.Donors).Should(HaveLen(1))
Expect(nodeVersion.Status.Heirs).Should(HaveLen(1))
Expect(k8sClient.Get(ctx, joiningPendingNodeLookupKey, pendingNode)).Should(Not(Succeed()))
By("checking that node labels are copied to the heir")
@ -268,15 +268,15 @@ var _ = Describe("NodeImage controller", func() {
By("checking that all nodes are up-to-date")
Eventually(func() int {
err := k8sClient.Get(ctx, nodeImageLookupKey, nodeImage)
err := k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion)
if err != nil {
return 0
}
return len(nodeImage.Status.UpToDate)
return len(nodeVersion.Status.UpToDate)
}, timeout, interval).Should(Equal(1))
By("cleaning up all resources")
Expect(k8sClient.Delete(ctx, nodeImage)).Should(Succeed())
Expect(k8sClient.Delete(ctx, nodeVersion)).Should(Succeed())
Expect(k8sClient.Delete(ctx, scalingGroup)).Should(Succeed())
Expect(k8sClient.Delete(ctx, autoscalerDeployment)).Should(Succeed())
Expect(k8sClient.Delete(ctx, strategy)).Should(Succeed())

View file

@ -107,7 +107,7 @@ func TestAnnotateNodes(t *testing.T) {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
reconciler := NodeImageReconciler{
reconciler := NodeVersionReconciler{
nodeReplacer: &stubNodeReplacerReader{
nodeImage: "node-image",
scalingGroupID: "scaling-group-id",
@ -217,13 +217,13 @@ func TestPairDonorsAndHeirs(t *testing.T) {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
reconciler := NodeImageReconciler{
reconciler := NodeVersionReconciler{
nodeReplacer: &stubNodeReplacerReader{},
Client: &stubReadWriterClient{
stubReaderClient: *newStubReaderClient(t, []runtime.Object{&tc.outdatedNode, &tc.mintNode.node}, nil, nil),
},
}
nodeImage := updatev1alpha1.NodeImage{}
nodeImage := updatev1alpha1.NodeVersion{}
pairs := reconciler.pairDonorsAndHeirs(context.Background(), &nodeImage, []corev1.Node{tc.outdatedNode}, []mintNode{tc.mintNode})
if tc.wantPair == nil {
assert.Len(pairs, 0)
@ -307,7 +307,7 @@ func TestMatchDonorsAndHeirs(t *testing.T) {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
reconciler := NodeImageReconciler{
reconciler := NodeVersionReconciler{
nodeReplacer: &stubNodeReplacerReader{},
Client: &stubReadWriterClient{
stubReaderClient: *newStubReaderClient(t, []runtime.Object{&tc.donor, &tc.heir}, nil, nil),
@ -578,12 +578,12 @@ func TestCreateNewNodes(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
desiredNodeImage := updatev1alpha1.NodeImage{
Spec: updatev1alpha1.NodeImageSpec{
desiredNodeImage := updatev1alpha1.NodeVersion{
Spec: updatev1alpha1.NodeVersionSpec{
ImageReference: "image",
},
}
reconciler := NodeImageReconciler{
reconciler := NodeVersionReconciler{
nodeReplacer: &stubNodeReplacerWriter{},
Client: &stubReadWriterClient{
stubReaderClient: *newStubReaderClient(t, []runtime.Object{}, nil, nil),
@ -600,6 +600,7 @@ func TestCreateNewNodes(t *testing.T) {
func TestGroupNodes(t *testing.T) {
latestImageReference := "latest-image"
latestK8sComponentsReference := "latest-k8s-components-ref"
scalingGroup := "scaling-group"
wantNodeGroups := nodeGroups{
Outdated: []corev1.Node{
@ -607,8 +608,19 @@ func TestGroupNodes(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "outdated",
Annotations: map[string]string{
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: "old-image",
scalingGroupAnnotation: scalingGroup,
NodeKubernetesComponentsReferenceAnnotationKey: latestK8sComponentsReference,
nodeImageAnnotation: "old-image",
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "outdated",
Annotations: map[string]string{
scalingGroupAnnotation: scalingGroup,
NodeKubernetesComponentsReferenceAnnotationKey: "old-ref",
nodeImageAnnotation: latestImageReference,
},
},
},
@ -618,8 +630,9 @@ func TestGroupNodes(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "uptodate",
Annotations: map[string]string{
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
NodeKubernetesComponentsReferenceAnnotationKey: latestK8sComponentsReference,
},
},
},
@ -629,9 +642,21 @@ func TestGroupNodes(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "donor",
Annotations: map[string]string{
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: "old-image",
heirAnnotation: "heir",
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: "old-image",
NodeKubernetesComponentsReferenceAnnotationKey: latestK8sComponentsReference,
heirAnnotation: "heir",
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "donor",
Annotations: map[string]string{
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
NodeKubernetesComponentsReferenceAnnotationKey: "old-ref",
heirAnnotation: "heir",
},
},
},
@ -641,9 +666,10 @@ func TestGroupNodes(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "heir",
Annotations: map[string]string{
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
donorAnnotation: "donor",
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
NodeKubernetesComponentsReferenceAnnotationKey: latestK8sComponentsReference,
donorAnnotation: "donor",
},
},
},
@ -653,9 +679,10 @@ func TestGroupNodes(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "obsolete",
Annotations: map[string]string{
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
obsoleteAnnotation: "true",
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
NodeKubernetesComponentsReferenceAnnotationKey: latestK8sComponentsReference,
obsoleteAnnotation: "true",
},
},
},
@ -666,8 +693,9 @@ func TestGroupNodes(t *testing.T) {
ObjectMeta: metav1.ObjectMeta{
Name: "mint",
Annotations: map[string]string{
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
scalingGroupAnnotation: scalingGroup,
nodeImageAnnotation: latestImageReference,
NodeKubernetesComponentsReferenceAnnotationKey: latestK8sComponentsReference,
},
},
},
@ -695,7 +723,7 @@ func TestGroupNodes(t *testing.T) {
}
assert := assert.New(t)
groups := groupNodes(nodes, pendingNodes, latestImageReference)
groups := groupNodes(nodes, pendingNodes, latestImageReference, latestK8sComponentsReference)
assert.Equal(wantNodeGroups, groups)
}

View file

@ -94,22 +94,22 @@ func nodeMaintenanceSucceededPredicate() predicate.Predicate {
}
// findObjectsForScalingGroup requests a reconcile call for the node image referenced by a scaling group.
func (r *NodeImageReconciler) findObjectsForScalingGroup(rawScalingGroup client.Object) []reconcile.Request {
func (r *NodeVersionReconciler) findObjectsForScalingGroup(rawScalingGroup client.Object) []reconcile.Request {
scalingGroup := rawScalingGroup.(*updatev1alpha1.ScalingGroup)
return []reconcile.Request{
{NamespacedName: types.NamespacedName{Name: scalingGroup.Spec.NodeImage}},
{NamespacedName: types.NamespacedName{Name: scalingGroup.Spec.NodeVersion}},
}
}
// findAllNodeImages requests a reconcile call for all node images.
func (r *NodeImageReconciler) findAllNodeImages(_ client.Object) []reconcile.Request {
var nodeImageList updatev1alpha1.NodeImageList
err := r.List(context.TODO(), &nodeImageList)
// findAllNodeVersions requests a reconcile call for all node versions.
func (r *NodeVersionReconciler) findAllNodeVersions(_ client.Object) []reconcile.Request {
var nodeVersionList updatev1alpha1.NodeVersionList
err := r.List(context.TODO(), &nodeVersionList)
if err != nil {
return []reconcile.Request{}
}
requests := make([]reconcile.Request, len(nodeImageList.Items))
for i, item := range nodeImageList.Items {
requests := make([]reconcile.Request, len(nodeVersionList.Items))
for i, item := range nodeVersionList.Items {
requests[i] = reconcile.Request{
NamespacedName: types.NamespacedName{Name: item.GetName()},
}

View file

@ -237,39 +237,39 @@ func TestNodeMaintenanceSucceededPredicate(t *testing.T) {
func TestFindObjectsForScalingGroup(t *testing.T) {
scalingGroup := updatev1alpha1.ScalingGroup{
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: "nodeimage",
NodeVersion: "nodeversion",
},
}
wantRequests := []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Name: "nodeimage",
Name: "nodeversion",
},
},
}
assert := assert.New(t)
reconciler := NodeImageReconciler{}
reconciler := NodeVersionReconciler{}
requests := reconciler.findObjectsForScalingGroup(&scalingGroup)
assert.ElementsMatch(wantRequests, requests)
}
func TestFindAllNodeImages(t *testing.T) {
func TestFindAllNodeVersions(t *testing.T) {
testCases := map[string]struct {
nodeImage client.Object
listNodeImagesErr error
wantRequests []reconcile.Request
nodeVersion client.Object
listNodeVersionsErr error
wantRequests []reconcile.Request
}{
"getting the corresponding node images fails": {
listNodeImagesErr: errors.New("get-node-images-err"),
listNodeVersionsErr: errors.New("get-node-version-err"),
},
"node image reconcile request is returned": {
nodeImage: &updatev1alpha1.NodeImage{
ObjectMeta: metav1.ObjectMeta{Name: "nodeimage"},
nodeVersion: &updatev1alpha1.NodeVersion{
ObjectMeta: metav1.ObjectMeta{Name: "nodeversion"},
},
wantRequests: []reconcile.Request{
{
NamespacedName: types.NamespacedName{
Name: "nodeimage",
Name: "nodeversion",
},
},
},
@ -280,10 +280,10 @@ func TestFindAllNodeImages(t *testing.T) {
t.Run(name, func(t *testing.T) {
assert := assert.New(t)
reconciler := NodeImageReconciler{
Client: newStubReaderClient(t, []runtime.Object{tc.nodeImage}, nil, tc.listNodeImagesErr),
reconciler := NodeVersionReconciler{
Client: newStubReaderClient(t, []runtime.Object{tc.nodeVersion}, nil, tc.listNodeVersionsErr),
}
requests := reconciler.findAllNodeImages(nil)
requests := reconciler.findAllNodeVersions(nil)
assert.ElementsMatch(tc.wantRequests, requests)
})
}

View file

@ -28,7 +28,7 @@ import (
)
const (
nodeImageField = ".spec.nodeImage"
nodeVersionField = ".spec.nodeVersion"
conditionScalingGroupUpToDateReason = "ScalingGroupNodeImageUpToDate"
conditionScalingGroupUpToDateMessage = "Scaling group will use the latest image when creating new nodes"
conditionScalingGroupOutOfDateReason = "ScalingGroupNodeImageOutOfDate"
@ -54,10 +54,10 @@ func NewScalingGroupReconciler(scalingGroupUpdater scalingGroupUpdater, client c
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=scalinggroups,verbs=get;list;watch;create;update;patch;delete
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=scalinggroups/status,verbs=get;update;patch
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=scalinggroups/finalizers,verbs=update
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeimage,verbs=get;list;watch
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeimages/status,verbs=get
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeversion,verbs=get;list;watch
//+kubebuilder:rbac:groups=update.edgeless.systems,resources=nodeversion/status,verbs=get
// Reconcile reads the latest node image from the referenced NodeImage spec and updates the scaling group to match.
// Reconcile reads the latest node image from the referenced NodeVersion spec and updates the scaling group to match.
func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
logr := log.FromContext(ctx)
@ -66,9 +66,9 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request
logr.Error(err, "Unable to fetch ScalingGroup")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
var desiredNodeImage updatev1alpha1.NodeImage
if err := r.Get(ctx, client.ObjectKey{Name: desiredScalingGroup.Spec.NodeImage}, &desiredNodeImage); err != nil {
logr.Error(err, "Unable to fetch NodeImage")
var desiredNodeVersion updatev1alpha1.NodeVersion
if err := r.Get(ctx, client.ObjectKey{Name: desiredScalingGroup.Spec.NodeVersion}, &desiredNodeVersion); err != nil {
logr.Error(err, "Unable to fetch NodeVersion")
return ctrl.Result{}, client.IgnoreNotFound(err)
}
nodeImage, err := r.scalingGroupUpdater.GetScalingGroupImage(ctx, desiredScalingGroup.Spec.GroupID)
@ -81,7 +81,7 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request
outdatedCondition := metav1.Condition{
Type: updatev1alpha1.ConditionOutdated,
}
imagesMatch := strings.EqualFold(nodeImage, desiredNodeImage.Spec.ImageReference)
imagesMatch := strings.EqualFold(nodeImage, desiredNodeVersion.Spec.ImageReference)
if imagesMatch {
outdatedCondition.Status = metav1.ConditionFalse
outdatedCondition.Reason = conditionScalingGroupUpToDateReason
@ -99,7 +99,7 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request
if !imagesMatch {
logr.Info("ScalingGroup NodeImage is out of date")
if err := r.scalingGroupUpdater.SetScalingGroupImage(ctx, desiredScalingGroup.Spec.GroupID, desiredNodeImage.Spec.ImageReference); err != nil {
if err := r.scalingGroupUpdater.SetScalingGroupImage(ctx, desiredScalingGroup.Spec.GroupID, desiredNodeVersion.Spec.ImageReference); err != nil {
logr.Error(err, "Unable to set ScalingGroup NodeImage")
return ctrl.Result{}, err
}
@ -111,31 +111,31 @@ func (r *ScalingGroupReconciler) Reconcile(ctx context.Context, req ctrl.Request
// SetupWithManager sets up the controller with the Manager.
func (r *ScalingGroupReconciler) SetupWithManager(mgr ctrl.Manager) error {
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &updatev1alpha1.ScalingGroup{}, nodeImageField, func(rawObj client.Object) []string {
// Extract the NodeImage name from the ScalingGroup Spec, if one is provided
if err := mgr.GetFieldIndexer().IndexField(context.Background(), &updatev1alpha1.ScalingGroup{}, nodeVersionField, func(rawObj client.Object) []string {
// Extract the NodeVersion name from the ScalingGroup Spec, if one is provided
scalingGroup := rawObj.(*updatev1alpha1.ScalingGroup)
if scalingGroup.Spec.NodeImage == "" {
if scalingGroup.Spec.NodeVersion == "" {
return nil
}
return []string{scalingGroup.Spec.NodeImage}
return []string{scalingGroup.Spec.NodeVersion}
}); err != nil {
return err
}
return ctrl.NewControllerManagedBy(mgr).
For(&updatev1alpha1.ScalingGroup{}).
Watches(
&source.Kind{Type: &updatev1alpha1.NodeImage{}},
handler.EnqueueRequestsFromMapFunc(r.findObjectsForNodeImage),
&source.Kind{Type: &updatev1alpha1.NodeVersion{}},
handler.EnqueueRequestsFromMapFunc(r.findObjectsForNodeVersion),
builder.WithPredicates(predicate.ResourceVersionChangedPredicate{}),
).
Complete(r)
}
// findObjectsForNodeImage requests reconcile calls for every scaling group referencing the node image.
func (r *ScalingGroupReconciler) findObjectsForNodeImage(nodeImage client.Object) []reconcile.Request {
// findObjectsForNodeVersion requests reconcile calls for every scaling group referencing the node image.
func (r *ScalingGroupReconciler) findObjectsForNodeVersion(nodeVersion client.Object) []reconcile.Request {
attachedScalingGroups := &updatev1alpha1.ScalingGroupList{}
listOps := &client.ListOptions{
FieldSelector: fields.OneTermEqualSelector(nodeImageField, nodeImage.GetName()),
FieldSelector: fields.OneTermEqualSelector(nodeVersionField, nodeVersion.GetName()),
}
if err := r.List(context.TODO(), attachedScalingGroups, listOps); err != nil {
return []reconcile.Request{}

View file

@ -23,7 +23,7 @@ import (
var _ = Describe("ScalingGroup controller", func() {
// Define utility constants for object names and testing timeouts/durations and intervals.
const (
nodeImageName = "node-image"
nodeVersionName = "node-version"
scalingGroupName = "test-group"
timeout = time.Second * 20
@ -31,30 +31,30 @@ var _ = Describe("ScalingGroup controller", func() {
interval = time.Millisecond * 250
)
nodeImageLookupKey := types.NamespacedName{Name: nodeImageName}
nodeVersionLookupKey := types.NamespacedName{Name: nodeVersionName}
Context("When changing a node image resource spec", func() {
It("Should update corresponding scaling group images", func() {
By("creating a node image resource")
ctx := context.Background()
nodeImage := &updatev1alpha1.NodeImage{
nodeVersion := &updatev1alpha1.NodeVersion{
TypeMeta: metav1.TypeMeta{
APIVersion: "update.edgeless.systems/v1alpha1",
Kind: "NodeImage",
Kind: "NodeVersion",
},
ObjectMeta: metav1.ObjectMeta{
Name: nodeImageName,
Name: nodeVersionName,
},
Spec: updatev1alpha1.NodeImageSpec{
Spec: updatev1alpha1.NodeVersionSpec{
ImageReference: "image-1",
},
}
Expect(k8sClient.Create(ctx, nodeImage)).Should(Succeed())
createdNodeImage := &updatev1alpha1.NodeImage{}
Expect(k8sClient.Create(ctx, nodeVersion)).Should(Succeed())
createdNodeVersion := &updatev1alpha1.NodeVersion{}
Eventually(func() error {
return k8sClient.Get(ctx, nodeImageLookupKey, createdNodeImage)
return k8sClient.Get(ctx, nodeVersionLookupKey, createdNodeVersion)
}, timeout, interval).Should(Succeed())
Expect(createdNodeImage.Spec.ImageReference).Should(Equal("image-1"))
Expect(createdNodeVersion.Spec.ImageReference).Should(Equal("image-1"))
By("creating a scaling group")
scalingGroup := &updatev1alpha1.ScalingGroup{
@ -66,8 +66,8 @@ var _ = Describe("ScalingGroup controller", func() {
Name: scalingGroupName,
},
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: nodeImageName,
GroupID: "group-id",
NodeVersion: nodeVersionName,
GroupID: "group-id",
},
}
Expect(k8sClient.Create(ctx, scalingGroup)).Should(Succeed())
@ -98,9 +98,9 @@ var _ = Describe("ScalingGroup controller", func() {
}, duration, interval).Should(Equal("image-1"))
By("updating the node image")
Expect(k8sClient.Get(ctx, nodeImageLookupKey, nodeImage)).Should(Succeed())
nodeImage.Spec.ImageReference = "image-2"
Expect(k8sClient.Update(ctx, nodeImage)).Should(Succeed())
Expect(k8sClient.Get(ctx, nodeVersionLookupKey, nodeVersion)).Should(Succeed())
nodeVersion.Spec.ImageReference = "image-2"
Expect(k8sClient.Update(ctx, nodeVersion)).Should(Succeed())
By("checking the scaling group eventually uses the latest image")
Eventually(func() string {
@ -118,7 +118,7 @@ var _ = Describe("ScalingGroup controller", func() {
}, duration, interval).Should(Equal("image-2"))
By("cleaning up all resources")
Expect(k8sClient.Delete(ctx, createdNodeImage)).Should(Succeed())
Expect(k8sClient.Delete(ctx, createdNodeVersion)).Should(Succeed())
Expect(k8sClient.Delete(ctx, scalingGroup)).Should(Succeed())
})
})

View file

@ -115,7 +115,7 @@ var _ = BeforeSuite(func() {
}).SetupWithManager(k8sManager)
Expect(err).ToNot(HaveOccurred())
err = (&NodeImageReconciler{
err = (&NodeVersionReconciler{
nodeReplacer: fakes.nodeReplacer,
Client: k8sManager.GetClient(),
Scheme: k8sManager.GetScheme(),

View file

@ -9,8 +9,8 @@ package constants
const (
// AutoscalingStrategyResourceName resource name used for AutoscalingStrategy.
AutoscalingStrategyResourceName = "autoscalingstrategy"
// NodeImageResourceName resource name used for NodeImage.
NodeImageResourceName = "constellation-os"
// NodeVersionResourceName resource name used for NodeVersion.
NodeVersionResourceName = "constellation-version"
// ControlPlaneScalingGroupResourceName resource name used for ControlPlaneScalingGroup.
ControlPlaneScalingGroupResourceName = "scalinggroup-controlplane"
// WorkerScalingGroupResourceName resource name used for WorkerScaling.

View file

@ -12,9 +12,11 @@ import (
"errors"
"fmt"
"strings"
"time"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/internal/constants"
corev1 "k8s.io/api/core/v1"
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
@ -22,7 +24,7 @@ import (
)
// InitialResources creates the initial resources for the node operator.
func InitialResources(ctx context.Context, k8sClient client.Writer, imageInfo imageInfoGetter, scalingGroupGetter scalingGroupGetter, uid string) error {
func InitialResources(ctx context.Context, k8sClient client.Client, imageInfo imageInfoGetter, scalingGroupGetter scalingGroupGetter, uid string) error {
logr := log.FromContext(ctx)
controlPlaneGroupIDs, workerGroupIDs, err := scalingGroupGetter.ListScalingGroups(ctx, uid)
if err != nil {
@ -50,8 +52,8 @@ func InitialResources(ctx context.Context, k8sClient client.Writer, imageInfo im
imageVersion = ""
}
if err := createNodeImage(ctx, k8sClient, imageReference, imageVersion); err != nil {
return fmt.Errorf("creating initial node image %q: %w", imageReference, err)
if err := createNodeVersion(ctx, k8sClient, imageReference, imageVersion); err != nil {
return fmt.Errorf("creating initial node version %q: %w", imageReference, err)
}
for _, groupID := range controlPlaneGroupIDs {
groupName, err := scalingGroupGetter.GetScalingGroupName(groupID)
@ -110,22 +112,61 @@ func createAutoscalingStrategy(ctx context.Context, k8sClient client.Writer, pro
return err
}
// createNodeImage creates the initial nodeimage resource if it does not exist yet.
func createNodeImage(ctx context.Context, k8sClient client.Writer, imageReference, imageVersion string) error {
err := k8sClient.Create(ctx, &updatev1alpha1.NodeImage{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
// createNodeVersion creates the initial nodeversion resource if it does not exist yet.
func createNodeVersion(ctx context.Context, k8sClient client.Client, imageReference, imageVersion string) error {
k8sComponentsRef, err := findLatestK8sComponentsConfigMap(ctx, k8sClient)
if err != nil {
return fmt.Errorf("finding latest k8s-components configmap: %w", err)
}
err = k8sClient.Create(ctx, &updatev1alpha1.NodeVersion{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeVersion"},
ObjectMeta: metav1.ObjectMeta{
Name: constants.NodeImageResourceName,
Name: constants.NodeVersionResourceName,
},
Spec: updatev1alpha1.NodeImageSpec{
ImageReference: imageReference,
ImageVersion: imageVersion,
Spec: updatev1alpha1.NodeVersionSpec{
ImageReference: imageReference,
ImageVersion: imageVersion,
KubernetesComponentsReference: k8sComponentsRef,
},
})
if k8sErrors.IsAlreadyExists(err) {
return nil
} else if err != nil {
return err
}
return err
return nil
}
// findLatestK8sComponentsConfigMap finds most recently created k8s-components configmap in the kube-system namespace.
// It returns an error if there is no or multiple configmaps matching the prefix "k8s-components".
func findLatestK8sComponentsConfigMap(ctx context.Context, k8sClient client.Client) (string, error) {
var configMaps corev1.ConfigMapList
err := k8sClient.List(ctx, &configMaps, client.InNamespace("kube-system"))
if err != nil {
return "", fmt.Errorf("listing configmaps: %w", err)
}
// collect all k8s-components configmaps
componentConfigMaps := make(map[string]time.Time)
for _, configMap := range configMaps.Items {
if strings.HasPrefix(configMap.Name, "k8s-components") {
componentConfigMaps[configMap.Name] = configMap.CreationTimestamp.Time
}
}
if len(componentConfigMaps) == 0 {
return "", fmt.Errorf("no configmaps found")
}
// find latest configmap
var latestConfigMap string
var latestTime time.Time
for configMap, creationTime := range componentConfigMaps {
if creationTime.After(latestTime) {
latestConfigMap = configMap
latestTime = creationTime
}
}
return latestConfigMap, nil
}
// createScalingGroup creates an initial scaling group resource if it does not exist yet.
@ -136,7 +177,7 @@ func createScalingGroup(ctx context.Context, config newScalingGroupConfig) error
Name: strings.ToLower(config.groupName),
},
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: constants.NodeImageResourceName,
NodeVersion: constants.NodeVersionResourceName,
GroupID: config.groupID,
AutoscalerGroupName: config.autoscalingGroupName,
Min: 1,

View file

@ -10,18 +10,22 @@ import (
"context"
"errors"
"testing"
"time"
updatev1alpha1 "github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/api/v1alpha1"
"github.com/edgelesssys/constellation/operators/constellation-node-operator/v2/internal/constants"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
k8sErrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func TestInitialResources(t *testing.T) {
k8sComponentsReference := "k8s-components-sha256-ABC"
testCases := map[string]struct {
items []scalingGroupStoreItem
imageErr error
@ -85,7 +89,16 @@ func TestInitialResources(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
k8sClient := &stubK8sClient{createErr: tc.createErr}
k8sClient := &fakeK8sClient{
createErr: tc.createErr,
listConfigMaps: []corev1.ConfigMap{
{
ObjectMeta: metav1.ObjectMeta{
Name: k8sComponentsReference,
},
},
},
}
scalingGroupGetter := newScalingGroupGetter(tc.items, tc.imageErr, tc.nameErr, tc.listErr)
err := InitialResources(context.Background(), k8sClient, &stubImageInfo{}, scalingGroupGetter, "uid")
if tc.wantErr {
@ -156,7 +169,7 @@ func TestCreateAutoscalingStrategy(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
k8sClient := &stubK8sClient{createErr: tc.createErr}
k8sClient := &fakeK8sClient{createErr: tc.createErr}
err := createAutoscalingStrategy(context.Background(), k8sClient, "stub")
if tc.wantErr {
assert.Error(err)
@ -169,21 +182,24 @@ func TestCreateAutoscalingStrategy(t *testing.T) {
}
}
func TestCreateNodeImage(t *testing.T) {
func TestCreateNodeVersion(t *testing.T) {
k8sComponentsReference := "k8s-components-sha256-reference"
testCases := map[string]struct {
createErr error
wantNodeImage *updatev1alpha1.NodeImage
wantErr bool
createErr error
existingNodeVersion *updatev1alpha1.NodeVersion
wantNodeVersion *updatev1alpha1.NodeVersion
wantErr bool
}{
"create works": {
wantNodeImage: &updatev1alpha1.NodeImage{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
wantNodeVersion: &updatev1alpha1.NodeVersion{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeVersion"},
ObjectMeta: metav1.ObjectMeta{
Name: constants.NodeImageResourceName,
Name: constants.NodeVersionResourceName,
},
Spec: updatev1alpha1.NodeImageSpec{
ImageReference: "image-reference",
ImageVersion: "image-version",
Spec: updatev1alpha1.NodeVersionSpec{
ImageReference: "image-reference",
ImageVersion: "image-version",
KubernetesComponentsReference: k8sComponentsReference,
},
},
},
@ -191,16 +207,28 @@ func TestCreateNodeImage(t *testing.T) {
createErr: errors.New("create failed"),
wantErr: true,
},
"image exists": {
createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.AutoscalingStrategyResourceName),
wantNodeImage: &updatev1alpha1.NodeImage{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeImage"},
"version exists": {
createErr: k8sErrors.NewAlreadyExists(schema.GroupResource{}, constants.NodeVersionResourceName),
existingNodeVersion: &updatev1alpha1.NodeVersion{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeVersion"},
ObjectMeta: metav1.ObjectMeta{
Name: constants.NodeImageResourceName,
Name: constants.NodeVersionResourceName,
},
Spec: updatev1alpha1.NodeImageSpec{
ImageReference: "image-reference",
ImageVersion: "image-version",
Spec: updatev1alpha1.NodeVersionSpec{
ImageReference: "image-reference2",
ImageVersion: "image-version2",
KubernetesComponentsReference: "components-reference2",
},
},
wantNodeVersion: &updatev1alpha1.NodeVersion{
TypeMeta: metav1.TypeMeta{APIVersion: "update.edgeless.systems/v1alpha1", Kind: "NodeVersion"},
ObjectMeta: metav1.ObjectMeta{
Name: constants.NodeVersionResourceName,
},
Spec: updatev1alpha1.NodeVersionSpec{
ImageReference: "image-reference2",
ImageVersion: "image-version2",
KubernetesComponentsReference: "components-reference2",
},
},
},
@ -211,15 +239,28 @@ func TestCreateNodeImage(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
k8sClient := &stubK8sClient{createErr: tc.createErr}
err := createNodeImage(context.Background(), k8sClient, "image-reference", "image-version")
k8sClient := &fakeK8sClient{
createErr: tc.createErr,
listConfigMaps: []corev1.ConfigMap{
{
ObjectMeta: metav1.ObjectMeta{
Name: k8sComponentsReference,
CreationTimestamp: metav1.Time{Time: time.Unix(1, 0)},
},
},
},
}
if tc.existingNodeVersion != nil {
k8sClient.createdObjects = append(k8sClient.createdObjects, tc.existingNodeVersion)
}
err := createNodeVersion(context.Background(), k8sClient, "image-reference", "image-version")
if tc.wantErr {
assert.Error(err)
return
}
require.NoError(err)
assert.Len(k8sClient.createdObjects, 1)
assert.Equal(tc.wantNodeImage, k8sClient.createdObjects[0])
assert.Equal(tc.wantNodeVersion, k8sClient.createdObjects[0])
})
}
}
@ -237,7 +278,7 @@ func TestCreateScalingGroup(t *testing.T) {
Name: "group-name",
},
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: constants.NodeImageResourceName,
NodeVersion: constants.NodeVersionResourceName,
GroupID: "group-id",
AutoscalerGroupName: "group-Name",
Min: 1,
@ -258,7 +299,7 @@ func TestCreateScalingGroup(t *testing.T) {
Name: "group-name",
},
Spec: updatev1alpha1.ScalingGroupSpec{
NodeImage: constants.NodeImageResourceName,
NodeVersion: constants.NodeVersionResourceName,
GroupID: "group-id",
AutoscalerGroupName: "group-Name",
Min: 1,
@ -274,7 +315,7 @@ func TestCreateScalingGroup(t *testing.T) {
assert := assert.New(t)
require := require.New(t)
k8sClient := &stubK8sClient{createErr: tc.createErr}
k8sClient := &fakeK8sClient{createErr: tc.createErr}
newScalingGroupConfig := newScalingGroupConfig{k8sClient, "group-id", "group-Name", "group-Name", updatev1alpha1.WorkerRole}
err := createScalingGroup(context.Background(), newScalingGroupConfig)
if tc.wantErr {
@ -288,17 +329,65 @@ func TestCreateScalingGroup(t *testing.T) {
}
}
type stubK8sClient struct {
type fakeK8sClient struct {
createdObjects []client.Object
createErr error
client.Writer
listConfigMaps []corev1.ConfigMap
listErr error
getErr error
updateErr error
client.Client
}
func (s *stubK8sClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
func (s *fakeK8sClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
for _, o := range s.createdObjects {
if obj.GetName() == o.GetName() {
return k8sErrors.NewAlreadyExists(schema.GroupResource{}, obj.GetName())
}
}
s.createdObjects = append(s.createdObjects, obj)
return s.createErr
}
func (s *fakeK8sClient) Get(ctx context.Context, key types.NamespacedName, obj client.Object, opts ...client.GetOption) error {
if ObjNodeVersion, ok := obj.(*updatev1alpha1.NodeVersion); ok {
for _, o := range s.createdObjects {
if createdNodeVersion, ok := o.(*updatev1alpha1.NodeVersion); ok && createdNodeVersion != nil {
if createdNodeVersion.Name == key.Name {
ObjNodeVersion.ObjectMeta = createdNodeVersion.ObjectMeta
ObjNodeVersion.TypeMeta = createdNodeVersion.TypeMeta
ObjNodeVersion.Spec = createdNodeVersion.Spec
return nil
}
}
}
}
return s.getErr
}
func (s *fakeK8sClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
if updatedObjectNodeVersion, ok := obj.(*updatev1alpha1.NodeVersion); ok {
for i, o := range s.createdObjects {
if createdObjectNodeVersion, ok := o.(*updatev1alpha1.NodeVersion); ok && createdObjectNodeVersion != nil {
if createdObjectNodeVersion.Name == updatedObjectNodeVersion.Name {
s.createdObjects[i] = obj
return nil
}
}
}
}
return s.updateErr
}
func (s *fakeK8sClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
if configMapList, ok := list.(*corev1.ConfigMapList); ok {
configMapList.Items = append(configMapList.Items, s.listConfigMaps...)
}
return s.listErr
}
type stubImageInfo struct {
imageVersion string
err error

View file

@ -134,10 +134,10 @@ func main() {
setupLog.Error(err, "Unable to deploy initial resources")
os.Exit(1)
}
if err = controllers.NewNodeImageReconciler(
if err = controllers.NewNodeVersionReconciler(
cspClient, etcdClient, mgr.GetClient(), mgr.GetScheme(),
).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "Unable to create controller", "controller", "NodeImage")
setupLog.Error(err, "Unable to create controller", "controller", "NodeVersion")
os.Exit(1)
}
if err = (&controllers.AutoscalingStrategyReconciler{