Constellation conformance mode (#161)

* add conformance mode
This commit is contained in:
3u13r 2022-09-20 10:07:55 +02:00 committed by GitHub
parent 9c00f4efc2
commit 774e300a32
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 74 additions and 36 deletions

View File

@ -23,6 +23,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Added
- Loadbalancer for control-plane recovery
- K8s conformance mode
### Changed
<!-- For changes in existing functionality. -->

View File

@ -22,7 +22,7 @@ type clusterFake struct{}
// InitCluster fakes bootstrapping a new cluster with the current node being the master, returning the arguments required to join the cluster.
func (c *clusterFake) InitCluster(
context.Context, []string, string, string, []byte, []uint32, bool, []byte, bool,
resources.KMSConfig, map[string]string, []byte, *logger.Logger,
resources.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
) ([]byte, error) {
return []byte{}, nil
}

View File

@ -38,6 +38,7 @@ type InitRequest struct {
HelmDeployments []byte `protobuf:"bytes,11,opt,name=helm_deployments,json=helmDeployments,proto3" json:"helm_deployments,omitempty"`
EnforcedPcrs []uint32 `protobuf:"varint,12,rep,packed,name=enforced_pcrs,json=enforcedPcrs,proto3" json:"enforced_pcrs,omitempty"`
EnforceIdkeydigest bool `protobuf:"varint,13,opt,name=enforce_idkeydigest,json=enforceIdkeydigest,proto3" json:"enforce_idkeydigest,omitempty"`
ConformanceMode bool `protobuf:"varint,14,opt,name=conformance_mode,json=conformanceMode,proto3" json:"conformance_mode,omitempty"`
}
func (x *InitRequest) Reset() {
@ -163,6 +164,13 @@ func (x *InitRequest) GetEnforceIdkeydigest() bool {
return false
}
func (x *InitRequest) GetConformanceMode() bool {
if x != nil {
return x.ConformanceMode
}
return false
}
type InitResponse struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@ -285,7 +293,7 @@ var File_init_proto protoreflect.FileDescriptor
var file_init_proto_rawDesc = []byte{
0x0a, 0x0a, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x6e,
0x69, 0x74, 0x22, 0xb6, 0x04, 0x0a, 0x0b, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
0x69, 0x74, 0x22, 0xe1, 0x04, 0x0a, 0x0b, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65,
0x73, 0x74, 0x12, 0x36, 0x0a, 0x17, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e,
0x67, 0x5f, 0x6e, 0x6f, 0x64, 0x65, 0x5f, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x18, 0x01, 0x20,
0x03, 0x28, 0x09, 0x52, 0x15, 0x61, 0x75, 0x74, 0x6f, 0x73, 0x63, 0x61, 0x6c, 0x69, 0x6e, 0x67,
@ -320,26 +328,29 @@ var file_init_proto_rawDesc = []byte{
0x66, 0x6f, 0x72, 0x63, 0x65, 0x64, 0x50, 0x63, 0x72, 0x73, 0x12, 0x2f, 0x0a, 0x13, 0x65, 0x6e,
0x66, 0x6f, 0x72, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x6b, 0x65, 0x79, 0x64, 0x69, 0x67, 0x65, 0x73,
0x74, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x08, 0x52, 0x12, 0x65, 0x6e, 0x66, 0x6f, 0x72, 0x63, 0x65,
0x49, 0x64, 0x6b, 0x65, 0x79, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x22, 0x68, 0x0a, 0x0c, 0x49,
0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b,
0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52,
0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f,
0x77, 0x6e, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f,
0x77, 0x6e, 0x65, 0x72, 0x49, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65,
0x72, 0x5f, 0x69, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73,
0x74, 0x65, 0x72, 0x49, 0x64, 0x22, 0x47, 0x0a, 0x0a, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72,
0x4b, 0x65, 0x79, 0x12, 0x1a, 0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12,
0x1d, 0x0a, 0x0a, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20,
0x01, 0x28, 0x09, 0x52, 0x09, 0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x32, 0x34,
0x0a, 0x03, 0x41, 0x50, 0x49, 0x12, 0x2d, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e,
0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74,
0x1a, 0x12, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70,
0x6f, 0x6e, 0x73, 0x65, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63,
0x6f, 0x6d, 0x2f, 0x65, 0x64, 0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63,
0x6f, 0x6e, 0x73, 0x74, 0x65, 0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x62, 0x6f, 0x6f,
0x74, 0x73, 0x74, 0x72, 0x61, 0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72,
0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
0x49, 0x64, 0x6b, 0x65, 0x79, 0x64, 0x69, 0x67, 0x65, 0x73, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x63,
0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e, 0x63, 0x65, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x18,
0x0e, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x6e,
0x63, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x22, 0x68, 0x0a, 0x0c, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65,
0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x6b, 0x75, 0x62, 0x65, 0x63, 0x6f,
0x6e, 0x66, 0x69, 0x67, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x0a, 0x6b, 0x75, 0x62, 0x65,
0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x19, 0x0a, 0x08, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x5f,
0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x07, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x49,
0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x5f, 0x69, 0x64, 0x18,
0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x09, 0x63, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x49, 0x64,
0x22, 0x47, 0x0a, 0x0a, 0x53, 0x53, 0x48, 0x55, 0x73, 0x65, 0x72, 0x4b, 0x65, 0x79, 0x12, 0x1a,
0x0a, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
0x52, 0x08, 0x75, 0x73, 0x65, 0x72, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x70, 0x75,
0x62, 0x6c, 0x69, 0x63, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09,
0x70, 0x75, 0x62, 0x6c, 0x69, 0x63, 0x4b, 0x65, 0x79, 0x32, 0x34, 0x0a, 0x03, 0x41, 0x50, 0x49,
0x12, 0x2d, 0x0a, 0x04, 0x49, 0x6e, 0x69, 0x74, 0x12, 0x11, 0x2e, 0x69, 0x6e, 0x69, 0x74, 0x2e,
0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x12, 0x2e, 0x69, 0x6e,
0x69, 0x74, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42,
0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x65, 0x64,
0x67, 0x65, 0x6c, 0x65, 0x73, 0x73, 0x73, 0x79, 0x73, 0x2f, 0x63, 0x6f, 0x6e, 0x73, 0x74, 0x65,
0x6c, 0x6c, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x62, 0x6f, 0x6f, 0x74, 0x73, 0x74, 0x72, 0x61,
0x70, 0x70, 0x65, 0x72, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06,
0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (

View File

@ -22,6 +22,7 @@ message InitRequest {
bytes helm_deployments = 11;
repeated uint32 enforced_pcrs = 12;
bool enforce_idkeydigest = 13;
bool conformance_mode = 14;
}
message InitResponse {

View File

@ -143,6 +143,7 @@ func (s *Server) Init(ctx context.Context, req *initproto.InitRequest) (*initpro
},
sshProtoKeysToMap(req.SshUserKeys),
req.HelmDeployments,
req.ConformanceMode,
s.log,
)
if err != nil {
@ -244,6 +245,7 @@ type ClusterInitializer interface {
kmsConfig resources.KMSConfig,
sshUserKeys map[string]string,
helmDeployments []byte,
conformanceMode bool,
log *logger.Logger,
) ([]byte, error)
}

View File

@ -290,7 +290,7 @@ type stubClusterInitializer struct {
func (i *stubClusterInitializer) InitCluster(
context.Context, []string, string, string, []byte, []uint32, bool, []byte, bool,
resources.KMSConfig, map[string]string, []byte, *logger.Logger,
resources.KMSConfig, map[string]string, []byte, bool, *logger.Logger,
) ([]byte, error) {
return i.initClusterKubeconfig, i.initClusterErr
}

View File

@ -131,7 +131,7 @@ func (k *KubernetesUtil) InstallComponents(ctx context.Context, version versions
}
func (k *KubernetesUtil) InitCluster(
ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger,
ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger,
) error {
// TODO: audit policy should be user input
auditPolicy, err := resources.NewDefaultAuditPolicy().Marshal()
@ -188,7 +188,12 @@ func (k *KubernetesUtil) InitCluster(
// initialize the cluster
log.Infof("Initializing the cluster using kubeadm init")
cmd = exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", "--skip-phases=preflight,certs,addon/kube-proxy", "--config", initConfigFile.Name())
skipPhases := "--skip-phases=preflight,certs"
if !conformanceMode {
skipPhases += ",addon/kube-proxy"
}
cmd = exec.CommandContext(ctx, kubeadmPath, "init", "-v=5", skipPhases, "--config", initConfigFile.Name())
out, err = cmd.CombinedOutput()
if err != nil {
var exitErr *exec.ExitError

View File

@ -19,7 +19,7 @@ import (
type clusterUtil interface {
InstallComponents(ctx context.Context, version versions.ValidK8sVersion) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger) error
InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error
JoinCluster(ctx context.Context, joinConfig []byte, peerRole role.Role, controlPlaneEndpoint string, log *logger.Logger) error
SetupHelmDeployments(ctx context.Context, client k8sapi.Client, helmDeployments []byte, in k8sapi.SetupPodNetworkInput, log *logger.Logger) error
SetupAccessManager(kubectl k8sapi.Client, sshUsers kubernetes.Marshaler) error

View File

@ -77,7 +77,7 @@ func New(cloudProvider string, clusterUtil clusterUtil, configProvider configura
// InitCluster initializes a new Kubernetes cluster and applies pod network provider.
func (k *KubeWrapper) InitCluster(
ctx context.Context, autoscalingNodeGroups []string, cloudServiceAccountURI, versionString string, measurementSalt []byte,
enforcedPCRs []uint32, enforceIdKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig resources.KMSConfig, sshUsers map[string]string, helmDeployments []byte, log *logger.Logger,
enforcedPCRs []uint32, enforceIdKeyDigest bool, idKeyDigest []byte, azureCVM bool, kmsConfig resources.KMSConfig, sshUsers map[string]string, helmDeployments []byte, conformanceMode bool, log *logger.Logger,
) ([]byte, error) {
k8sVersion, err := versions.NewValidK8sVersion(versionString)
if err != nil {
@ -155,7 +155,7 @@ func (k *KubeWrapper) InitCluster(
return nil, fmt.Errorf("encoding kubeadm init configuration as YAML: %w", err)
}
log.Infof("Initializing Kubernetes cluster")
if err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, validIPs, controlPlaneEndpoint, log); err != nil {
if err := k.clusterUtil.InitCluster(ctx, initConfigYAML, nodeName, validIPs, controlPlaneEndpoint, conformanceMode, log); err != nil {
return nil, fmt.Errorf("kubeadm init: %w", err)
}
kubeConfig, err := k.GetKubeconfig()

View File

@ -320,7 +320,7 @@ func TestInitCluster(t *testing.T) {
_, err := kube.InitCluster(
context.Background(), autoscalingNodeGroups, serviceAccountURI, string(tc.k8sVersion),
nil, nil, false, nil, true, resources.KMSConfig{MasterSecret: masterSecret}, nil, nil, logger.NewTest(t),
nil, nil, false, nil, true, resources.KMSConfig{MasterSecret: masterSecret}, nil, nil, false, logger.NewTest(t),
)
if tc.wantErr {
@ -557,7 +557,7 @@ func (s *stubClusterUtil) InstallComponents(ctx context.Context, version version
return s.installComponentsErr
}
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, log *logger.Logger) error {
func (s *stubClusterUtil) InitCluster(ctx context.Context, initConfig []byte, nodeName string, ips []net.IP, controlPlaneEndpoint string, conformanceMode bool, log *logger.Logger) error {
s.initConfigs = append(s.initConfigs, initConfig)
return s.initClusterErr
}

View File

@ -7,13 +7,13 @@ SPDX-License-Identifier: AGPL-3.0-only
package cmd
type helmLoader interface {
Load(csp string) ([]byte, error)
Load(csp string, conformanceMode bool) ([]byte, error)
}
type stubHelmLoader struct {
loadErr error
}
func (d *stubHelmLoader) Load(csp string) ([]byte, error) {
func (d *stubHelmLoader) Load(csp string, conformanceMode bool) ([]byte, error) {
return nil, d.loadErr
}

View File

@ -56,6 +56,7 @@ func NewInitCmd() *cobra.Command {
cmd.Flags().String("master-secret", "", "path to base64-encoded master secret")
cmd.Flags().String("endpoint", "", "endpoint of the bootstrapper, passed as HOST[:PORT]")
cmd.Flags().Bool("autoscale", false, "enable Kubernetes cluster-autoscaler")
cmd.Flags().Bool("conformance", false, "enable conformance mode")
return cmd
}
@ -138,7 +139,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
autoscalingNodeGroups = append(autoscalingNodeGroups, workers.GroupID)
}
helmDeployments, err := helmLoader.Load(stat.CloudProvider)
helmDeployments, err := helmLoader.Load(stat.CloudProvider, flags.conformance)
if err != nil {
return fmt.Errorf("loading Helm charts: %w", err)
}
@ -163,6 +164,7 @@ func initialize(cmd *cobra.Command, newDialer func(validator *cloudcmd.Validator
HelmDeployments: helmDeployments,
EnforcedPcrs: getEnforcedMeasurements(provider, config),
EnforceIdkeydigest: getEnforceIdKeyDigest(provider, config),
ConformanceMode: flags.conformance,
}
resp, err := initCall(cmd.Context(), newDialer(validator), flags.endpoint, req)
if err != nil {
@ -289,6 +291,10 @@ func evalFlagArgs(cmd *cobra.Command, fileHandler file.Handler) (initFlags, erro
if err != nil {
return initFlags{}, fmt.Errorf("parsing autoscale flag: %w", err)
}
conformance, err := cmd.Flags().GetBool("conformance")
if err != nil {
return initFlags{}, fmt.Errorf("parsing autoscale flag: %w", err)
}
configPath, err := cmd.Flags().GetString("config")
if err != nil {
return initFlags{}, fmt.Errorf("parsing config path flag: %w", err)
@ -298,6 +304,7 @@ func evalFlagArgs(cmd *cobra.Command, fileHandler file.Handler) (initFlags, erro
configPath: configPath,
endpoint: endpoint,
autoscale: autoscale,
conformance: conformance,
masterSecretPath: masterSecretPath,
}, nil
}
@ -308,6 +315,7 @@ type initFlags struct {
masterSecretPath string
endpoint string
autoscale bool
conformance bool
}
// masterSecret holds the master key and salt for deriving keys.

View File

@ -31,8 +31,8 @@ var HelmFS embed.FS
type ChartLoader struct{}
func (i *ChartLoader) Load(csp string) ([]byte, error) {
ciliumDeployment, err := i.loadCilium(csp)
func (i *ChartLoader) Load(csp string, conformanceMode bool) ([]byte, error) {
ciliumDeployment, err := i.loadCilium(csp, conformanceMode)
if err != nil {
return nil, err
}
@ -44,7 +44,7 @@ func (i *ChartLoader) Load(csp string) ([]byte, error) {
return depl, nil
}
func (i *ChartLoader) loadCilium(csp string) (helm.Deployment, error) {
func (i *ChartLoader) loadCilium(csp string, conformanceMode bool) (helm.Deployment, error) {
chart, err := loadChartsDir(HelmFS, "charts/cilium")
if err != nil {
return helm.Deployment{}, err
@ -60,6 +60,16 @@ func (i *ChartLoader) loadCilium(csp string) (helm.Deployment, error) {
default:
return helm.Deployment{}, fmt.Errorf("unknown csp: %s", csp)
}
if conformanceMode {
ciliumVals["kubeProxyReplacementHealthzBindAddr"] = ""
ciliumVals["kubeProxyReplacement"] = "partial"
ciliumVals["sessionAffinity"] = true
ciliumVals["cni"] = map[string]interface{}{
"chainingMode": "portmap",
}
}
return helm.Deployment{Chart: chart, Values: ciliumVals}, nil
}