Merge 6fbcba4de0 into 036a4f2ee1

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

bazel/toolchains/container_images.bzl

@@ -16,6 +16,6 @@ def containter_image_deps():
     )
     oci_pull(
         name = "libvirtd_base",
-        digest = "sha256:527fc93a1a53c08b51f87295ff45745dab4570da7cbeb28e93f359e813aba29b",
+        digest = "sha256:b1a65581c445a1da618e33743578c4ea80e6a724659fbad9e6555a6a9f48b37a",
         image = "ghcr.io/edgelesssys/constellation/libvirtd-base",
     )

bazel/toolchains/go_module_deps.bzl

@@ -2197,9 +2197,8 @@ def go_dependencies():
         build_file_generation = "on",
         build_file_proto_mode = "disable_global",
         importpath = "github.com/google/go-sev-guest",
-        replace = "github.com/google/go-sev-guest",
-        sum = "h1:6o4Z/vQqNUH+cEagfx1Ez5ElK70iZulEXZwmLnRo44I=",
-        version = "v0.0.0-20230928233922-2dcbba0a4b9d",
+        sum = "h1:gnww4U8fHV5DCPz4gykr1s8SEX1fFNcxCBy+vvXN24k=",
+        version = "v0.11.1",
     )
     go_repository(
         name = "com_github_google_go_tdx_guest",

go.mod

@@ -34,7 +34,6 @@ replace (
 )
 
 replace (
-	github.com/google/go-sev-guest => github.com/google/go-sev-guest v0.0.0-20230928233922-2dcbba0a4b9d
 	github.com/martinjungblut/go-cryptsetup => github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c
 	github.com/tink-crypto/tink-go/v2 v2.0.0 => github.com/derpsteb/tink-go/v2 v2.0.0-20231002051717-a808e454eed6
 )

go.sum

@@ -418,8 +418,8 @@ github.com/google/go-configfs-tsm v0.2.2 h1:YnJ9rXIOj5BYD7/0DNnzs8AOp7UcvjfTvt21
 github.com/google/go-configfs-tsm v0.2.2/go.mod h1:EL1GTDFMb5PZQWDviGfZV9n87WeGTR/JUg13RfwkgRo=
 github.com/google/go-containerregistry v0.19.0 h1:uIsMRBV7m/HDkDxE/nXMnv1q+lOOSPlQ/ywc5JbB8Ic=
 github.com/google/go-containerregistry v0.19.0/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ=
-github.com/google/go-sev-guest v0.0.0-20230928233922-2dcbba0a4b9d h1:6o4Z/vQqNUH+cEagfx1Ez5ElK70iZulEXZwmLnRo44I=
-github.com/google/go-sev-guest v0.0.0-20230928233922-2dcbba0a4b9d/go.mod h1:hc1R4R6f8+NcJwITs0L90fYWTsBpd1Ix+Gur15sqHDs=
+github.com/google/go-sev-guest v0.11.1 h1:gnww4U8fHV5DCPz4gykr1s8SEX1fFNcxCBy+vvXN24k=
+github.com/google/go-sev-guest v0.11.1/go.mod h1:qBOfb+JmgsUI3aUyzQoGC13Kpp9zwLeWvuyXmA9q77w=
 github.com/google/go-tdx-guest v0.3.1 h1:gl0KvjdsD4RrJzyLefDOvFOUH3NAJri/3qvaL5m83Iw=
 github.com/google/go-tdx-guest v0.3.1/go.mod h1:/rc3d7rnPykOPuY8U9saMyEps0PZDThLk/RygXm04nE=
 github.com/google/go-tpm v0.9.1-0.20240510201744-5c2f0887e003 h1:gfGQAIxsEEAuYuFvjCGpDnTwisMJOz+rUfJMkk4yTmc=

internal/attestation/aws/snp/BUILD.bazel

@@ -18,7 +18,6 @@ go_library(
         "//internal/attestation/vtpm",
         "//internal/config",
         "@com_github_google_go_sev_guest//abi",
-        "@com_github_google_go_sev_guest//client",
         "@com_github_google_go_sev_guest//kds",
         "@com_github_google_go_sev_guest//proto/sevsnp",
         "@com_github_google_go_sev_guest//validate",

internal/attestation/aws/snp/issuer.go

@@ -21,7 +21,6 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/attestation/vtpm"
 
 	"github.com/google/go-sev-guest/abi"
-	sevclient "github.com/google/go-sev-guest/client"
 	"github.com/google/go-tpm-tools/client"
 	tpmclient "github.com/google/go-tpm-tools/client"
 )
@@ -70,13 +69,7 @@ func getInstanceInfo(_ context.Context, tpm io.ReadWriteCloser, _ []byte) ([]byt
 
 	akDigest := sha512.Sum512(encoded)
 
-	device, err := sevclient.OpenDevice()
-	if err != nil {
-		return nil, fmt.Errorf("opening sev device: %w", err)
-	}
-	defer device.Close()
-
-	report, certs, err := sevclient.GetRawExtendedReportAtVmpl(device, akDigest, 0)
+	report, certs, err := snp.GetExtendedReport(akDigest)
 	if err != nil {
 		return nil, fmt.Errorf("getting extended report: %w", err)
 	}

internal/attestation/azure/snp/validator_test.go

@@ -368,7 +368,7 @@ func TestTrustedKeyFromSNP(t *testing.T) {
 			),
 			wantErr: true,
 			assertion: func(assert *assert.Assertions, err error) {
-				assert.ErrorContains(err, "could not interpret VCEK DER bytes: x509: malformed certificate")
+				assert.ErrorContains(err, "x509: malformed certificate")
 			},
 		},
 		"invalid certchain fall back to embedded": {

internal/attestation/gcp/snp/BUILD.bazel

@@ -17,7 +17,6 @@ go_library(
         "//internal/attestation/vtpm",
         "//internal/config",
         "@com_github_google_go_sev_guest//abi",
-        "@com_github_google_go_sev_guest//client",
         "@com_github_google_go_sev_guest//kds",
         "@com_github_google_go_sev_guest//proto/sevsnp",
         "@com_github_google_go_sev_guest//validate",

internal/attestation/gcp/snp/issuer.go

@@ -21,7 +21,6 @@ import (
 	"github.com/edgelesssys/constellation/v2/internal/attestation/vtpm"
 
 	"github.com/google/go-sev-guest/abi"
-	sevclient "github.com/google/go-sev-guest/client"
 	"github.com/google/go-tpm-tools/client"
 	tpmclient "github.com/google/go-tpm-tools/client"
 	"github.com/google/go-tpm-tools/proto/attest"
@@ -65,13 +64,7 @@ func getInstanceInfo(_ context.Context, _ io.ReadWriteCloser, extraData []byte)
 	var extraData64 [64]byte
 	copy(extraData64[:], extraData)
 
-	device, err := sevclient.OpenDevice()
-	if err != nil {
-		return nil, fmt.Errorf("opening sev device: %w", err)
-	}
-	defer device.Close()
-
-	report, certs, err := sevclient.GetRawExtendedReportAtVmpl(device, extraData64, 0)
+	report, certs, err := snp.GetExtendedReport(extraData64)
 	if err != nil {
 		return nil, fmt.Errorf("getting extended report: %w", err)
 	}

internal/attestation/snp/BUILD.bazel

@@ -9,6 +9,7 @@ go_library(
     deps = [
         "//internal/attestation",
         "@com_github_google_go_sev_guest//abi",
+        "@com_github_google_go_sev_guest//client",
         "@com_github_google_go_sev_guest//kds",
         "@com_github_google_go_sev_guest//proto/sevsnp",
         "@com_github_google_go_sev_guest//verify/trust",

internal/attestation/snp/snp.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/edgelesssys/constellation/v2/internal/attestation"
 	"github.com/google/go-sev-guest/abi"
+	"github.com/google/go-sev-guest/client"
 	"github.com/google/go-sev-guest/kds"
 	spb "github.com/google/go-sev-guest/proto/sevsnp"
 	"github.com/google/go-sev-guest/verify/trust"
@@ -32,6 +33,26 @@ func Product() *spb.SevProduct {
 	return &spb.SevProduct{Name: spb.SevProduct_SEV_PRODUCT_MILAN, Stepping: 0} // Milan-B0
 }
 
+// GetExtendedReport retrieves the extended SNP report from the CVM.
+func GetExtendedReport(reportData [64]byte) (report, certChain []byte, err error) {
+	qp, err := client.GetLeveledQuoteProvider()
+	if err != nil {
+		return nil, nil, fmt.Errorf("getting quote provider: %w", err)
+	}
+	quote, err := qp.GetRawQuoteAtLevel(reportData, 0)
+	if err != nil {
+		return nil, nil, fmt.Errorf("getting extended report: %w", err)
+	}
+
+	// Parse the report and certificate chain from the quote.
+	report = quote
+	if len(quote) > abi.ReportSize {
+		report = quote[:abi.ReportSize]
+		certChain = quote[abi.ReportSize:]
+	}
+	return report, certChain, nil
+}
+
 // InstanceInfo contains the necessary information to establish trust in a SNP CVM.
 type InstanceInfo struct {
 	// ReportSigner is the PEM-encoded certificate used to validate the attestation report's signature.
@@ -110,7 +131,7 @@ func (a *InstanceInfo) AttestationWithCerts(getter trust.HTTPSGetter,
 		return nil, fmt.Errorf("converting report to proto: %w", err)
 	}
 
-	productName := kds.ProductString(Product())
+	productName := kds.ProductLine(Product())
 
 	att := &spb.Attestation{
 		Report:           report,

internal/verify/verify.go

@@ -131,7 +131,7 @@ func getCertChain(cfg config.AttestationCfg) ([]byte, error) {
 	}
 
 	if awsCfg.AMDSigningKey.Equal(config.Certificate{}) {
-		certs, err := trust.GetProductChain(kds.ProductString(snp.Product()), abi.VlekReportSigner, trust.DefaultHTTPSGetter())
+		certs, err := trust.GetProductChain(kds.ProductLine(snp.Product()), abi.VlekReportSigner, trust.DefaultHTTPSGetter())
 		if err != nil {
 			return nil, fmt.Errorf("getting product certificate chain: %w", err)
 		}

internal/versions/versions.go

@@ -173,11 +173,11 @@ const (
 	// NodeMaintenanceOperatorImage is the image for the node maintenance operator.
 	NodeMaintenanceOperatorImage = "quay.io/medik8s/node-maintenance-operator:v0.15.0@sha256:8cb8dad93283268282c30e75c68f4bd76b28def4b68b563d2f9db9c74225d634" // renovate:container
 	// LogstashImage is the container image of logstash, used for log collection by debugd.
-	LogstashImage = "ghcr.io/edgelesssys/constellation/logstash-debugd:v2.15.0-pre.0.20231220180720-ced03202a944@sha256:54e0beb2fad83509c1d79c866652bdd94125ce5a4c9947be8c63cd74a2079e70" // renovate:container
+	LogstashImage = "ghcr.io/edgelesssys/constellation/logstash-debugd:v2.17.0-pre.0.20240513104207-d76c9ac82de7@sha256:e7d77bea381354c58fd9783eb9e2b846b14ea846d531eb3c54d5dd89917908c4" // renovate:container
 	// FilebeatImage is the container image of filebeat, used for log collection by debugd.
-	FilebeatImage = "ghcr.io/edgelesssys/constellation/filebeat-debugd:v2.15.0-pre.0.20231220180720-ced03202a944@sha256:1a57ad12dd0d1a7514f2360f37108925e103e7d0e5b8f24b12e8f266b78d570e" // renovate:container
+	FilebeatImage = "ghcr.io/edgelesssys/constellation/filebeat-debugd:v2.17.0-pre.0.20240513104207-d76c9ac82de7@sha256:6567d682385c06b49f6d56fdf3f20d5c24809bbfced15b816f4717bf837fc776" // renovate:container
 	// MetricbeatImage is the container image of filebeat, used for log collection by debugd.
-	MetricbeatImage = "ghcr.io/edgelesssys/constellation/metricbeat-debugd:v2.15.0-pre.0.20231220180720-ced03202a944@sha256:60bdd7cd868841385da230d4eab4600235b22fe1b3e0e865dda3f9720534ea7e" // renovate:container
+	MetricbeatImage = "ghcr.io/edgelesssys/constellation/metricbeat-debugd:v2.17.0-pre.0.20240513104207-d76c9ac82de7@sha256:d16fe0371fbb383ca3cfa70d4d10ea5124fd87054bfac189ce5458ba350bb25d" // renovate:container
 
 	// currently supported versions.
 	//nolint:revive

rfc/016-node-access.md

@@ -0,0 +1,184 @@
+# Node Access
+
+## Background
+
+A production Constellation cluster is currently configured not to allow any kind of remote administrative access.
+This choice is deliberate: any mechanism for remote accesss can potentially be exploited, or may leak sensitive data.
+
+However, some operations on a Kubernetes cluster require some form of access to the nodes.
+A good class of examples are etcd cluster maintenance tasks, like backup and recovery, or emergency operations like removing a permanently failed member.
+Some kubeadm operations, like certificate rotation, also require some form of cluster access.
+
+While some tasks can be accomplished by DaemonSets, CronJobs and the like, relying on Kubernetes objects is insufficient.
+Executing commands in a Kubernetes pod may fail because Kubernetes is not healthy, etcd is bricked or the network is down.
+Administrative access to the nodes through a side channel would greatly help remediate, or at least debug, those situations.
+
+## Requirements
+
+Constellation admins can log into Constellation nodes for maintenance, subject to the following restrictions:
+
+* Access must be encrypted end-to-end to protect from CSP snooping.
+* Access must be possible even if the Kubernetes API server is down.
+
+Nice-to-have:
+
+* The method of access should not require long-term storage of a second secret.
+* The method of access should be time-limited.
+
+## Proposed Design
+
+Core to the proposal is [certificate-based authentication for OpenSSH](https://en.wikibooks.org/wiki/OpenSSH/Cookbook/Certificate-based_Authentication).
+We can derive a valid SSH key from the Constellation master secret.
+An OpenSSH server on the node accepts certificates issued by this CA key.
+Admins can derive the CA key from the master secret on demand, and issue certificates for arbitrary public keys.
+An example program is in the [Appendix](#appendix).
+
+### Key Details
+
+We use an HKDF to derive an ed25519 private key from the master secret.
+This private key acts as an SSH certificate authority, whose signed certs allow access to cluster nodes.
+Since the master secret is available to both the cluster owner and the nodes, no communication with the cluster is needed to mint valid certificates.
+The choice of curve allows to directly use the derived secret bytes as key.
+This makes the implementation deterministic, and thus the CA key recoverable.
+
+### Server-side Details
+
+An OpenSSH server is added to the node image software stack.
+It's configured with a `TrustedUserCAKeys` file and a `RevokedKeys` file, both being empty on startup.
+All other means of authentication are disabled.
+
+After initialization, the bootstrapper fills the `TrustedUserCAKeys` file with the derived CA's public key.
+Joining nodes send their public host key as part of the `IssueJoinTokenRequest` and receive the CA certificate and an indefinitely valid certificate as response.
+
+The `RevokedKeys` KRL is an option for the cluster administrator to revoke keys, but it's not managed by Constellation.
+
+### Client-side Details
+
+A new `ssh` subcommand is added to the CLI.
+The exact name is TBD, but it should fit in with other key-related activity, like generating volume keys.
+It takes the master secret file and an SSH pub key file as arguments, and writes a certificate to stdout.
+Optional arguments may include principals or vailidity period.
+The implementation could roughly follow the PoC in the [Appendix](#appendix).
+
+As an extension, the subcommand could allow generating a key pair and a matching certificate in a temp dir, and `exec` the ssh program directly.
+This would encourage use of very short-lived certificates.
+
+## Security Considerations
+
+Exposing an additional service to the outside world increases the attack surface.
+We propose the following mitigations:
+
+1. The SSH port is only exposed to the VPC.
+   This restricts the attackers to malicious co-tenants and the CSP.
+   In an emergency, admins need to add a load balancer to be able to reach the nodes.
+2. A hardened OpenSSH config only allows the options strictly necessary for the scheme proposed here.
+   Authorized keys and passwords must be disabled.
+   Cipher suites should be restricted. etc.
+
+## Alternatives Considered
+
+### Enable Serial Console
+
+Serial consoles for cloud VMs are tunnelled through the CSP in the clear.
+To make this solution secure, an encrypted channel would need to be established on top of the serial connection.
+The author is not aware of any software providing such a channel.
+
+### SSH with Authorized Keys
+
+We could ask users to add a public key to their `constellation-conf.yaml` and add that to `/root/.ssh/authorized_keys` after joining.
+This would require the cluster owner to permanently manage a second secret, and there would be no built-in way to revoke access.
+
+### Debug Pod
+
+Some node administration tasks can be performed with a [debug pod].
+If privileged access is required, it's usually necessary to schedule a custom pod.
+This only works if the Kubernetes API server is still processing requests, pods can be scheduled on the target node and the network allows connecting to it.
+
+[debug pod]: https://kubernetes.io/docs/tasks/debug/debug-cluster/kubectl-node-debug/
+
+### Host an Admin API Server
+
+There are alternatives to SSH that allow fine-grained authorization of node operations.
+An example would be [SansShell], which verifies node access requests with a policy.
+Setting up such a tool requires a detailed understanding of the use cases, of which some might be hard to foresee.
+This may be better suited as an extension of the low-level emergency access mechanisms.
+
+[SansShell]: https://www.snowflake.com/blog/sansshell-local-host-agent/
+
+## Appendix
+
+A proof-of-concept implementation of the certificate generation.
+Constellation nodes would stop after deriving the CA public key.
+
+```golang
+package main
+
+import (
+	"crypto/ed25519"
+	"crypto/rand"
+	"crypto/sha256"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"time"
+
+	"golang.org/x/crypto/hkdf"
+	"golang.org/x/crypto/ssh"
+)
+
+type secret struct {
+	Key  []byte `json:"key,omitempty"`
+	Salt []byte `json:"salt,omitempty"`
+}
+
+var permissions = ssh.Permissions{
+	Extensions: map[string]string{
+		"permit-port-forwarding": "yes",
+		"permit-pty":             "yes",
+	},
+}
+
+func main() {
+	masterSecret := flag.String("secret", "", "")
+	flag.Parse()
+
+	secretJSON, err := os.ReadFile(*masterSecret)
+	must(err)
+	var secret secret
+	must(json.Unmarshal(secretJSON, &secret))
+
+	hkdf := hkdf.New(sha256.New, secret.Key, secret.Salt, []byte("ssh-ca"))
+
+	_, priv, err := ed25519.GenerateKey(hkdf)
+	must(err)
+
+	ca, err := ssh.NewSignerFromSigner(priv)
+	must(err)
+
+	log.Printf("CA KEY: %s", string(ssh.MarshalAuthorizedKey(ca.PublicKey())))
+
+	buf, err := os.ReadFile(flag.Arg(0))
+	must(err)
+	pub, _, _, _, err := ssh.ParseAuthorizedKey(buf)
+	must(err)
+	certificate := ssh.Certificate{
+		Key:             pub,
+		CertType:        ssh.UserCert,
+		ValidAfter:      uint64(time.Now().Unix()),
+		ValidBefore:     uint64(time.Now().Add(24 * time.Hour).Unix()),
+		ValidPrincipals: []string{"root"},
+		Permissions:     permissions,
+	}
+	must(certificate.SignCert(rand.Reader, ca))
+
+	fmt.Printf("%s\n", string(ssh.MarshalAuthorizedKey(&certificate)))
+}
+
+func must(err error) {
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+```

s3proxy/deploy/s3proxy/values.yaml

@@ -3,7 +3,7 @@ awsAccessKeyID: "replaceme"
 awsSecretAccessKey: "replaceme"
 
 # Pod image to deploy.
-image: "ghcr.io/edgelesssys/constellation/s3proxy:v2.16.0-pre.0.20240221184016-522f2858c6ef"
+image: "ghcr.io/edgelesssys/constellation/s3proxy:v2.17.0"
 
 # Control if multipart uploads are blocked.
 allowMultipart: false