AB#1915 Local PCR calculation (#243)

* Add QEMU cloud-logging

* Add QEMU metadata endpoints to collect logs during cluster boot

* Send PCRs to QEMU metadata if boot fails on Azure or GCP

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

coordinator/cloudprovider/qemu/logger.go

@@ -0,0 +1,32 @@
+package qemu
+
+import (
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+// Logger is a Cloud Logger for QEMU.
+type Logger struct{}
+
+// NewLogger creates a new Cloud Logger for QEMU.
+func NewLogger() *Logger {
+	return &Logger{}
+}
+
+// Disclose writes log information to QEMU's cloud log.
+// This is done by sending a POST request to the QEMU's metadata endpoint.
+func (l *Logger) Disclose(msg string) {
+	url := &url.URL{
+		Scheme: "http",
+		Host:   qemuMetadataEndpoint,
+		Path:   "/log",
+	}
+
+	_, _ = http.Post(url.String(), "application/json", strings.NewReader(msg))
+}
+
+// Close is a no-op.
+func (l *Logger) Close() error {
+	return nil
+}

coordinator/cmd/coordinator/main.go

@@ -162,9 +162,7 @@ func main() {
 		issuer = qemu.NewIssuer()
 		validator = qemu.NewValidator(pcrs)
 
-		// no support for cloud logging in qemu
+		cloudLogger = qemucloud.NewLogger()
-		cloudLogger = &logging.NopLogger{}
-
 		metadata := &qemucloud.Metadata{}
 		pcrsJSON, err := json.Marshal(pcrs)
 		if err != nil {

hack/qemu-metadata-api/main.go

@@ -5,7 +5,9 @@ import (
 
 	"github.com/edgelesssys/constellation/hack/qemu-metadata-api/server"
 	"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
+	"github.com/edgelesssys/constellation/internal/file"
 	"github.com/edgelesssys/constellation/internal/logger"
+	"github.com/spf13/afero"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"libvirt.org/go/libvirt"
@@ -23,7 +25,7 @@ func main() {
 	}
 	defer conn.Close()
 
-	serv := server.New(log, &virtwrapper.Connect{Conn: conn})
+	serv := server.New(log, &virtwrapper.Connect{Conn: conn}, file.NewHandler(afero.NewOsFs()))
 	if err := serv.ListenAndServe(*bindPort); err != nil {
 		log.With(zap.Error(err)).Fatalf("Failed to serve")
 	}

hack/qemu-metadata-api/server/server.go

@@ -1,8 +1,10 @@
 package server
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net"
 	"net/http"
 	"strings"
@@ -10,19 +12,24 @@ import (
 	"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
 	"github.com/edgelesssys/constellation/coordinator/role"
 	"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
+	"github.com/edgelesssys/constellation/internal/file"
 	"github.com/edgelesssys/constellation/internal/logger"
 	"go.uber.org/zap"
 )
 
+const exportedPCRsDir = "/pcrs/"
+
 type Server struct {
 	log  *logger.Logger
 	virt virConnect
+	file file.Handler
 }
 
-func New(log *logger.Logger, conn virConnect) *Server {
+func New(log *logger.Logger, conn virConnect, file file.Handler) *Server {
 	return &Server{
 		log:  log,
 		virt: conn,
+		file: file,
 	}
 }
 
@@ -30,6 +37,8 @@ func (s *Server) ListenAndServe(port string) error {
 	mux := http.NewServeMux()
 	mux.Handle("/self", http.HandlerFunc(s.listSelf))
 	mux.Handle("/peers", http.HandlerFunc(s.listPeers))
+	mux.Handle("/log", http.HandlerFunc(s.postLog))
+	mux.Handle("/pcrs", http.HandlerFunc(s.exportPCRs))
 
 	server := http.Server{
 		Handler: mux,
@@ -101,6 +110,98 @@ func (s *Server) listPeers(w http.ResponseWriter, r *http.Request) {
 	log.Infof("Request successful")
 }
 
+// postLog writes implements cloud-logging for QEMU instances.
+func (s *Server) postLog(w http.ResponseWriter, r *http.Request) {
+	log := s.log.With(zap.String("peer", r.RemoteAddr))
+	if r.Method != http.MethodPost {
+		log.With(zap.String("method", r.Method)).Errorf("Invalid method for /log")
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	log.Infof("Serving POST request for /log")
+
+	if r.Body == nil {
+		log.Errorf("Request body is empty")
+		http.Error(w, "Request body is empty", http.StatusBadRequest)
+		return
+	}
+
+	msg, err := io.ReadAll(r.Body)
+	if err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to read request body")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	log.With(zap.String("message", string(msg))).Infof("Cloud-logging entry")
+}
+
+// exportPCRs allows QEMU instances to export their TPM state during boot.
+// This can be used to check expected PCRs for GCP/Azure cloud images locally.
+func (s *Server) exportPCRs(w http.ResponseWriter, r *http.Request) {
+	log := s.log.With(zap.String("peer", r.RemoteAddr))
+	if r.Method != http.MethodPost {
+		log.With(zap.String("method", r.Method)).Errorf("Invalid method for /log")
+		http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
+		return
+	}
+
+	log.Infof("Serving POST request for /pcrs")
+
+	if r.Body == nil {
+		log.Errorf("Request body is empty")
+		http.Error(w, "Request body is empty", http.StatusBadRequest)
+		return
+	}
+
+	// unmarshal the request body into a map of PCRs
+	var pcrs map[uint32][]byte
+	if err := json.NewDecoder(r.Body).Decode(&pcrs); err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to read request body")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// get name of the node sending the export request
+	var nodeName string
+	peers, err := s.listAll()
+	if err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to list peer metadata")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+	remoteIP, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to parse remote address")
+		http.Error(w, fmt.Sprintf("Failed to parse remote address: %s\n", err), http.StatusInternalServerError)
+		return
+	}
+	for _, peer := range peers {
+		if peer.PublicIPs[0] == remoteIP {
+			nodeName = peer.Name
+		}
+	}
+
+	// write PCRs as JSON and YAML to disk
+	if err := s.file.WriteJSON(exportedPCRsDir+nodeName+"_pcrs.json", pcrs, file.OptOverwrite); err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to write pcrs to JSON")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+
+	// convert []byte to base64 encoded strings for YAML encoding
+	pcrsYAML := make(map[uint32]string)
+	for k, v := range pcrs {
+		pcrsYAML[k] = base64.StdEncoding.EncodeToString(v)
+	}
+	if err := s.file.WriteYAML(exportedPCRsDir+nodeName+"_pcrs.yaml", pcrsYAML, file.OptOverwrite); err != nil {
+		log.With(zap.Error(err)).Errorf("Failed to write pcrs to YAML")
+		http.Error(w, err.Error(), http.StatusInternalServerError)
+		return
+	}
+}
+
 // listAll returns a list of all active peers.
 func (s *Server) listAll() ([]cloudtypes.Instance, error) {
 	net, err := s.virt.LookupNetworkByName("constellation")

hack/qemu-metadata-api/server/server_test.go

@@ -6,11 +6,14 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/edgelesssys/constellation/coordinator/cloudprovider/cloudtypes"
 	"github.com/edgelesssys/constellation/hack/qemu-metadata-api/virtwrapper"
+	"github.com/edgelesssys/constellation/internal/file"
 	"github.com/edgelesssys/constellation/internal/logger"
+	"github.com/spf13/afero"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"libvirt.org/go/libvirt"
@@ -63,7 +66,7 @@ func TestListAll(t *testing.T) {
 		t.Run(name, func(t *testing.T) {
 			assert := assert.New(t)
 
-			server := New(logger.NewTest(t), tc.connect)
+			server := New(logger.NewTest(t), tc.connect, file.Handler{})
 
 			res, err := server.listAll()
 
@@ -140,7 +143,7 @@ func TestListSelf(t *testing.T) {
 			assert := assert.New(t)
 			require := require.New(t)
 
-			server := New(logger.NewTest(t), tc.connect)
+			server := New(logger.NewTest(t), tc.connect, file.Handler{})
 
 			req, err := http.NewRequest(http.MethodGet, "http://192.0.0.1/self", nil)
 			require.NoError(err)
@@ -202,7 +205,7 @@ func TestListPeers(t *testing.T) {
 			assert := assert.New(t)
 			require := require.New(t)
 
-			server := New(logger.NewTest(t), tc.connect)
+			server := New(logger.NewTest(t), tc.connect, file.Handler{})
 
 			req, err := http.NewRequest(http.MethodGet, "http://192.0.0.1/peers", nil)
 			require.NoError(err)
@@ -226,6 +229,147 @@ func TestListPeers(t *testing.T) {
 	}
 }
 
+func TestPostLog(t *testing.T) {
+	testCases := map[string]struct {
+		remoteAddr string
+		message    io.Reader
+		method     string
+		wantErr    bool
+	}{
+		"success": {
+			remoteAddr: "192.0.100.1:1234",
+			method:     http.MethodPost,
+			message:    strings.NewReader("test message"),
+		},
+		"no body": {
+			remoteAddr: "192.0.100.1:1234",
+			method:     http.MethodPost,
+			message:    nil,
+			wantErr:    true,
+		},
+		"incorrect method": {
+			remoteAddr: "192.0.100.1:1234",
+			method:     http.MethodGet,
+			message:    strings.NewReader("test message"),
+			wantErr:    true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			server := New(logger.NewTest(t), &stubConnect{}, file.NewHandler(afero.NewMemMapFs()))
+
+			req, err := http.NewRequest(tc.method, "http://192.0.0.1/logs", tc.message)
+			require.NoError(err)
+			req.RemoteAddr = tc.remoteAddr
+
+			w := httptest.NewRecorder()
+			server.postLog(w, req)
+
+			if tc.wantErr {
+				assert.NotEqual(http.StatusOK, w.Code)
+			} else {
+				assert.Equal(http.StatusOK, w.Code)
+			}
+		})
+	}
+}
+
+func TestExportPCRs(t *testing.T) {
+	defaultConnect := &stubConnect{
+		network: stubNetwork{
+			leases: []libvirt.NetworkDHCPLease{
+				{
+					IPaddr:   "192.0.100.1",
+					Hostname: "control-plane-0",
+				},
+			},
+		},
+	}
+
+	testCases := map[string]struct {
+		remoteAddr string
+		connect    *stubConnect
+		message    string
+		method     string
+		wantErr    bool
+	}{
+		"success": {
+			remoteAddr: "192.0.100.1:1234",
+			connect:    defaultConnect,
+			method:     http.MethodPost,
+			message:    mustMarshal(t, map[uint32][]byte{0: []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")}),
+		},
+		"incorrect method": {
+			remoteAddr: "192.0.100.1:1234",
+			connect:    defaultConnect,
+			message:    mustMarshal(t, map[uint32][]byte{0: []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")}),
+			method:     http.MethodGet,
+			wantErr:    true,
+		},
+		"listAll error": {
+			remoteAddr: "192.0.100.1:1234",
+			connect: &stubConnect{
+				getNetworkErr: errors.New("error"),
+			},
+			message: mustMarshal(t, map[uint32][]byte{0: []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")}),
+			method:  http.MethodPost,
+			wantErr: true,
+		},
+		"invalid message": {
+			remoteAddr: "192.0.100.1:1234",
+			connect:    defaultConnect,
+			method:     http.MethodPost,
+			message:    "message",
+			wantErr:    true,
+		},
+		"invalid remote address": {
+			remoteAddr: "localhost",
+			connect:    defaultConnect,
+			method:     http.MethodPost,
+			message:    mustMarshal(t, map[uint32][]byte{0: []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")}),
+			wantErr:    true,
+		},
+	}
+
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			assert := assert.New(t)
+			require := require.New(t)
+
+			file := file.NewHandler(afero.NewMemMapFs())
+			server := New(logger.NewTest(t), tc.connect, file)
+
+			req, err := http.NewRequest(tc.method, "http://192.0.0.1/pcrs", strings.NewReader(tc.message))
+			require.NoError(err)
+			req.RemoteAddr = tc.remoteAddr
+
+			w := httptest.NewRecorder()
+			server.exportPCRs(w, req)
+
+			if tc.wantErr {
+				assert.NotEqual(http.StatusOK, w.Code)
+				return
+			}
+
+			assert.Equal(http.StatusOK, w.Code)
+			output, err := file.Read(exportedPCRsDir + tc.connect.network.leases[0].Hostname + "_pcrs.json")
+			require.NoError(err)
+			assert.JSONEq(tc.message, string(output))
+		})
+	}
+}
+
+func mustMarshal(t *testing.T, v interface{}) string {
+	t.Helper()
+	b, err := json.Marshal(v)
+	require.NoError(t, err)
+	return string(b)
+}
+
 type stubConnect struct {
 	network       stubNetwork
 	getNetworkErr error

state/cmd/main.go

@@ -1,9 +1,12 @@
 package main
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
 	"flag"
-	"fmt"
+	"net/http"
+	"net/url"
 	"path/filepath"
 	"strings"
 	"time"
@@ -21,6 +24,8 @@ import (
 	"github.com/edgelesssys/constellation/state/keyservice"
 	"github.com/edgelesssys/constellation/state/mapper"
 	"github.com/edgelesssys/constellation/state/setup"
+	tpmClient "github.com/google/go-tpm-tools/client"
+	"github.com/google/go-tpm/tpm2"
 	"github.com/spf13/afero"
 	"go.uber.org/zap"
 )
@@ -42,13 +47,16 @@ func main() {
 
 	// set up metadata API and quote issuer for aTLS connections
 	var err error
-	var diskPathErr error
 	var diskPath string
 	var issuer core.QuoteIssuer
 	var metadata core.ProviderMetadata
 	switch strings.ToLower(*csp) {
 	case "azure":
-		diskPath, diskPathErr = filepath.EvalSymlinks(azureStateDiskPath)
+		diskPath, err = filepath.EvalSymlinks(azureStateDiskPath)
+		if err != nil {
+			_ = exportPCRs()
+			log.With(zap.Error(err)).Fatalf("Unable to resolve Azure state disk path")
+		}
 		metadata, err = azurecloud.NewMetadata(context.Background())
 		if err != nil {
 			log.With(zap.Error).Fatalf("Failed to create Azure metadata API")
@@ -56,7 +64,11 @@ func main() {
 		issuer = azure.NewIssuer()
 
 	case "gcp":
-		diskPath, diskPathErr = filepath.EvalSymlinks(gcpStateDiskPath)
+		diskPath, err = filepath.EvalSymlinks(gcpStateDiskPath)
+		if err != nil {
+			_ = exportPCRs()
+			log.With(zap.Error(err)).Fatalf("Unable to resolve GCP state disk path")
+		}
 		issuer = gcp.NewIssuer()
 		gcpClient, err := gcpcloud.NewClient(context.Background())
 		if err != nil {
@@ -70,10 +82,7 @@ func main() {
 		metadata = &qemucloud.Metadata{}
 
 	default:
-		diskPathErr = fmt.Errorf("csp %q is not supported by Constellation", *csp)
+		log.Fatalf("CSP %s is not supported by Constellation", *csp)
-	}
-	if diskPathErr != nil {
-		log.With(zap.Error(diskPathErr)).Fatalf("Unable to determine state disk path")
 	}
 
 	// initialize device mapper
@@ -103,3 +112,28 @@ func main() {
 		log.With(zap.Error(err)).Fatalf("Failed to prepare state disk")
 	}
 }
+
+// exportPCRs tries to export the node's PCRs to QEMU's metadata API.
+// This function is called when an Azure or GCP image boots, but is unable to find a state disk.
+// This happens when we boot such an image in QEMU.
+// We can use this to calculate the PCRs of the image locally.
+func exportPCRs() error {
+	// get TPM state
+	pcrs, err := vtpm.GetSelectedPCRs(vtpm.OpenVTPM, tpmClient.FullPcrSel(tpm2.AlgSHA256))
+	if err != nil {
+		return err
+	}
+	pcrsPretty, err := json.Marshal(pcrs)
+	if err != nil {
+		return err
+	}
+
+	// send PCRs to metadata API
+	url := &url.URL{
+		Scheme: "http",
+		Host:   "10.42.0.1:8080", // QEMU metadata endpoint
+		Path:   "/pcrs",
+	}
+	_, err = http.Post(url.String(), "application/json", bytes.NewBuffer(pcrsPretty))
+	return err
+}

terraform/libvirt/README.md

@@ -11,7 +11,7 @@ Prerequisite:
 Optional: Write a `terraform.tfvars` file in the terraform workspace (`terraform/libvirt`), defining required variables and overriding optional variables.
 See [variables.tf](./variables.tf) for a description of all available variables.
 ```tfvars
-constellation_coreos_image_qcow2="/path/to/image.qcow2"
+constellation_coreos_image="/path/to/image.qcow2"
 # optional other vars, uncomment and change as needed
 # control_plane_count=3
 # worker_count=2

terraform/libvirt/main.tf

@@ -39,6 +39,11 @@ resource "docker_container" "qemu-metadata" {
     target = "/var/run/libvirt/libvirt-sock"
     type = "bind" 
   }
+  mounts {
+    source = var.metadata_api_log_dir
+    target = "/pcrs"
+    type = "bind"
+  }
 }
 
 module "control_plane" {
@@ -80,8 +85,8 @@ resource "libvirt_pool" "cluster" {
 resource "libvirt_volume" "constellation_coreos_image" {
   name   = "constellation-coreos-image"
   pool   = libvirt_pool.cluster.name
-  source = var.constellation_coreos_image_qcow2
+  source = var.constellation_coreos_image
-  format = "qcow2"
+  format = var.image_format
 }
 
 resource "libvirt_network" "constellation" {

terraform/libvirt/variables.tf

@@ -1,6 +1,12 @@
-variable "constellation_coreos_image_qcow2" {
+variable "constellation_coreos_image" {
   type        = string
-  description = "constellation OS qcow file path"
+  description = "constellation OS file path"
+}
+
+variable "image_format" {
+  type        = string
+  default     = "qcow2"
+  description = "image format"
 }
 
 variable "control_plane_count" {
@@ -45,3 +51,8 @@ variable "machine" {
   default     = "q35"
   description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
 }
+
+variable "metadata_api_log_dir" {
+  type = string
+  description = "directory to store metadata log files. This must be an absolute path"
+}