Enable integrity protection on boot (#300)

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

CHANGELOG.md

@@ -27,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Kubernetes version 1.24 is now supported.
 - Kubernetes version 1.22 is now supported.
 - Log the disk UUID to cloud logging for recovery.
+- Configurable disk type for Azure and GCP.
 
 ### Changed
 
@@ -47,6 +48,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - Create Kubernetes CA signed kubelet certificates on activation.
 - Add salt to key derivation
+- Enable integrity protection of state disks.
 
 ### Internal
 

CMakeLists.txt

@@ -68,4 +68,4 @@ add_custom_target(cdbg ALL
 add_test(NAME unit-main COMMAND go test -race -count=3 ./... WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})
 add_test(NAME unit-hack COMMAND go test -race -count=3 ./... WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/hack)
 add_test(NAME integration-mount COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v -v 9"  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/mount)
-add_test(NAME integration-dm COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v"  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/state)
+add_test(NAME integration-dm COMMAND bash -c "go test -tags integration -c ./test/ && sudo ./test.test -test.v"  WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/state/internal)

cli/internal/proto/recover.go

@@ -6,7 +6,7 @@ import (
 
 	"github.com/edgelesssys/constellation/internal/atls"
 	"github.com/edgelesssys/constellation/internal/grpc/atlscredentials"
-	"github.com/edgelesssys/constellation/state/keyservice/keyproto"
+	"github.com/edgelesssys/constellation/state/keyproto"
 	"google.golang.org/grpc"
 )
 

mount/cryptmapper/cryptmapper.go

@@ -36,7 +36,7 @@ var packageLock = sync.Mutex{}
 
 func init() {
 	cryptsetup.SetDebugLevel(cryptsetup.CRYPT_LOG_NORMAL)
-	cryptsetup.SetLogCallback(func(level int, message string) { klog.V(4).Infof("libcryptsetup: %s", message) })
+	cryptsetup.SetLogCallback(func(_ int, message string) { klog.V(4).Infof("libcryptsetup: %s", message) })
 }
 
 // KeyCreator is an interface to create data encryption keys.

proto/Dockerfile.gen-proto

@@ -26,7 +26,7 @@ RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v${GEN_GO_VER} && \
 
 ## disk-mapper keyservice api
 WORKDIR /disk-mapper
-COPY state/keyservice/keyproto/*.proto /disk-mapper
+COPY state/keyproto/*.proto /disk-mapper
 RUN protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative *.proto
 
 ## debugd service
@@ -54,7 +54,7 @@ COPY bootstrapper/initproto/*.proto /init
 RUN protoc --go_out=. --go_opt=paths=source_relative --go-grpc_out=. --go-grpc_opt=paths=source_relative *.proto
 
 FROM scratch as export
-COPY --from=build /disk-mapper/*.go state/keyservice/keyproto/
+COPY --from=build /disk-mapper/*.go state/keyproto/
 COPY --from=build /service/*.go debugd/service/
 COPY --from=build /kms/*.go kms/kmsproto/
 COPY --from=build /joinservice/*.go joinservice/joinproto/

state/cmd/main.go

@@ -21,9 +21,9 @@ import (
 	"github.com/edgelesssys/constellation/internal/cloud/metadata"
 	"github.com/edgelesssys/constellation/internal/constants"
 	"github.com/edgelesssys/constellation/internal/logger"
-	"github.com/edgelesssys/constellation/state/keyservice"
-	"github.com/edgelesssys/constellation/state/mapper"
-	"github.com/edgelesssys/constellation/state/setup"
+	"github.com/edgelesssys/constellation/state/internal/keyservice"
+	"github.com/edgelesssys/constellation/state/internal/mapper"
+	"github.com/edgelesssys/constellation/state/internal/setup"
 	tpmClient "github.com/google/go-tpm-tools/client"
 	"github.com/google/go-tpm/tpm2"
 	"github.com/spf13/afero"
@@ -86,7 +86,7 @@ func main() {
 	}
 
 	// initialize device mapper
-	mapper, err := mapper.New(diskPath)
+	mapper, err := mapper.New(diskPath, log)
 	if err != nil {
 		log.With(zap.Error(err)).Fatalf("Failed to initialize device mapper")
 	}

state/internal/keyservice/keyservice.go

@@ -14,7 +14,7 @@ import (
 	"github.com/edgelesssys/constellation/internal/logger"
 	"github.com/edgelesssys/constellation/internal/oid"
 	"github.com/edgelesssys/constellation/joinservice/joinproto"
-	"github.com/edgelesssys/constellation/state/keyservice/keyproto"
+	"github.com/edgelesssys/constellation/state/keyproto"
 	"go.uber.org/zap"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"

state/internal/keyservice/keyservice_test.go

@@ -14,7 +14,7 @@ import (
 	"github.com/edgelesssys/constellation/internal/logger"
 	"github.com/edgelesssys/constellation/internal/oid"
 	"github.com/edgelesssys/constellation/joinservice/joinproto"
-	"github.com/edgelesssys/constellation/state/keyservice/keyproto"
+	"github.com/edgelesssys/constellation/state/keyproto"
 	"github.com/stretchr/testify/assert"
 	"go.uber.org/goleak"
 	"google.golang.org/grpc"

state/internal/mapper/cryptdevice.go

@@ -7,6 +7,9 @@ type cryptDevice interface {
 	// Returns nil on success, or an error otherwise.
 	// C equivalent: crypt_activate_by_passphrase
 	ActivateByPassphrase(deviceName string, keyslot int, passphrase string, flags int) error
+	// ActivateByVolumeKey activates a device by using a volume key.
+	// Returns nil on success, or an error otherwise.
+	ActivateByVolumeKey(deviceName string, volumeKey string, volumeKeySize int, flags int) error
 	// Deactivate deactivates a device.
 	// Returns nil on success, or an error otherwise.
 	// C equivalent: crypt_deactivate
@@ -29,4 +32,7 @@ type cryptDevice interface {
 	// Returns nil on success, or an error otherwise.
 	// C equivalent: crypt_keyslot_add_by_volume_key
 	KeyslotAddByVolumeKey(keyslot int, volumeKey string, passphrase string) error
+	// Wipe removes existing data and clears the device for use with dm-integrity.
+	// Returns nil on success, or an error otherwise.
+	Wipe(devicePath string, pattern int, offset, length uint64, wipeBlockSize int, flags int, progress func(size, offset uint64) int) error
 }

state/internal/mapper/mapper.go

@@ -0,0 +1,148 @@
+package mapper
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/edgelesssys/constellation/internal/logger"
+	cryptsetup "github.com/martinjungblut/go-cryptsetup"
+	"go.uber.org/zap"
+)
+
+// packageLock is needed to block concurrent use of package functions, since libcryptsetup is not thread safe.
+// See: https://gitlab.com/cryptsetup/cryptsetup/-/issues/710
+// 		https://stackoverflow.com/questions/30553386/cryptsetup-backend-safe-with-multithreading
+var packageLock = sync.Mutex{}
+
+// Mapper handles actions for formating and mapping crypt devices.
+type Mapper struct {
+	device cryptDevice
+	log    *logger.Logger
+}
+
+// New creates a new crypt device for the device at path.
+func New(path string, log *logger.Logger) (*Mapper, error) {
+	packageLock.Lock()
+	device, err := cryptsetup.Init(path)
+	if err != nil {
+		return nil, fmt.Errorf("initializing crypt device for disk %q: %w", path, err)
+	}
+	return &Mapper{device: device, log: log}, nil
+}
+
+// Close closes and frees memory allocated for the crypt device.
+func (m *Mapper) Close() error {
+	defer packageLock.Unlock()
+	if m.device.Free() {
+		return nil
+	}
+	return errors.New("unable to close crypt device")
+}
+
+// IsLUKSDevice returns true if the device is formatted as a LUKS device.
+func (m *Mapper) IsLUKSDevice() bool {
+	return m.device.Load(cryptsetup.LUKS2{}) == nil
+}
+
+// DiskUUID gets the device's UUID.
+func (m *Mapper) DiskUUID() string {
+	return strings.ToLower(m.device.GetUUID())
+}
+
+// FormatDisk formats the disk and adds passphrase in keyslot 0.
+func (m *Mapper) FormatDisk(passphrase string) error {
+	luksParams := cryptsetup.LUKS2{
+		SectorSize: 4096,
+		Integrity:  "hmac(sha256)",
+		PBKDFType: &cryptsetup.PbkdfType{
+			// Use low memory recommendation from https://datatracker.ietf.org/doc/html/rfc9106#section-7
+			Type:            "argon2id",
+			TimeMs:          2000,
+			Iterations:      3,
+			ParallelThreads: 4,
+			MaxMemoryKb:     65536, // ~64MiB
+		},
+	}
+
+	genericParams := cryptsetup.GenericParams{
+		Cipher:        "aes",
+		CipherMode:    "xts-plain64",
+		VolumeKeySize: 96, // 32*2 bytes for aes-xts-plain64 encryption, 32 bytes for hmac(sha256) integrity
+	}
+
+	if err := m.device.Format(luksParams, genericParams); err != nil {
+		return fmt.Errorf("formatting disk: %w", err)
+	}
+
+	if err := m.device.KeyslotAddByVolumeKey(0, "", passphrase); err != nil {
+		return fmt.Errorf("adding keyslot: %w", err)
+	}
+
+	// wipe using 64MiB block size
+	if err := m.Wipe(67108864); err != nil {
+		return fmt.Errorf("wiping disk: %w", err)
+	}
+
+	return nil
+}
+
+// MapDisk maps a crypt device to /dev/mapper/target using the provided passphrase.
+func (m *Mapper) MapDisk(target, passphrase string) error {
+	if err := m.device.ActivateByPassphrase(target, 0, passphrase, 0); err != nil {
+		return fmt.Errorf("mapping disk as %q: %w", target, err)
+	}
+	return nil
+}
+
+// UnmapDisk removes the mapping of target.
+func (m *Mapper) UnmapDisk(target string) error {
+	return m.device.Deactivate(target)
+}
+
+// Wipe overwrites the device with zeros to initialize integrity checksums.
+func (m *Mapper) Wipe(blockWipeSize int) error {
+	// Activate as temporary device using the internal volume key
+	tmpDevice := "tmp-cryptsetup-integrity"
+	if err := m.device.ActivateByVolumeKey(tmpDevice, "", 0, (cryptsetup.CRYPT_ACTIVATE_PRIVATE | cryptsetup.CRYPT_ACTIVATE_NO_JOURNAL)); err != nil {
+		return fmt.Errorf("activating as temporary device: %w", err)
+	}
+
+	// set progress logging callback once every 30 seconds
+	ticker := time.NewTicker(30 * time.Second)
+	firstReq := make(chan struct{}, 1)
+	firstReq <- struct{}{}
+	defer ticker.Stop()
+	logProgress := func(size, offset uint64) {
+		prog := (float64(offset) / float64(size)) * 100
+		m.log.With(zap.String("progress", fmt.Sprintf("%.2f%%", prog))).Infof("Wiping disk")
+	}
+
+	progressCallback := func(size, offset uint64) int {
+		select {
+		case <-firstReq:
+			logProgress(size, offset)
+		case <-ticker.C:
+			logProgress(size, offset)
+		default:
+		}
+
+		return 0
+	}
+
+	start := time.Now()
+	// wipe the device
+	if err := m.device.Wipe("/dev/mapper/"+tmpDevice, cryptsetup.CRYPT_WIPE_ZERO, 0, 0, blockWipeSize, 0, progressCallback); err != nil {
+		return fmt.Errorf("wiping disk: %w", err)
+	}
+	m.log.With(zap.Duration("duration", time.Since(start))).Infof("Wiping disk successful")
+
+	// Deactivate the temporary device
+	if err := m.device.Deactivate(tmpDevice); err != nil {
+		return fmt.Errorf("deactivating temporary device: %w", err)
+	}
+
+	return nil
+}

state/internal/setup/setup_test.go

@@ -261,12 +261,12 @@ func (s *stubMapper) DiskUUID() string {
 	return s.uuid
 }
 
-func (s *stubMapper) FormatDisk(passphrase string) error {
+func (s *stubMapper) FormatDisk(string) error {
 	s.formatDiskCalled = true
 	return s.formatDiskErr
 }
 
-func (s *stubMapper) MapDisk(target string, passphrase string) error {
+func (s *stubMapper) MapDisk(string, string) error {
 	if s.mapDiskRepeatedCalls == 0 {
 		s.mapDiskErr = nil
 	}

state/internal/test/benchmark_test.go

@@ -0,0 +1,108 @@
+//go:build integration
+
+package integration
+
+import (
+	"fmt"
+	"math"
+	"testing"
+
+	"github.com/edgelesssys/constellation/internal/logger"
+	"github.com/edgelesssys/constellation/state/internal/mapper"
+	"github.com/martinjungblut/go-cryptsetup"
+	"go.uber.org/zap/zapcore"
+)
+
+func BenchmarkMapper(b *testing.B) {
+	cryptsetup.SetDebugLevel(cryptsetup.CRYPT_LOG_ERROR)
+	cryptsetup.SetLogCallback(func(_ int, message string) { fmt.Println(message) })
+
+	testPath := *diskPath
+	if testPath == "" {
+		// no disk specified, use 1GB loopback disk
+		testPath = devicePath
+		if err := setup(1); err != nil {
+			b.Fatal("Failed to setup test environment:", err)
+		}
+
+		defer func() {
+			if err := teardown(); err != nil {
+				b.Fatal("failed to delete test disk:", err)
+			}
+		}()
+	}
+
+	passphrase := "benchmark"
+	mapper, err := mapper.New(testPath, logger.New(logger.PlainLog, zapcore.InfoLevel))
+	if err != nil {
+		b.Fatal("Failed to create mapper:", err)
+	}
+	defer mapper.Close()
+
+	if err := mapper.FormatDisk(passphrase); err != nil {
+		b.Fatal("Failed to format disk:", err)
+	}
+
+	testCases := map[string]struct {
+		wipeBlockSize int
+	}{
+		"16KiB": {
+			wipeBlockSize: int(math.Pow(2, 14)),
+		},
+		"32KiB": {
+			wipeBlockSize: int(math.Pow(2, 15)),
+		},
+		"64KiB": {
+			wipeBlockSize: int(math.Pow(2, 16)),
+		},
+		"128KiB": {
+			wipeBlockSize: int(math.Pow(2, 17)),
+		},
+		"256KiB": {
+			wipeBlockSize: int(math.Pow(2, 18)),
+		},
+		"512KiB": {
+			wipeBlockSize: int(math.Pow(2, 19)),
+		},
+		"1MiB": {
+			wipeBlockSize: int(math.Pow(2, 20)),
+		},
+		"2MiB": {
+			wipeBlockSize: int(math.Pow(2, 21)),
+		},
+		"4MiB": {
+			wipeBlockSize: int(math.Pow(2, 22)),
+		},
+		"8MiB": {
+			wipeBlockSize: int(math.Pow(2, 23)),
+		},
+		"16MiB": {
+			wipeBlockSize: int(math.Pow(2, 24)),
+		},
+		"32MiB": {
+			wipeBlockSize: int(math.Pow(2, 25)),
+		},
+		"64MiB": {
+			wipeBlockSize: int(math.Pow(2, 26)),
+		},
+		"128MiB": {
+			wipeBlockSize: int(math.Pow(2, 27)),
+		},
+		"256MiB": {
+			wipeBlockSize: int(math.Pow(2, 28)),
+		},
+		"512MiB": {
+			wipeBlockSize: int(math.Pow(2, 29)),
+		},
+	}
+
+	for name, tc := range testCases {
+		b.Run(name, func(b *testing.B) {
+			for i := 0; i < b.N; i++ {
+				if err := mapper.Wipe(tc.wipeBlockSize); err != nil {
+					b.Fatal("Failed to wipe disk:", err)
+				}
+			}
+		})
+	}
+}

state/internal/test/integration_test.go

@@ -4,6 +4,7 @@ package integration
 
 import (
 	"context"
+	"flag"
 	"fmt"
 	"net"
 	"os"
@@ -17,9 +18,9 @@ import (
 	"github.com/edgelesssys/constellation/internal/grpc/atlscredentials"
 	"github.com/edgelesssys/constellation/internal/logger"
 	"github.com/edgelesssys/constellation/internal/oid"
-	"github.com/edgelesssys/constellation/state/keyservice"
-	"github.com/edgelesssys/constellation/state/keyservice/keyproto"
-	"github.com/edgelesssys/constellation/state/mapper"
+	"github.com/edgelesssys/constellation/state/internal/keyservice"
+	"github.com/edgelesssys/constellation/state/internal/mapper"
+	"github.com/edgelesssys/constellation/state/keyproto"
 	"github.com/martinjungblut/go-cryptsetup"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -32,8 +33,10 @@ const (
 	mappedDevice = "mappedDevice"
 )
 
-func setup() error {
-	return exec.Command("/bin/dd", "if=/dev/zero", fmt.Sprintf("of=%s", devicePath), "bs=64M", "count=1").Run()
+var diskPath = flag.String("disk", "", "Path to the disk to use for the benchmark")
+
+func setup(sizeGB int) error {
+	return exec.Command("/bin/dd", "if=/dev/random", fmt.Sprintf("of=%s", devicePath), "bs=1G", fmt.Sprintf("count=%d", sizeGB)).Run()
 }
 
 func teardown() error {
@@ -41,6 +44,8 @@ func teardown() error {
 }
 
 func TestMain(m *testing.M) {
+	flag.Parse()
+
 	if os.Getuid() != 0 {
 		fmt.Printf("This test suite requires root privileges, as libcrypsetup uses the kernel's device mapper.\n")
 		os.Exit(1)
@@ -56,14 +61,15 @@ func TestMain(m *testing.M) {
 }
 
 func TestMapper(t *testing.T) {
-	cryptsetup.SetDebugLevel(cryptsetup.CRYPT_LOG_VERBOSE)
-	cryptsetup.SetLogCallback(func(level int, message string) { fmt.Println(message) })
+	cryptsetup.SetDebugLevel(cryptsetup.CRYPT_LOG_ERROR)
+	cryptsetup.SetLogCallback(func(_ int, message string) { fmt.Println(message) })
+
 	assert := assert.New(t)
 	require := require.New(t)
-	require.NoError(setup(), "failed to setup test disk")
+	require.NoError(setup(1), "failed to setup test disk")
 	defer func() { require.NoError(teardown(), "failed to delete test disk") }()
 
-	mapper, err := mapper.New(devicePath)
+	mapper, err := mapper.New(devicePath, logger.NewTest(t))
 	require.NoError(err, "failed to initialize crypt device")
 	defer func() { require.NoError(mapper.Close(), "failed to close crypt device") }()
 

state/mapper/mapper.go

@@ -1,85 +0,0 @@
-package mapper
-
-import (
-	"errors"
-	"fmt"
-	"strings"
-
-	cryptsetup "github.com/martinjungblut/go-cryptsetup"
-)
-
-// Mapper handles actions for formating and mapping crypt devices.
-type Mapper struct {
-	device cryptDevice
-}
-
-// New creates a new crypt device for the device at path.
-func New(path string) (*Mapper, error) {
-	device, err := cryptsetup.Init(path)
-	if err != nil {
-		return nil, fmt.Errorf("initializing crypt device for disk %q: %w", path, err)
-	}
-	return &Mapper{device: device}, nil
-}
-
-// Close closes and frees memory allocated for the crypt device.
-func (m *Mapper) Close() error {
-	if m.device.Free() {
-		return nil
-	}
-	return errors.New("unable to close crypt device")
-}
-
-// IsLUKSDevice returns true if the device is formatted as a LUKS device.
-func (m *Mapper) IsLUKSDevice() bool {
-	return m.device.Load(cryptsetup.LUKS2{}) == nil
-}
-
-// DiskUUID gets the device's UUID.
-func (m *Mapper) DiskUUID() string {
-	return strings.ToLower(m.device.GetUUID())
-}
-
-// FormatDisk formats the disk and adds passphrase in keyslot 0.
-func (m *Mapper) FormatDisk(passphrase string) error {
-	luksParams := cryptsetup.LUKS2{
-		SectorSize: 4096,
-		PBKDFType: &cryptsetup.PbkdfType{
-			// Use low memory recommendation from https://datatracker.ietf.org/doc/html/rfc9106#section-7
-			Type:            "argon2id",
-			TimeMs:          2000,
-			Iterations:      3,
-			ParallelThreads: 4,
-			MaxMemoryKb:     65536, // ~64MiB
-		},
-	}
-
-	genericParams := cryptsetup.GenericParams{
-		Cipher:        "aes",
-		CipherMode:    "xts-plain64",
-		VolumeKeySize: 64,
-	}
-
-	if err := m.device.Format(luksParams, genericParams); err != nil {
-		return fmt.Errorf("formating disk: %w", err)
-	}
-
-	if err := m.device.KeyslotAddByVolumeKey(0, "", passphrase); err != nil {
-		return fmt.Errorf("adding keyslot: %w", err)
-	}
-
-	return nil
-}
-
-// MapDisk maps a crypt device to /dev/mapper/target using the provided passphrase.
-func (m *Mapper) MapDisk(target, passphrase string) error {
-	if err := m.device.ActivateByPassphrase(target, 0, passphrase, 0); err != nil {
-		return fmt.Errorf("mapping disk as %q: %w", target, err)
-	}
-	return nil
-}
-
-// UnmapDisk removes the mapping of target.
-func (m *Mapper) UnmapDisk(target string) error {
-	return m.device.Deactivate(target)
-}