disk-mapper: set LUKS2 token to allow reusing unintialized state disks (#2083)

Signed-off-by: Daniel Weiße <dw@edgeless.systems>

bazel/toolchains/go_module_deps.bzl

@@ -3569,8 +3569,9 @@ def go_dependencies():
         patches = [
             "//3rdparty/bazel/com_github_martinjungblut_go_cryptsetup:com_github_martinjungblut_go_cryptsetup.patch",  # keep
         ],
-        sum = "h1:YDjLk3wsL5ZLhLC4TIwIvT2NkSCAdAV6pzzZaRfj4jk=",
+        replace = "github.com/daniel-weisse/go-cryptsetup",
-        version = "v0.0.0-20220520180014-fd0874fd07a6",
+        sum = "h1:ToajP6trZoiqlZ3Z4uoG1P02/wtqSw1AcowOXOYjATk=",
+        version = "v0.0.0-20230705150314-d8c07bd1723c",
     )
     go_repository(
         name = "com_github_masterminds_goutils",

bootstrapper/internal/diskencryption/diskencryption.go

@@ -52,7 +52,12 @@ func (c *DiskEncryption) UpdatePassphrase(passphrase string) error {
 	if err != nil {
 		return err
 	}
-	return c.device.KeyslotChangeByPassphrase(keyslot, keyslot, initialPassphrase, passphrase)
+	if err := c.device.KeyslotChangeByPassphrase(keyslot, keyslot, initialPassphrase, passphrase); err != nil {
+		return err
+	}
+
+	// Set token as initialized.
+	return c.device.SetConstellationStateDiskToken(cryptsetup.SetDiskInitialized)
 }
 
 // getInitialPassphrase retrieves the initial passphrase used on first boot.
@@ -68,4 +73,5 @@ type cryptdevice interface {
 	InitByName(name string) (func(), error)
 	GetUUID() (string, error)
 	KeyslotChangeByPassphrase(currentKeyslot int, newKeyslot int, currentPassphrase string, newPassphrase string) error
+	SetConstellationStateDiskToken(bool) error
 }

bootstrapper/internal/diskencryption/diskencryption_test.go

@@ -90,6 +90,6 @@ func (s *stubCryptdevice) KeyslotChangeByPassphrase(_, _ int, _, _ string) error
 	return s.keyslotChangeErr
 }
 
-func (s *stubCryptdevice) Close() error {
+func (s *stubCryptdevice) SetConstellationStateDiskToken(bool) error {
 	return nil
 }

disk-mapper/cmd/main.go

@@ -147,7 +147,7 @@ func main() {
 	}
 
 	// prepare the state disk
-	if mapper.IsLUKSDevice() {
+	if mapper.IsInitialized() {
 		// set up rejoin client
 		var self metadata.InstanceMetadata
 		self, err = metadataClient.Self(context.Background())

disk-mapper/internal/diskencryption/diskencryption.go

@@ -24,23 +24,30 @@ import (
 
 // DiskEncryption handles actions for formatting and mapping crypt devices.
 type DiskEncryption struct {
-	device cryptDevice
+	device     cryptDevice
-	log    *logger.Logger
+	devicePath string
+	log        *logger.Logger
 }
 
 // New creates a new crypt device for the device at path.
 func New(path string, log *logger.Logger) (*DiskEncryption, func(), error) {
 	device := cryptsetup.New()
-	free, err := device.Init(path)
+	_, err := device.Init(path)
 	if err != nil {
 		return nil, nil, fmt.Errorf("initializing crypt device for disk %q: %w", path, err)
 	}
-	return &DiskEncryption{device: device, log: log}, free, nil
+	d := &DiskEncryption{device: device, devicePath: path, log: log}
+	return d, d.free, nil
 }
 
-// IsLUKSDevice returns true if the device is formatted as a LUKS device.
+// IsInitialized returns true if the device is formatted as a LUKS device,
-func (d *DiskEncryption) IsLUKSDevice() bool {
+// and has been successfully initialized before (successfully joined a cluster).
-	return d.device.LoadLUKS2() == nil
+func (d *DiskEncryption) IsInitialized() bool {
+	if err := d.device.LoadLUKS2(); err != nil {
+		return false
+	}
+
+	return d.device.ConstellationStateDiskTokenIsInitialized()
 }
 
 // DiskUUID gets the device's UUID.
@@ -50,6 +57,14 @@ func (d *DiskEncryption) DiskUUID() (string, error) {
 
 // FormatDisk formats the disk and adds passphrase in keyslot 0.
 func (d *DiskEncryption) FormatDisk(passphrase string) error {
+	// Successfully calling LoadLUKS2() before FormatDisk() will cause format to fail.
+	// To make sure format is idempotent, we need to run it on a freshly initialized device.
+	// Therefore we free the device and reinitialize it.
+	d.free()
+	if _, err := d.device.Init(d.devicePath); err != nil {
+		return fmt.Errorf("re-initializing crypt device for disk %q: %w", d.devicePath, err)
+	}
+
 	if err := d.device.Format(cryptsetup.FormatIntegrity); err != nil {
 		return fmt.Errorf("formatting disk: %w", err)
 	}
@@ -63,6 +78,9 @@ func (d *DiskEncryption) FormatDisk(passphrase string) error {
 		return fmt.Errorf("wiping disk: %w", err)
 	}
 
+	if err := d.device.SetConstellationStateDiskToken(cryptsetup.SetDiskNotInitialized); err != nil {
+		return fmt.Errorf("setting disk token: %w", err)
+	}
 	return nil
 }
 
@@ -96,13 +114,21 @@ func (d *DiskEncryption) Wipe(blockWipeSize int) error {
 	return nil
 }
 
+func (d *DiskEncryption) free() {
+	d.device.Free()
+}
+
 type cryptDevice interface {
 	ActivateByPassphrase(deviceName string, keyslot int, passphrase string, flags int) error
 	ActivateByVolumeKey(deviceName string, volumeKey string, volumeKeySize int, flags int) error
 	Deactivate(deviceName string) error
 	Format(integrity bool) error
+	Free()
 	GetUUID() (string, error)
+	Init(path string) (func(), error)
 	LoadLUKS2() error
 	KeyslotAddByVolumeKey(keyslot int, volumeKey string, passphrase string) error
+	SetConstellationStateDiskToken(diskIsInitialized bool) error
+	ConstellationStateDiskTokenIsInitialized() bool
 	Wipe(name string, wipeBlockSize int, flags int, logCallback func(size, offset uint64), logFrequency time.Duration) error
 }

disk-mapper/internal/setup/setup_test.go

@@ -413,6 +413,10 @@ func (s *stubMapper) UnmapDisk(string) error {
 	return s.unmapDiskErr
 }
 
+func (s *stubMapper) SetDiskToInitialized() error {
+	return nil
+}
+
 type stubMounter struct {
 	mountCalled   bool
 	mountErr      error

disk-mapper/internal/test/integration_test.go

@@ -9,6 +9,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 package integration
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
@@ -16,8 +17,9 @@ import (
 	"testing"
 
 	"github.com/edgelesssys/constellation/v2/disk-mapper/internal/diskencryption"
+	ccryptsetup "github.com/edgelesssys/constellation/v2/internal/cryptsetup"
 	"github.com/edgelesssys/constellation/v2/internal/logger"
-	"github.com/martinjungblut/go-cryptsetup"
+	cryptsetup "github.com/martinjungblut/go-cryptsetup"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.uber.org/goleak"
@@ -68,7 +70,7 @@ func TestMapper(t *testing.T) {
 	require.NoError(err, "failed to initialize crypt device")
 	defer free()
 
-	assert.False(mapper.IsLUKSDevice())
+	assert.False(mapper.IsInitialized())
 
 	// Format and map disk
 	passphrase := "unit-test"
@@ -76,23 +78,33 @@ func TestMapper(t *testing.T) {
 	require.NoError(mapper.MapDisk(mappedDevice, passphrase), "failed to map disk")
 	require.NoError(mapper.UnmapDisk(mappedDevice), "failed to remove disk mapping")
 
-	assert.True(mapper.IsLUKSDevice())
+	// Make sure token was set
+	ccrypt := ccryptsetup.New()
+	freeDevice, err := ccrypt.Init(devicePath)
+	require.NoError(err, "failed to initialize crypt device")
+	defer freeDevice()
+	require.NoError(ccrypt.LoadLUKS2(), "failed to load LUKS2")
+
+	tokenJSON, err := ccrypt.TokenJSONGet(ccryptsetup.ConstellationStateDiskTokenID)
+	require.NoError(err, "token should have been set")
+	var token struct {
+		Type              string   `json:"type"`
+		Keyslots          []string `json:"keyslots"`
+		DiskIsInitialized bool     `json:"diskIsInitialized"`
+	}
+	require.NoError(json.Unmarshal([]byte(tokenJSON), &token))
+	assert.False(token.DiskIsInitialized, "disk should be marked as not initialized")
+	assert.False(ccrypt.ConstellationStateDiskTokenIsInitialized(), "disk should be marked as not initialized")
+
+	// Disk should still be marked as not initialized because token is set to false.
+	assert.False(mapper.IsInitialized())
 
 	// Try to map disk with incorrect passphrase
 	assert.Error(mapper.MapDisk(mappedDevice, "invalid-passphrase"), "was able to map disk with incorrect passphrase")
-}
 
-/*
+	// Disk can be reformatted without manually re-initializing a mapper
-type fakeMetadataAPI struct{}
+	passphrase2 := passphrase + "2"
-
+	require.NoError(mapper.FormatDisk(passphrase2), "failed to format disk")
-func (f *fakeMetadataAPI) List(ctx context.Context) ([]metadata.InstanceMetadata, error) {
+	require.NoError(mapper.MapDisk(mappedDevice, passphrase2), "failed to map disk")
-	return []metadata.InstanceMetadata{
+	require.NoError(mapper.UnmapDisk(mappedDevice), "failed to remove disk mapping")
-		{
-			Name:       "instanceName",
-			ProviderID: "fake://instance-id",
-			Role:       role.Unknown,
-			VPCIP:      "192.0.2.1",
-		},
-	}, nil
 }
-*/

go.mod

@@ -34,6 +34,7 @@ replace (
 replace (
 	github.com/edgelesssys/constellation/v2/operators/constellation-node-operator/v2/api => ./operators/constellation-node-operator/api
 	github.com/google/go-tpm => github.com/thomasten/go-tpm v0.0.0-20230629092004-f43f8e2a59eb
+	github.com/martinjungblut/go-cryptsetup => github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c
 )
 
 require (

go.sum

@@ -279,6 +279,8 @@ github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7 h
 github.com/cyberphone/json-canonicalization v0.0.0-20220623050100-57a0ce2678a7/go.mod h1:uzvlm1mxhHkdfqitSA92i7Se+S9ksOn3a3qmv/kyOCw=
 github.com/cyphar/filepath-securejoin v0.2.3 h1:YX6ebbZCZP7VkM3scTTokDgBL2TY741X51MTk3ycuNI=
 github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c h1:ToajP6trZoiqlZ3Z4uoG1P02/wtqSw1AcowOXOYjATk=
+github.com/daniel-weisse/go-cryptsetup v0.0.0-20230705150314-d8c07bd1723c/go.mod h1:gZoZ0+POlM1ge/VUxWpMmZVNPzzMJ7l436CgkQ5+qzU=
 github.com/danieljoos/wincred v1.1.2 h1:QLdCxFs1/Yl4zduvBdcHB8goaYk9RARS2SgLLRuAyr0=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -756,8 +758,6 @@ github.com/markbates/oncer v1.0.0 h1:E83IaVAHygyndzPimgUYJjbshhDTALZyXxvk9FOlQRY
 github.com/markbates/oncer v1.0.0/go.mod h1:Z59JA581E9GP6w96jai+TGqafHPW+cPfRxz2aSZ0mcI=
 github.com/markbates/safe v1.0.1 h1:yjZkbvRM6IzKj9tlu/zMJLS0n/V351OZWRnF3QfaUxI=
 github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0=
-github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6 h1:YDjLk3wsL5ZLhLC4TIwIvT2NkSCAdAV6pzzZaRfj4jk=
-github.com/martinjungblut/go-cryptsetup v0.0.0-20220520180014-fd0874fd07a6/go.mod h1:gZoZ0+POlM1ge/VUxWpMmZVNPzzMJ7l436CgkQ5+qzU=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=

internal/cryptsetup/cryptsetup.go

@@ -14,6 +14,7 @@ There should only be one instance using this package per process.
 package cryptsetup
 
 import (
+	"encoding/json"
 	"errors"
 	"fmt"
 	"strings"
@@ -22,6 +23,13 @@ import (
 )
 
 const (
+	// ConstellationStateDiskTokenID is the ID of Constellation's state disk token.
+	ConstellationStateDiskTokenID = 0
+	// SetDiskInitialized is a flag to set the Constellation state disk token to initialized.
+	SetDiskInitialized = true
+	// SetDiskNotInitialized is a flag to set the Constellation state disk token to not initialized.
+	SetDiskNotInitialized = false
+
 	// FormatIntegrity is a flag to enable dm-integrity for a crypt device when formatting.
 	FormatIntegrity = true
 	// FormatNoIntegrity is a flag to disable dm-integrity for a crypt device when formatting.
@@ -220,6 +228,70 @@ func (c *CryptSetup) Resize(name string, newSize uint64) error {
 	return nil
 }
 
+// TokenJSONGet gets the JSON data for a token.
+func (c *CryptSetup) TokenJSONGet(token int) (string, error) {
+	packageLock.Lock()
+	defer packageLock.Unlock()
+	if c.device == nil {
+		return "", errDeviceNotOpen
+	}
+	json, err := c.device.TokenJSONGet(token)
+	if err != nil {
+		return "", fmt.Errorf("getting JSON data for token %d: %w", token, err)
+	}
+	return json, nil
+}
+
+// TokenJSONSet sets the JSON data for a token.
+// The JSON data must be a valid LUKS2 token.
+// Required fields are:
+//   - type [string] the token type (tokens with luks2- prefix are reserved)
+//   - keyslots [array] the array of keyslot objects names that are assigned to the token
+//
+// Returns the allocated token ID on success.
+func (c *CryptSetup) TokenJSONSet(token int, json string) (int, error) {
+	packageLock.Lock()
+	defer packageLock.Unlock()
+	if c.device == nil {
+		return -1, errDeviceNotOpen
+	}
+	tokenID, err := c.device.TokenJSONSet(token, json)
+	if err != nil {
+		return -1, fmt.Errorf("setting JSON data for token %d: %w", token, err)
+	}
+	return tokenID, nil
+}
+
+// SetConstellationStateDiskToken sets the Constellation state disk token.
+func (c *CryptSetup) SetConstellationStateDiskToken(diskIsInitialized bool) error {
+	token := constellationLUKS2Token{
+		Type:              "constellation-state-disk",
+		Keyslots:          []string{},
+		DiskIsInitialized: diskIsInitialized,
+	}
+	json, err := json.Marshal(token)
+	if err != nil {
+		return fmt.Errorf("marshaling token: %w", err)
+	}
+	if _, err := c.device.TokenJSONSet(ConstellationStateDiskTokenID, string(json)); err != nil {
+		return fmt.Errorf("setting token: %w", err)
+	}
+	return nil
+}
+
+// ConstellationStateDiskTokenIsInitialized returns true if the Constellation state disk token is set to initialized.
+func (c *CryptSetup) ConstellationStateDiskTokenIsInitialized() bool {
+	stateDiskToken, err := c.device.TokenJSONGet(ConstellationStateDiskTokenID)
+	if err != nil {
+		return false
+	}
+	var token constellationLUKS2Token
+	if err := json.Unmarshal([]byte(stateDiskToken), &token); err != nil {
+		return false
+	}
+	return token.DiskIsInitialized
+}
+
 // Wipe overwrites the device with zeros to initialize integrity checksums.
 func (c *CryptSetup) Wipe(
 	name string, blockWipeSize int, flags int, logCallback func(size, offset uint64), logFrequency time.Duration,
@@ -264,6 +336,12 @@ func (c *CryptSetup) Wipe(
 	return nil
 }
 
+type constellationLUKS2Token struct {
+	Type              string   `json:"type"`
+	Keyslots          []string `json:"keyslots"`
+	DiskIsInitialized bool     `json:"diskIsInitialized"`
+}
+
 type cryptDevice interface {
 	ActivateByPassphrase(deviceName string, keyslot int, passphrase string, flags int) error
 	ActivateByVolumeKey(deviceName, volumeKey string, volumeKeySize, flags int) error
@@ -274,5 +352,7 @@ type cryptDevice interface {
 	KeyslotAddByVolumeKey(keyslot int, volumeKey string, passphrase string) error
 	KeyslotChangeByPassphrase(currentKeyslot, newKeyslot int, currentPassphrase, newPassphrase string) error
 	Resize(name string, newSize uint64) error
+	TokenJSONGet(token int) (string, error)
+	TokenJSONSet(token int, json string) (int, error)
 	Wipe(devicePath string, pattern int, offset, length uint64, wipeBlockSize int, flags int, progress func(size, offset uint64) int) error
 }