Create mkosi image build pipeline

2024-12-24 15:09:39 -05:00 · 2022-10-19 13:10:15 +02:00 · 2022-10-19 13:10:15 +02:00 · 34367ea3cc
commit 34367ea3cc
parent e5aaf0a42f
107 changed files with 2733 additions and 105 deletions
--- a/.github/actions/setup_mkosi/action.yaml
+++ b/.github/actions/setup_mkosi/action.yaml
@ -0,0 +1,42 @@
+name: Setup mkosi
+description: Install mkosi and all its dependencies
+inputs:
+  version:
+    description: "Version (commit hash) of mkosi to install."
+    required: true
+runs:
+  using: composite
+  steps:
+    - name: Dependencies
+      shell: bash
+      run: |
+        echo "::group::Dependencies"
+        echo "deb-src http://archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
+        sudo add-apt-repository ppa:michel-slm/kernel-utils
+        sudo apt-get update
+        sudo apt-get install --assume-yes --no-install-recommends \
+          dnf \
+          makepkg \
+          systemd-container \
+          qemu-system-x86 \
+          ovmf \
+          e2fsprogs \
+          squashfs-tools
+        echo "::endgroup::"
+    # Try to eliminate "Failed to dissect image: Connection timed out" errors from nspawn by compiling
+    # systemd-nspawn from v251 from source.
+    - name: Update systemd-nspawn
+      shell: bash
+      working-directory: ${{ github.action_path }}
+      run: |
+        echo "::group::Update systemd-nspawn"
+        sudo apt-get build-dep systemd
+        git clone https://github.com/systemd/systemd-stable --branch v251.2 --depth=1
+        meson systemd-stable/build systemd-stable
+        ninja -C systemd-stable/build systemd-nspawn
+        sudo ln -svf $PWD/systemd-stable/build/systemd-nspawn $(which systemd-nspawn)
+        systemd-nspawn --version
+        echo "::endgroup::"
+    - name: Install
+      shell: bash
+      run: sudo python3 -m pip install git+https://github.com/systemd/mkosi.git@${{ inputs.version }}
--- a/.github/workflows/build-os-image.yml
+++ b/.github/workflows/build-os-image.yml
@ -0,0 +1,327 @@
+name: Build and Upload OS image
+on:
+  workflow_dispatch:
+    inputs:
+      imageVersion:
+        description: "Semantic version including patch e.g. v<major>.<minor>.<patch> (only used for releases)"
+        required: false
+      debug:
+        description: "Build debug image"
+        type: boolean
+        default: false
+        required: false
+
+jobs:
+  build-dependencies:
+    name: "Build binaries for embedding in the OS"
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791
+
+      - name: Build bootstrapper
+        if: ${{ inputs.debug == false }}
+        uses: ./.github/actions/build_bootstrapper
+        with:
+          outputPath: ${{ github.workspace }}/build/bootstrapper
+
+      - name: Build debugd
+        if: ${{ inputs.debug == true }}
+        uses: ./.github/actions/build_debugd
+        with:
+          outputPath: ${{ github.workspace }}/build/bootstrapper
+
+      - name: Build disk-mapper
+        uses: ./.github/actions/build_disk_mapper
+        with:
+          outputPath: ${{ github.workspace }}/build/disk-mapper
+
+      - name: Upload dependencies
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+        with:
+          name: dependencies
+          path: |
+            ${{ github.workspace }}/build/bootstrapper
+            ${{ github.workspace }}/build/disk-mapper
+
+      - name: Collect hashes
+        run: |
+          echo "::set-output name=bootstrapper-sha256::$(sha256sum bootstrapper)"
+          echo "::set-output name=disk-mapper-sha256::$(sha256sum disk-mapper)"
+        working-directory: ${{ github.workspace }}/build
+
+  make-os-image:
+    name: "Build OS using mkosi"
+    needs: build-dependencies
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        csp: [azure, gcp, qemu]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791
+
+      - name: Download build dependencies
+        uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
+        with:
+          name: dependencies
+          path: ${{ github.workspace }}/build
+
+      - name: Mark bootstrapper and disk-mapper as executable
+        run: |
+          chmod +x ${{ github.workspace }}/build/bootstrapper
+          chmod +x ${{ github.workspace }}/build/disk-mapper
+
+      - name: Setup mkosi
+        uses: ./.github/actions/setup_mkosi
+        with:
+          version: 1750fc261a071f39c54f8b64ce2424195f750ee0
+
+      - name: Prepare PKI for secure boot signing
+        shell: bash
+        run: |
+          ln -s pki_testing pki
+          echo "${DB_KEY}" > pki/db.key
+        working-directory: ${{ github.workspace }}/image/mkosi
+        env:
+          DB_KEY: ${{ secrets.SECURE_BOOT_TESTING_DB_KEY }}
+
+      - name: Build
+        shell: bash
+        run: |
+          echo "::group::Build"
+          sudo make "${CSP}"
+          echo "::endgroup::"
+        working-directory: ${{ github.workspace }}/image/mkosi
+        env:
+          BOOTSTRAPPER_BINARY: ${{ github.workspace }}/build/bootstrapper
+          DISK_MAPPER_BINARY: ${{ github.workspace }}/build/disk-mapper
+          CSP: ${{ matrix.csp }}
+
+      - name: Upload raw OS image as artifact
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+        with:
+          name: image-${{ matrix.csp }}
+          path: ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.raw
+        if: always()
+        continue-on-error: true
+
+      - name: Upload individual OS parts as artifacts
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+        with:
+          name: parts-${{ matrix.csp }}
+          path: |
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.cmdline
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.efi
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.initrd
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.root.raw
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.root.roothash
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.root.verity
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.vmlinuz
+        if: always()
+        continue-on-error: true
+
+      - name: Upload manifest as artifact
+        uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
+        with:
+          name: manifest-${{ matrix.csp }}
+          path: |
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.raw.changelog
+            ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.raw.manifest
+        if: always()
+        continue-on-error: true
+
+  upload-os-image:
+    name: "Upload OS image to CSP"
+    needs: make-os-image
+    runs-on: ubuntu-22.04
+    strategy:
+      matrix:
+        csp: [azure, gcp]
+        upload-variant: [""]
+        include:
+          - csp: azure
+            upload-variant: TrustedLaunch
+    steps:
+      - name: Checkout
+        uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791
+
+      - name: Download OS image artifact
+        uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
+        with:
+          name: image-${{ matrix.csp }}
+          path: ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36
+
+      - name: Install tools
+        shell: bash
+        run: |
+          echo "::group::Install tools"
+          sudo apt-get update
+          sudo apt-get install -y pigz qemu-utils
+          echo "::endgroup::"
+
+      - name: Login to Azure
+        uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2
+        if: ${{ matrix.csp == 'azure' }}
+        with:
+          creds: ${{ secrets.AZURE_CREDENTIALS }}
+
+      - name: Login to GCP
+        uses: ./.github/actions/gcp_login
+        if: ${{ matrix.csp == 'gcp' }}
+        with:
+          gcp_service_account_json: ${{ secrets.GCP_IMAGE_UPLOAD_SERVICE_ACCOUNT }}
+
+      - name: Prepare PKI for image upload
+        shell: bash
+        run: ln -s pki_testing pki
+        working-directory: ${{ github.workspace }}/image/mkosi
+
+      - name: Determine version
+        id: version
+        uses: ./.github/actions/pseudo_version
+
+      # Make sure to set valid names for GCP and Azure
+      #  Azure
+      #    gallery name may include alphanumeric characters, dots and underscores. Must end and begin with an alphanumeric character
+      #    image definition may include alphanumeric characters, dots, dashes and underscores. Must end and begin with an alphanumeric character
+      #    image version has to be semantic version in the form <uint>.<uint>.<uint> . uint may not be larger than 2,147,483,647
+      #
+      #  GCP
+      #    image family and image name may include lowercase alphanumeric characters and dashes. Must not end or begin with a dash
+      - name: Configure input variables
+        shell: bash
+        run: |
+          timestamp=${{ steps.version.outputs.timestamp }}
+          semver=${{ steps.version.outputs.semanticVersion }}
+          imageVersion=${{ inputs.imageVersion }}
+          pseudover=${{ steps.version.outputs.pseudoVersion }}
+          echo "PKI=${{ github.workspace }}/image/mkosi/pki" >> $GITHUB_ENV
+          echo "GCP_PROJECT=constellation-images" >> $GITHUB_ENV
+          echo "GCP_BUCKET=constellation-images" >> $GITHUB_ENV
+          echo "GCP_REGION=europe-west3" >> $GITHUB_ENV
+          echo "GCP_RAW_IMAGE_PATH=${{ github.workspace }}/image/mkosi/mkosi.output.gcp/fedora~36/image.raw" >> $GITHUB_ENV
+          echo "GCP_IMAGE_PATH=${{ github.workspace }}/image/mkosi/mkosi.output.gcp/fedora~36/image.tar.gz" >> $GITHUB_ENV
+          echo "AZURE_RESOURCE_GROUP_NAME=constellation-images" >> $GITHUB_ENV
+          echo "AZURE_REGION=northeurope" >> $GITHUB_ENV
+          echo "AZURE_REPLICATION_REGIONS=northeurope eastus westeurope westus" >> $GITHUB_ENV
+          echo "AZURE_SKU=constellation" >> $GITHUB_ENV
+          echo "AZURE_PUBLISHER=edgelesssys" >> $GITHUB_ENV
+          echo "AZURE_RAW_IMAGE_PATH=${{ github.workspace }}/image/mkosi/mkosi.output.azure/fedora~36/image.raw" >> $GITHUB_ENV
+          echo "AZURE_IMAGE_PATH=${{ github.workspace }}/image/mkosi/mkosi.output.azure/fedora~36/image.vhd" >> $GITHUB_ENV
+          # TODO: set default security type to "ConfidentialVM" once replication is possible
+          AZURE_SECURITY_TYPE=${{ matrix.upload-variant }}
+          if [ -z "${AZURE_SECURITY_TYPE}" ]; then
+            AZURE_SECURITY_TYPE=ConfidentialVMSupported
+          fi
+          echo "AZURE_SECURITY_TYPE=${AZURE_SECURITY_TYPE}" >> $GITHUB_ENV
+          echo "AZURE_DISK_NAME=constellation-${pseudover//./-}-${AZURE_SECURITY_TYPE,,}" >> $GITHUB_ENV
+          if [ "${{ startsWith(github.ref, 'refs/heads/release/') && (inputs.debug == false) }}" = true ]
+          then
+            GCP_IMAGE_NAME=constellation-${imageVersion//./-}
+            echo "GCP_IMAGE_FAMILY=constellation" >> $GITHUB_ENV
+            AZURE_IMAGE_DEFINITION=constellation
+            echo "AZURE_IMAGE_VERSION=${imageVersion:1}" >> $GITHUB_ENV
+            AZURE_GALLERY_NAME=Constellation
+          elif [ "${{ ((github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/heads/release/')) && (inputs.debug == true) }}" = true ]
+          then
+            GCP_IMAGE_NAME=constellation-${{ steps.version.outputs.timestamp }}
+            echo "GCP_IMAGE_FAMILY=constellation-debug-${semver//./-}" >> $GITHUB_ENV
+            AZURE_IMAGE_DEFINITION=${semver}
+            echo "AZURE_IMAGE_VERSION=${timestamp:0:4}.${timestamp:4:4}.${timestamp:8}" >> $GITHUB_ENV
+            AZURE_GALLERY_NAME=Constellation_Debug
+          else
+            GCP_IMAGE_NAME=constellation-${{ steps.version.outputs.timestamp }}
+            echo "GCP_IMAGE_FAMILY=constellation-${{ steps.version.outputs.branchName }}" >> $GITHUB_ENV
+            AZURE_IMAGE_DEFINITION=${{ steps.version.outputs.branchName }}
+            echo "AZURE_IMAGE_VERSION=${timestamp:0:4}.${timestamp:4:4}.${timestamp:8}" >> $GITHUB_ENV
+            AZURE_GALLERY_NAME=Constellation_Testing
+          fi
+          # TODO: enable VMGS upload for ConfidentialVM images once replication is possible
+          if [ "${AZURE_SECURITY_TYPE}" == "ConfidentialVMSupported" ]; then
+            echo "AZURE_GALLERY_NAME=${AZURE_GALLERY_NAME}_CVM" >> $GITHUB_ENV
+            echo "AZURE_VMGS_PATH=" >> $GITHUB_ENV
+          else
+            echo "AZURE_GALLERY_NAME=${AZURE_GALLERY_NAME}" >> $GITHUB_ENV
+            echo "AZURE_VMGS_PATH=${{ github.workspace }}/image/mkosi/pki/${AZURE_SECURITY_TYPE}.vmgs" >> $GITHUB_ENV
+          fi
+          echo "AZURE_IMAGE_DEFINITION=${AZURE_IMAGE_DEFINITION}" >> $GITHUB_ENV
+          echo "AZURE_IMAGE_OFFER=${AZURE_IMAGE_DEFINITION}" >> $GITHUB_ENV
+          echo "GCP_IMAGE_NAME=${GCP_IMAGE_NAME}" >> $GITHUB_ENV
+          echo "GCP_IMAGE_FILENAME=${GCP_IMAGE_NAME}.tar.gz" >> $GITHUB_ENV
+
+      - name: Download VMGS blob
+        run: aws s3 cp \
+          s3://constellation-secure-boot/pki_testing/${AZURE_SECURITY_TYPE}.vmgs \
+          pki_testing/${AZURE_SECURITY_TYPE}.vmgs \
+          --no-progress
+        working-directory: ${{ github.workspace }}/image/mkosi
+        if: ${{ matrix.csp == 'azure' }}
+
+      - name: Upload GCP image
+        shell: bash
+        run: |
+          echo "::group::Upload GCP image"
+          upload/pack.sh gcp "${GCP_RAW_IMAGE_PATH}" "${GCP_IMAGE_PATH}"
+          upload/upload_gcp.sh
+          echo -e "Uploaded GCP image: \`projects/${GCP_PROJECT}/global/images/${GCP_IMAGE_NAME}\`" >> $GITHUB_STEP_SUMMARY
+          echo "::endgroup::"
+        working-directory: ${{ github.workspace }}/image/mkosi
+        if: ${{ matrix.csp == 'gcp' }}
+
+      - name: Upload Azure image
+        shell: bash
+        run: |
+          echo "::group::Upload Azure image"
+          upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
+          upload/upload_azure.sh -g --disk-name "${AZURE_DISK_NAME}" "${AZURE_VMGS_PATH}"
+          echo -e "Uploaded Azure ${AZURE_SECURITY_TYPE} image: \`/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/${AZURE_RESOURCE_GROUP_NAME^^}/providers/Microsoft.Compute/galleries/${AZURE_GALLERY_NAME}/images/${AZURE_IMAGE_DEFINITION}/versions/${AZURE_IMAGE_VERSION}\`" >> $GITHUB_STEP_SUMMARY
+          echo "::endgroup::"
+        working-directory: ${{ github.workspace }}/image/mkosi
+        if: ${{ matrix.csp == 'azure' }}
+
+  generate-sbom:
+    name: "Generate SBOM"
+    needs: make-os-image
+    runs-on: ubuntu-22.04
+    steps:
+      - name: Install squashfs tools
+        run: |
+          echo "::group::Install squashfs tools"
+          sudo apt-get update
+          sudo apt-get install -y squashfs-tools
+          echo "::endgroup::"
+
+      - name: Download rootfs
+        uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
+        with:
+          # downloading / using only the QEMU rootfs is fine
+          # since the images only differ in the ESP partition
+          name: parts-qemu
+
+      - name: Unpack squashfs
+        run: |
+          echo "::group::Unpack squashfs"
+          unsquashfs -user-xattrs -d image.root.tree image.root.raw
+          echo "::endgroup::"
+
+      - uses: anchore/sbom-action@eda59434a8e5ce9bda93a202dfe50f6a2e637bb6
+        with:
+          path: image.root.tree
+          artifact-name: sbom.spdx.json
+          format: spdx-json
+
+      - uses: anchore/sbom-action@eda59434a8e5ce9bda93a202dfe50f6a2e637bb6
+        with:
+          path: image.root.tree
+          artifact-name: sbom.cyclonedx.json
+          format: cyclonedx-json
+
+      - uses: anchore/sbom-action@eda59434a8e5ce9bda93a202dfe50f6a2e637bb6
+        with:
+          path: image.root.tree
+          artifact-name: sbom.syft.json
+          format: syft-json
--- a/bootstrapper/internal/kubernetes/k8sapi/constants.go
+++ b/bootstrapper/internal/kubernetes/k8sapi/constants.go
@ -12,9 +12,10 @@ const (
 	binDir                  = "/run/state/bin"
 	kubeadmPath             = "/run/state/bin/kubeadm"
 	kubeletPath             = "/run/state/bin/kubelet"
-	kubeletServiceEtcPath   = "/etc/systemd/system/kubelet.service"
+	kubectlPath             = "/run/state/bin/kubectl"
+	kubeletServiceEtcPath   = "/run/systemd/system/kubelet.service"
 	kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
-	kubeadmConfEtcPath      = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
+	kubeadmConfEtcPath      = "/run/systemd/system/kubelet.service.d/10-kubeadm.conf"
 	kubeadmConfStatePath    = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf"
 	executablePerm          = 0o544
 	systemdUnitPerm         = 0o644
--- a/cli/internal/cloudcmd/create.go
+++ b/cli/internal/cloudcmd/create.go
@ -128,6 +128,7 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, config *c
 		StateDiskType:        config.Provider.Azure.StateDiskType,
 		ImageID:              config.Provider.Azure.Image,
 		ConfidentialVM:       *config.Provider.Azure.ConfidentialVM,
+		SecureBoot:           *config.Provider.Azure.SecureBoot,
 		Debug:                config.IsDebugCluster(),
 	}

@ -210,6 +211,8 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
 		MemorySizeMiB:      config.Provider.QEMU.Memory,
 		MetadataAPIImage:   config.Provider.QEMU.MetadataAPIImage,
 		MetadataLibvirtURI: metadataLibvirtURI,
+		NVRAM:              config.Provider.QEMU.NVRAM,
+		Firmware:           config.Provider.QEMU.Firmware,
 	}

 	ip, err := cl.CreateCluster(ctx, name, &vars)
--- a/cli/internal/libvirt/Dockerfile
+++ b/cli/internal/libvirt/Dockerfile
@ -17,6 +17,11 @@ RUN echo "cgroup_controllers = []" >> /etc/libvirt/qemu.conf && \
    echo "listen_addr = \"localhost\"" >> /etc/libvirt/libvirtd.conf && \
    echo "auth_tcp = \"none\"" >> /etc/libvirt/libvirtd.conf

+# Copy nvram templates
+COPY ./cli/internal/libvirt/nvram/constellation_vars.testing.fd /usr/share/OVMF/constellation_vars.testing.fd
+# TODO: Uncomment this line when we have a production template
+# COPY ./cli/internal/libvirt/nvram/constellation_vars.production.fd /usr/share/OVMF/constellation_vars.production.fd
+
 COPY --chmod=755 ./cli/internal/libvirt/start.sh /start.sh

 ENTRYPOINT ["/start.sh"]
--- a/cli/internal/libvirt/nvram/constellation_vars.testing.fd
+++ b/cli/internal/libvirt/nvram/constellation_vars.testing.fd
--- a/cli/internal/terraform/terraform/azure/main.tf
+++ b/cli/internal/terraform/terraform/azure/main.tf
@ -193,6 +193,7 @@ module "scale_set_control_plane" {
  location                  = var.location
  instance_type             = var.instance_type
  confidential_vm           = var.confidential_vm
+  secure_boot               = var.secure_boot
  tags                      = merge(local.tags, { role = "control-plane" })
  image_id                  = var.image_id
  user_assigned_identity    = var.user_assigned_identity
@ -215,6 +216,7 @@ module "scale_set_worker" {
  location                  = var.location
  instance_type             = var.instance_type
  confidential_vm           = var.confidential_vm
+  secure_boot               = var.secure_boot
  tags                      = merge(local.tags, { role = "worker" })
  image_id                  = var.image_id
  user_assigned_identity    = var.user_assigned_identity
--- a/cli/internal/terraform/terraform/azure/modules/scale_set/main.tf
+++ b/cli/internal/terraform/terraform/azure/modules/scale_set/main.tf
@ -27,7 +27,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
  vtpm_enabled                    = true
  disable_password_authentication = false
  upgrade_mode                    = "Manual"
-  secure_boot_enabled             = true
+  secure_boot_enabled             = var.secure_boot
  source_image_id                 = var.image_id
  tags                            = var.tags

--- a/cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf
+++ b/cli/internal/terraform/terraform/azure/modules/scale_set/variables.tf
@ -71,3 +71,9 @@ variable "confidential_vm" {
  default     = true
  description = "Whether to deploy the cluster nodes as confidential VMs."
 }
+
+variable "secure_boot" {
+  type        = bool
+  default     = false
+  description = "Whether to deploy the cluster nodes with secure boot."
+}
--- a/cli/internal/terraform/terraform/azure/variables.tf
+++ b/cli/internal/terraform/terraform/azure/variables.tf
@ -57,6 +57,12 @@ variable "confidential_vm" {
  description = "Whether to deploy the cluster nodes as confidential VMs."
 }

+variable "secure_boot" {
+  type        = bool
+  default     = false
+  description = "Whether to deploy the cluster nodes with secure boot."
+}
+
 variable "debug" {
  type        = bool
  default     = false
--- a/cli/internal/terraform/terraform/qemu/main.tf
+++ b/cli/internal/terraform/terraform/qemu/main.tf
@ -59,6 +59,8 @@ module "control_plane" {
  pool            = libvirt_pool.cluster.name
  boot_volume_id  = libvirt_volume.constellation_coreos_image.id
  machine         = var.machine
+  firmware        = var.firmware
+  nvram           = var.nvram
  name            = var.name
 }

@ -74,6 +76,8 @@ module "worker" {
  pool            = libvirt_pool.cluster.name
  boot_volume_id  = libvirt_volume.constellation_coreos_image.id
  machine         = var.machine
+  firmware        = var.firmware
+  nvram           = var.nvram
  name            = var.name
 }

--- a/cli/internal/terraform/terraform/qemu/modules/instance_group/domain.xsl
+++ b/cli/internal/terraform/terraform/qemu/modules/instance_group/domain.xsl
@ -6,10 +6,31 @@
        </xsl:copy>
    </xsl:template>
    <xsl:template match="os">
-        <os firmware="efi">
+        <os>
            <xsl:apply-templates select="@*|node()"/>
        </os>
    </xsl:template>
+    <xsl:template match="/domain/os/loader">
+        <xsl:copy>
+            <!--<xsl:apply-templates select="node()|@*"/>-->
+            <xsl:attribute name="secure">
+                <xsl:value-of select="'yes'"/>
+            </xsl:attribute>
+            <xsl:attribute name="readonly">
+                <xsl:value-of select="'yes'"/>
+            </xsl:attribute>
+            <xsl:attribute name="type">
+                <xsl:value-of select="'pflash'"/>
+            </xsl:attribute>
+            <xsl:value-of select="."/>
+        </xsl:copy>
+    </xsl:template>
+    <xsl:template match="/domain/features">
+        <xsl:copy>
+            <xsl:apply-templates select="node()|@*"/>
+            <xsl:element name ="smm" />
+        </xsl:copy>
+    </xsl:template>
    <xsl:template match="/domain/devices/tpm/backend">
    <xsl:copy>
        <xsl:apply-templates select="node()|@*"/>
--- a/cli/internal/terraform/terraform/qemu/modules/instance_group/main.tf
+++ b/cli/internal/terraform/terraform/qemu/modules/instance_group/main.tf
@ -13,11 +13,16 @@ locals {
 }

 resource "libvirt_domain" "instance_group" {
-  name    = "${var.name}-${var.role}-${count.index}"
-  count   = var.amount
-  memory  = var.memory
-  vcpu    = var.vcpus
-  machine = var.machine
+  name     = "${var.name}-${var.role}-${count.index}"
+  count    = var.amount
+  memory   = var.memory
+  vcpu     = var.vcpus
+  machine  = var.machine
+  firmware = var.firmware
+  nvram {
+    file     = "/var/lib/libvirt/qemu/nvram/${var.role}-${count.index}_VARS.fd"
+    template = var.nvram
+  }
  tpm {
    backend_type    = "emulator"
    backend_version = "2.0"
--- a/cli/internal/terraform/terraform/qemu/modules/instance_group/variables.tf
+++ b/cli/internal/terraform/terraform/qemu/modules/instance_group/variables.tf
@ -48,7 +48,18 @@ variable "machine" {
  description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
 }

+variable "firmware" {
+  type        = string
+  description = "path to UEFI firmware file."
+}
+
+variable "nvram" {
+  type        = string
+  description = "path to UEFI NVRAM template file. Used for secure boot."
+}
+
 variable "name" {
  type        = string
  description = "name prefix of the cluster VMs"
 }
+
--- a/cli/internal/terraform/terraform/qemu/variables.tf
+++ b/cli/internal/terraform/terraform/qemu/variables.tf
@ -45,6 +45,17 @@ variable "machine" {
  description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
 }

+variable "firmware" {
+  type        = string
+  default     = "/usr/share/OVMF/OVMF_CODE.secboot.fd"
+  description = "path to UEFI firmware file. Use \"OVMF_CODE_4M.ms.fd\" on Ubuntu and \"OVMF_CODE.secboot.fd\" on Fedora."
+}
+
+variable "nvram" {
+  type        = string
+  description = "path to UEFI NVRAM template file. Used for secure boot."
+}
+
 variable "metadata_api_image" {
  type        = string
  description = "container image of the QEMU metadata api server"
--- a/cli/internal/terraform/variables.go
+++ b/cli/internal/terraform/variables.go
@ -97,6 +97,8 @@ type AzureVariables struct {
 	ImageID string
 	// ConfidentialVM sets the VM to be confidential.
 	ConfidentialVM bool
+	// SecureBoot sets the VM to use secure boot.
+	SecureBoot bool
 	// Debug is true if debug mode is enabled.
 	Debug bool
 }
@ -112,6 +114,7 @@ func (v *AzureVariables) String() string {
 	writeLinef(b, "state_disk_type = %q", v.StateDiskType)
 	writeLinef(b, "image_id = %q", v.ImageID)
 	writeLinef(b, "confidential_vm = %t", v.ConfidentialVM)
+	writeLinef(b, "secure_boot = %t", v.SecureBoot)
 	writeLinef(b, "debug = %t", v.Debug)

 	return b.String()
@ -140,6 +143,10 @@ type QEMUVariables struct {
 	// In case of unix socket, this should be "qemu:///system".
 	// Other wise it should be the same as LibvirtURI.
 	MetadataLibvirtURI string
+	// NVRAM is the path to the NVRAM template.
+	NVRAM string
+	// Firmware is the path to the firmware.
+	Firmware string
 }

 // String returns a string representation of the variables, formatted as Terraform variables.
@ -154,6 +161,17 @@ func (v *QEMUVariables) String() string {
 	writeLinef(b, "memory = %d", v.MemorySizeMiB)
 	writeLinef(b, "metadata_api_image = %q", v.MetadataAPIImage)
 	writeLinef(b, "metadata_libvirt_uri = %q", v.MetadataLibvirtURI)
+	switch v.NVRAM {
+	case "production":
+		b.WriteString("nvram = \"/usr/share/OVMF/constellation_vars.production.fd\"\n")
+	case "testing":
+		b.WriteString("nvram = \"/usr/share/OVMF/constellation_vars.testing.fd\"\n")
+	default:
+		writeLinef(b, "nvram = %q", v.NVRAM)
+	}
+	if v.Firmware != "" {
+		writeLinef(b, "firmware = %q", v.Firmware)
+	}

 	return b.String()
 }
--- a/debugd/internal/debugd/constants.go
+++ b/debugd/internal/debugd/constants.go
@ -14,7 +14,7 @@ const (
 	SSHCheckInterval                 = 30 * time.Second
 	DiscoverDebugdInterval           = 30 * time.Second
 	BootstrapperDownloadRetryBackoff = 1 * time.Minute
-	BootstrapperDeployFilename       = "/opt/bootstrapper"
+	BootstrapperDeployFilename       = "/run/state/bin/bootstrapper"
 	Chunksize                        = 1024
 	BootstrapperSystemdUnitName      = "bootstrapper.service"
 	BootstrapperSystemdUnitContents  = `[Unit]
@ -25,12 +25,10 @@ After=network-online.target
 Type=simple
 RemainAfterExit=yes
 Restart=on-failure
-EnvironmentFile=/etc/constellation.env
+EnvironmentFile=/run/constellation.env
+Environment=PATH=/run/state/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
 ExecStartPre=-setenforce Permissive
-ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin/
-# merge all CNI binaries in writable folder until containerd can use multiple CNI bins: https://github.com/containerd/containerd/issues/6600
-ExecStartPre=/bin/sh -c "/usr/bin/cp /usr/libexec/cni/* /opt/cni/bin/"
-ExecStart=/opt/bootstrapper
+ExecStart=/run/state/bin/bootstrapper
 [Install]
 WantedBy=multi-user.target
 `
--- a/debugd/internal/debugd/deploy/download_test.go
+++ b/debugd/internal/debugd/deploy/download_test.go
@ -36,7 +36,7 @@ func TestMain(m *testing.M) {
 }

 func TestDownloadBootstrapper(t *testing.T) {
-	filename := "/opt/bootstrapper"
+	filename := "/run/state/bin/bootstrapper"
 	someErr := errors.New("failed")

 	testCases := map[string]struct {
--- a/debugd/internal/debugd/deploy/service.go
+++ b/debugd/internal/debugd/deploy/service.go
@ -18,7 +18,7 @@ import (
 )

 const (
-	systemdUnitFolder = "/etc/systemd/system"
+	systemdUnitFolder = "/run/systemd/system"
 )

 //go:generate stringer -type=SystemdAction
--- a/debugd/internal/debugd/server/server_test.go
+++ b/debugd/internal/debugd/server/server_test.go
@ -188,7 +188,7 @@ func TestUploadBootstrapper(t *testing.T) {
 			assert.Equal(tc.wantResponseStatus, resp.Status)
 			if tc.wantFile {
 				assert.Equal(tc.wantChunks, tc.streamer.writeStreamChunks)
-				assert.Equal("/opt/bootstrapper", tc.streamer.writeStreamFilename)
+				assert.Equal("/run/state/bin/bootstrapper", tc.streamer.writeStreamFilename)
 			} else {
 				assert.Empty(tc.streamer.writeStreamChunks)
 				assert.Empty(tc.streamer.writeStreamFilename)
@ -256,7 +256,7 @@ func TestDownloadBootstrapper(t *testing.T) {
 			}
 			require.NoError(err)
 			assert.Equal(tc.wantChunks, chunks)
-			assert.Equal("/opt/bootstrapper", tc.streamer.readStreamFilename)
+			assert.Equal("/run/state/bin/bootstrapper", tc.streamer.readStreamFilename)
 		})
 	}
 }
--- a/image/README.md
+++ b/image/README.md
@ -1,113 +1,193 @@
-# Constellation images
-
-We use the [Fedora CoreOS Assembler](https://coreos.github.io/coreos-assembler/) to build the base image for Constellation nodes.
-
 ## Setup

-1. Install prerequisites:
-   - [Docker](https://docs.docker.com/engine/install/) or [Podman](https://podman.io/getting-started/installation)
-   - [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-linux)
-   - [azcopy](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10)
-   - [Google Cloud CLI](https://cloud.google.com/sdk/docs/install)
-   - [gsutil](https://cloud.google.com/storage/docs/gsutil_install#linux)
-   - Ubuntu:
+- Install mkosi (from git):

-        ```shell-session
-        sudo apt install -y bash coreutils cryptsetup-bin grep libguestfs-tools make parted pv qemu-system qemu-utils sed tar util-linux wget
-        ```
-
-2. Log in to GCP and Azure
-
-   ```shell-session
-   gcloud auth login
-   az login
-   ```
-
-3. [Log in to the ghcr.io package registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-to-the-container-registry)
-4. Ensure read and write access to `/dev/kvm` (and repeat after every reboot)
-
-    ```shell-session
-    sudo chmod 666 /dev/kvm
+    ```sh
+    cd /tmp/
+    git clone https://github.com/systemd/mkosi
+    cd mkosi
+    tools/generate-zipapp.sh
+    cp builddir/mkosi /usr/local/bin/
    ```

-## Configuration
+- Install tools:

-Create a configuration file in `image/config.mk` to override any of the variables found at the top of the [Makefile](Makefile).
-Important settings are:
+    <details>
+    <summary>Ubuntu / Debian</summary>

- `BOOTSTRAPPER_BINARY`: path to a bootstrapper binary. Can be substituted with a path to a `debugd` binary if a debug image should be built. The binary has to be built before!
- `CONTAINER_ENGINE`: container engine used to run COSA. either `podman` or `docker`.
- `COSA_INIT_REPO`: Git repository containing CoreOS config. Cloned in `cosa-init` target.
- `COSA_INIT_BRANCH`: Git branch checked out from `COSA_INIT_REPO`. Can be used to test out changes on another branch before merging.
- `NETRC` path to a netrc file containing a GitHub PAT. Used to authenticate to GitHub from within the COSA container.
- `GCP_IMAGE_NAME`: Image name for the GCP image. Set to include a timestamp when using the build pipeline. Can be set to a custom value if you want to upload a custom image for testing on GCP.
- `AZURE_IMAGE_NAME`: Image name for the Azure image. Can be set to a custom value if you want to upload a custom image for testing on Azure.
+    ```sh
+    sudo apt-get update
+    sudo apt-get install --assume-yes --no-install-recommends \
+        dnf \
+        systemd-container \
+        qemu-system-x86 \
+        qemu-utils \
+        ovmf \
+        e2fsprogs \
+        squashfs-tools \
+        efitools \
+        sbsigntool \
+        coreutils \
+        curl \
+        jq \
+        util-linux \
+        virt-manager
+    ```

-Example `config.mk` to create a debug image with docker and name it `my-custom-image`:
+    </details>

-```Makefile
-BOOTSTRAPPER_BINARY = ../build/debugd
-CONTAINER_ENGINE = docker
-GCP_IMAGE_NAME = my-custom-image
-AZURE_IMAGE_NAME = my-custom-image
+    <details>
+    <summary>Fedora</summary>
+
+    ```sh
+    sudo dnf install -y \
+        edk2-ovmf \
+        systemd-container \
+        qemu \
+        e2fsprogs \
+        squashfs-tools \
+        efitools \
+        sbsigntools \
+        coreutils \
+        curl \
+        jq \
+        util-linux \
+        virt-manager
+    ```
+
+    </details>
+
+- Prepare secure boot PKI (see `secure-boot/genkeys.sh`)
+
+## Build
+
+When building your first image, prepare the secure boot PKI (see `secure-boot/genkeys.sh`) for self-signed, locally built images.
+
+After that, you can build the image with:
+
+```sh
+# OPTIONAL: to create a debug image, export the following line
+# export BOOTSTRAPPER_BINARY=$(realpath ${PWD}/../../build/debugd)
+# OPTIONAL: symlink custom path to secure boot PKI to ./pki
+# ln -s /path/to/pki/folder ./pki
+sudo make -j $(nproc)
 ```

-## Build an image
+Raw images will be placed in `mkosi.output.<CSP>/fedora~36/image.raw`.

-Ensure you have the modified cosa container image installed:
+## Prepare Secure Boot

-```shell-session
-docker image ls | grep localhost/coreos-assembler
+The generated images are partially signed by Microsoft ([shim loader](https://github.com/rhboot/shim)), and partially signed by Edgeless Systems (systemd-boot and unified kernel images consisting of the linux kernel, initramfs and kernel commandline).
+
+For QEMU and Azure, you can pre-generate the NVRAM variables for secure boot. This is not necessary for GCP, as you can specify secure boot parameters via the GCP API on image creation.
+
+<details>
+<summary>libvirt / QEMU / KVM</summary>
+
+```sh
+secure-boot/generate_nvram_vars.sh mkosi.output.qemu/fedora~36/image.raw
 ```

-or
+</details>

-```shell-session
-podman image ls | grep localhost/coreos-assembler
+<details>
+<summary><a id="azure-secure-boot">Azure</a></summary>
+
+These steps only have to performed once for a fresh set of secure boot certificates.
+VMGS blobs for testing and release images already exist.
+
+First, create a disk without embedded MOK EFI variables.
+
+```sh
+# set these variables
+export AZURE_SECURITY_TYPE=ConfidentialVM # or TrustedLaunch
+export AZURE_RESOURCE_GROUP_NAME= # e.g. "constellation-images"
+
+export AZURE_REGION=northeurope
+export AZURE_DISK_NAME=constellation-$(date +%s)
+export AZURE_SNAPSHOT_NAME=${AZURE_DISK_NAME}
+export AZURE_RAW_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.raw
+export AZURE_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.vhd
+export AZURE_VMGS_FILENAME=${AZURE_SECURITY_TYPE}.vmgs
+export BLOBS_DIR=${PWD}/blobs
+upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
+upload/upload_azure.sh --disk-name "${AZURE_DISK_NAME}-setup-secure-boot" ""
+secure-boot/azure/launch.sh -n "${AZURE_DISK_NAME}-setup-secure-boot" -d --secure-boot true --disk-name "${AZURE_DISK_NAME}-setup-secure-boot"
 ```

-If not present, install with
+Ignore the running launch script and connect to the serial console once available.
+The console shows the message "Verification failed: (0x1A) Security Violation". You can import the MOK certificate via the UEFI shell:

-```shell-session
-make cosa-image
+Press OK, then ENTER, then "Enroll key from disk".
+Select the following key: `/EFI/loader/keys/auto/db.cer`.
+Press Continue, then choose "Yes" to the question "Enroll the key(s)?".
+Choose reboot.
+
+Extract the VMGS from the running VM (this includes the MOK EFI variables) and delete the VM:
+
+```sh
+secure-boot/azure/extract_vmgs.sh --name "${AZURE_DISK_NAME}-setup-secure-boot"
+secure-boot/azure/delete.sh --name "${AZURE_DISK_NAME}-setup-secure-boot"
 ```

-> It is always advisable to create an image from a clean `build` dir.
+</details>

-Clean up the `build` dir and remove old images (⚠ this will undo any local changes to the CoreOS configuration!):
+## Upload to CSP

-```shell-session
-sudo make clean
+<details>
+<summary>GCP</summary>
+
+- Install `gcloud` and `gsutil` (see [here](https://cloud.google.com/sdk/docs/install))
+- Login to GCP (see [here](https://cloud.google.com/sdk/docs/authorizing))
+- Choose secure boot PKI public keys (one of `pki_dev`, `pki_test`, `pki_prod`)
+    - `pki_dev` can be used for local image builds
+    - `pki_test` is used by the CI for non-release images
+    - `pki_prod` is used for release images
+
+```sh
+# set these variables
+export GCP_IMAGE_FAMILY= # e.g. "constellation"
+export GCP_IMAGE_NAME= # e.g. "constellation-v1.0.0"
+export PKI=${PWD}/pki
+
+export GCP_PROJECT=constellation-images
+export GCP_REGION=europe-west3
+export GCP_BUCKET=constellation-images
+export GCP_RAW_IMAGE_PATH=${PWD}/mkosi.output.gcp/fedora~36/image.raw
+export GCP_IMAGE_FILENAME=$(date +%s).tar.gz
+export GCP_IMAGE_PATH=${PWD}/mkosi.output.gcp/fedora~36/image.tar.gz
+upload/pack.sh gcp ${GCP_RAW_IMAGE_PATH} ${GCP_IMAGE_PATH}
+upload/upload_gcp.sh
 ```

- Build QEMU image (for local testing only)
+</details>

-  ```shell-session
-  make coreos
-  ```
+<details>
+<summary>Azure</summary>

- Build Azure image (without upload)
+- Install `az` and `azcopy` (see [here](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli))
+- Login to Azure (see [here](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli))
+- [Prepare virtual machine guest state (VMGS) with customized NVRAM or use existing VMGS blob](#azure-secure-boot)

-  ```shell-session
-  make image-azure
-  ```
+```sh
+# set these variables
+export AZURE_GALLERY_NAME= # e.g. "Constellation"
+export AZURE_IMAGE_DEFINITION= # e.g. "constellation"
+export AZURE_IMAGE_VERSION= # e.g. "1.0.0"
+export AZURE_VMGS_PATH= # e.g. "path/to/ConfidentialVM.vmgs"
+export AZURE_SECURITY_TYPE=ConfidentialVM # or TrustedLaunch

- Build Azure image (with upload)
+export AZURE_RESOURCE_GROUP_NAME=constellation-images
+export AZURE_REGION=northeurope
+export AZURE_REPLICATION_REGIONS="northeurope eastus westeurope westus"
+export AZURE_IMAGE_OFFER=constellation
+export AZURE_SKU=constellation
+export AZURE_PUBLISHER=edgelesssys
+export AZURE_DISK_NAME=constellation-$(date +%s)
+export AZURE_RAW_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.raw
+export AZURE_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.vhd
+upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
+upload/upload_azure.sh -g --disk-name "${AZURE_DISK_NAME}" "${AZURE_VMGS_PATH}"
+```

-  ```shell-session
-  make image-azure upload-azure
-  ```
-
- Build GCP image (without upload)
-
-  ```shell-session
-  make image-gcp
-  ```
-
- Build GCP image (with upload)
-
-  ```shell-session
-  make image-gcp upload-gcp
-  ```
-
-Resulting images for the CSPs can be found under [images](images/). QEMU images are stored at `build/builds/latest/` with a name ending in `.qcow2`.
+</details>
--- a/image/mkosi/.gitattributes
+++ b/image/mkosi/.gitattributes
--- a/image/mkosi/.gitignore
+++ b/image/mkosi/.gitignore
@ -0,0 +1,5 @@
+mkosi.cache
+mkosi.extra
+pki
+image.*
+mkosi.output.*
--- a/image/mkosi/Makefile
+++ b/image/mkosi/Makefile
@ -0,0 +1,54 @@
+SHELL                             = /bin/bash
+SRC_PATH                          = $(CURDIR)
+BASE_PATH                        ?= $(SRC_PATH)
+BOOTSTRAPPER_BINARY              ?= $(BASE_PATH)/../../build/bootstrapper
+DISK_MAPPER_BINARY               ?= $(BASE_PATH)/../../build/disk-mapper
+PKI                              ?= $(BASE_PATH)/pki
+MKOSI_EXTRA                      ?= $(BASE_PATH)/mkosi.extra
+-include $(CURDIR)/config.mk
+csps := qemu gcp azure
+certs := $(PKI)/PK.cer $(PKI)/KEK.cer $(PKI)/db.cer
+
+.PHONY: all clean inject-bins $(csps)
+
+all: $(csps)
+
+$(csps): %: mkosi.output.%/fedora~36/image.raw
+
+mkosi.output.%/fedora~36/image.raw: mkosi.files/mkosi.%.conf inject-bins inject-certs
+	mkosi --config mkosi.files/mkosi.$*.conf build
+	secure-boot/signed-shim.sh $@
+	@if [ -n $(SUDO_UID) ] && [ -n $(SUDO_GID) ]; then \
+		chown -R $(SUDO_UID):$(SUDO_GID) mkosi.output.$*; \
+	fi
+	@echo "Image is ready: $@"
+
+inject-bins:
+	mkdir -p $(MKOSI_EXTRA)/usr/bin
+	mkdir -p $(MKOSI_EXTRA)/usr/sbin
+	cp $(BOOTSTRAPPER_BINARY) $(MKOSI_EXTRA)/usr/bin/bootstrapper
+	cp $(DISK_MAPPER_BINARY) $(MKOSI_EXTRA)/usr/sbin/disk-mapper
+
+inject-certs: $(certs)
+	# for auto enrollment using systemd-boot (not working yet)
+	mkdir -p "$(MKOSI_EXTRA)/boot/loader/keys/auto"
+	cp $(PKI)/{PK,KEK,db}.cer "$(MKOSI_EXTRA)/boot/loader/keys/auto"
+	cp $(PKI)/{MicWinProPCA2011_2011-10-19,MicCorUEFCA2011_2011-06-27,MicCorKEKCA2011_2011-06-24}.crt "$(MKOSI_EXTRA)/boot/loader/keys/auto"
+	cp $(PKI)/{PK,KEK,db}.esl "$(MKOSI_EXTRA)/boot/loader/keys/auto"
+	cp $(PKI)/{PK,KEK,db}.auth "$(MKOSI_EXTRA)/boot/loader/keys/auto"
+	# for manual enrollment using sbkeysync
+	mkdir -p $(MKOSI_EXTRA)/etc/secureboot/keys/{db,dbx,KEK,PK}
+	cp $(PKI)/db.auth "$(MKOSI_EXTRA)/etc/secureboot/keys/db/"
+	cp $(PKI)/KEK.auth "$(MKOSI_EXTRA)/etc/secureboot/keys/KEK/"
+	cp $(PKI)/PK.auth "$(MKOSI_EXTRA)/etc/secureboot/keys/PK/"
+
+clean-cache:
+	rm -rf mkosi.cache/*
+
+clean-%:
+	mkosi --config mkosi.files/mkosi.$*.conf clean
+
+clean:
+	rm -rf mkosi.output.*
+	rm -rf $(MKOSI_EXTRA)
+	mkdir -p $(MKOSI_EXTRA)
--- a/image/mkosi/README.md
+++ b/image/mkosi/README.md
@ -0,0 +1,187 @@
+## Setup
+
+- Install mkosi (from git):
+
+    ```sh
+    cd /tmp/
+    git clone https://github.com/systemd/mkosi
+    cd mkosi
+    tools/generate-zipapp.sh
+    cp builddir/mkosi /usr/local/bin/
+    ```
+
+- Install tools:
+
+    <details>
+    <summary>Ubuntu / Debian</summary>
+
+    ```sh
+    sudo apt-get update
+    sudo apt-get install --assume-yes --no-install-recommends \
+        dnf \
+        systemd-container \
+        qemu-system-x86 \
+        qemu-utils \
+        ovmf \
+        e2fsprogs \
+        squashfs-tools \
+        efitools \
+        sbsigntool \
+        coreutils \
+        curl \
+        jq \
+        util-linux \
+        virt-manager
+    ```
+
+    </details>
+
+    <details>
+    <summary>Fedora</summary>
+
+    ```sh
+    sudo dnf install -y \
+        edk2-ovmf \
+        systemd-container \
+        qemu \
+        e2fsprogs \
+        squashfs-tools \
+        efitools \
+        sbsigntools \
+        coreutils \
+        curl \
+        jq \
+        util-linux \
+        virt-manager
+    ```
+
+    </details>
+
+- Prepare secure boot PKI (see `secure-boot/genkeys.sh`)
+
+## Build
+
+```sh
+# OPTIONAL: to create a debug image, export the following line
+# export BOOTSTRAPPER_BINARY=$(realpath ${PWD}/../../build/debugd)
+# OPTIONAL: specify path to secure boot PKI
+# export PKI=/path/to/pki/folder
+sudo make -j $(nproc)
+```
+
+Raw images will be placed in `mkosi.output.<CSP>/fedora~36/image.raw`.
+
+## Prepare Secure Boot
+
+The generated images are partially signed by Microsoft ([shim loader](https://github.com/rhboot/shim)), and partially signed by Edgeless Systems (systemd-boot and unified kernel images consisting of the linux kernel, initramfs and kernel commandline).
+
+For QEMU and Azure, you can pre-generate the NVRAM variables for secure boot. This is not necessary for GCP, as you can specify secure boot parameters via the GCP API on image creation.
+
+<details>
+<summary>libvirt / QEMU / KVM</summary>
+
+```sh
+secure-boot/generate_nvram_vars.sh mkosi.output.qemu/fedora~36/image.raw
+```
+
+</details>
+
+<details>
+<summary><a id="azure-secure-boot">Azure</a></summary>
+
+These steps only have to performed once for a fresh set of secure boot certificates.
+VMGS blobs for testing and release images already exist.
+
+First, create a disk without embedded MOK EFI variables.
+
+```sh
+# set these variables
+export AZURE_SECURITY_TYPE=ConfidentialVM # or TrustedLaunch
+export AZURE_RESOURCE_GROUP_NAME= # e.g. "constellation-images"
+
+export AZURE_REGION=northeurope
+export AZURE_DISK_NAME=constellation-$(date +%s)
+export AZURE_SNAPSHOT_NAME=${AZURE_DISK_NAME}
+export AZURE_RAW_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.raw
+export AZURE_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.vhd
+export AZURE_VMGS_FILENAME=${AZURE_SECURITY_TYPE}.vmgs
+export BLOBS_DIR=${PWD}/blobs
+upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
+upload/upload_azure.sh --disk-name "${AZURE_DISK_NAME}-setup-secure-boot" ""
+secure-boot/azure/launch.sh -n "${AZURE_DISK_NAME}-setup-secure-boot" -d --secure-boot true --disk-name "${AZURE_DISK_NAME}-setup-secure-boot"
+```
+
+Ignore the running launch script and connect to the serial console once available.
+The console shows the message "Verification failed: (0x1A) Security Violation". You can import the MOK certificate via the UEFI shell:
+
+Press OK, then ENTER, then "Enroll key from disk".
+Select the following key: `/EFI/loader/keys/auto/db.cer`.
+Press Continue, then choose "Yes" to the question "Enroll the key(s)?".
+Choose reboot.
+
+Extract the VMGS from the running VM (this includes the MOK EFI variables) and delete the VM:
+
+```sh
+secure-boot/azure/extract_vmgs.sh --name "${AZURE_DISK_NAME}-setup-secure-boot"
+secure-boot/azure/delete.sh --name "${AZURE_DISK_NAME}-setup-secure-boot"
+```
+
+</details>
+
+## Upload to CSP
+
+<details>
+<summary>GCP</summary>
+
+- Install `gcloud` and `gsutil` (see [here](https://cloud.google.com/sdk/docs/install))
+- Login to GCP (see [here](https://cloud.google.com/sdk/docs/authorizing))
+- Prepare secure boot PKI (see `secure-boot/genkeys.sh`)
+
+```sh
+# set these variables
+export GCP_IMAGE_FAMILY= # e.g. "constellation"
+export GCP_IMAGE_NAME= # e.g. "constellation-v1.0.0"
+export PKI=${PWD}/pki
+
+export GCP_PROJECT=constellation-images
+export GCP_REGION=europe-west3
+export GCP_BUCKET=constellation-images
+export GCP_RAW_IMAGE_PATH=${PWD}/mkosi.output.gcp/fedora~36/image.raw
+export GCP_IMAGE_FILENAME=$(date +%s).tar.gz
+export GCP_IMAGE_PATH=${PWD}/mkosi.output.gcp/fedora~36/image.tar.gz
+upload/pack.sh gcp ${GCP_RAW_IMAGE_PATH} ${GCP_IMAGE_PATH}
+upload/upload_gcp.sh
+```
+
+</details>
+
+<details>
+<summary>Azure</summary>
+
+- Install `az` and `azcopy` (see [here](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli))
+- Login to Azure (see [here](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli))
+- Prepare secure boot PKI (see `secure-boot/genkeys.sh`)
+- [Prepare virtual machine guest state (VMGS) with customized NVRAM or use existing VMGS blob](#azure-secure-boot)
+
+```sh
+# set these variables
+export AZURE_GALLERY_NAME= # e.g. "Constellation"
+export AZURE_IMAGE_DEFINITION= # e.g. "constellation"
+export AZURE_IMAGE_VERSION= # e.g. "1.0.0"
+export AZURE_VMGS_PATH= # e.g. "path/to/ConfidentialVM.vmgs"
+export AZURE_SECURITY_TYPE=ConfidentialVM # or TrustedLaunch
+
+export AZURE_RESOURCE_GROUP_NAME=constellation-images
+export AZURE_REGION=northeurope
+export AZURE_REPLICATION_REGIONS="northeurope eastus westeurope westus"
+export AZURE_IMAGE_OFFER=constellation
+export AZURE_SKU=constellation
+export AZURE_PUBLISHER=edgelesssys
+export AZURE_DISK_NAME=constellation-$(date +%s)
+export AZURE_RAW_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.raw
+export AZURE_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.vhd
+upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
+upload/upload_azure.sh -g --disk-name "${AZURE_DISK_NAME}" "${AZURE_VMGS_PATH}"
+```
+
+</details>
--- a/image/mkosi/mkosi.cache/.gitkeep
+++ b/image/mkosi/mkosi.cache/.gitkeep
--- a/image/mkosi/mkosi.conf.d/azure.conf
+++ b/image/mkosi/mkosi.conf.d/azure.conf
@ -0,0 +1,3 @@
+[Content]
+Packages=
+           WALinuxAgent-udev
--- a/image/mkosi/mkosi.conf.d/containers.conf
+++ b/image/mkosi/mkosi.conf.d/containers.conf
@ -0,0 +1,9 @@
+[Content]
+Packages=
+           containerd,
+           containernetworking-plugins,
+           iptables-nft,
+           ethtool,
+           socat,
+           iproute-tc,
+           conntrack-tools
--- a/image/mkosi/mkosi.conf.d/gcp.conf
+++ b/image/mkosi/mkosi.conf.d/gcp.conf
@ -0,0 +1,3 @@
+[Content]
+Packages=
+           nvme-cli
--- a/image/mkosi/mkosi.conf.d/mkosi.conf
+++ b/image/mkosi/mkosi.conf.d/mkosi.conf
@ -0,0 +1,22 @@
+[Distribution]
+Distribution=fedora
+Release=36
+
+[Output]
+Format=gpt_squashfs
+ManifestFormat=json,changelog
+Bootable=yes
+KernelCommandLine=mitigations=auto,nosmt preempt=full
+WithUnifiedKernelImages=yes
+Verity=yes
+CompressFs=zstd
+SplitArtifacts=yes
+# Enable Secure Boot with own PKI
+SecureBoot=yes
+SecureBootKey=pki/db.key
+SecureBootCertificate=pki/db.crt
+# TODO: Wait for systemd 252 to bring systemd-measure
+# Measure=yes
+
+[Host]
+QemuHeadless=yes
--- a/image/mkosi/mkosi.conf.d/network.conf
+++ b/image/mkosi/mkosi.conf.d/network.conf
@ -0,0 +1,8 @@
+[Content]
+Packages=
+           iproute,
+           dbus-broker,
+           systemd-networkd,
+           systemd-resolved,
+           dracut-network,
+           dhclient, # prevent NetworkManager from being pulled in by dracut-network
--- a/image/mkosi/mkosi.conf.d/secure-boot-tpm.conf
+++ b/image/mkosi/mkosi.conf.d/secure-boot-tpm.conf
@ -0,0 +1,8 @@
+[Content]
+# Secure Boot / EFI related packages for manual enrollment / verification of Secure Boot
+Packages=
+           e2fsprogs,
+           sbsigntools,
+           efitools,
+           mokutil,
+           tpm2-tools
--- a/image/mkosi/mkosi.conf.d/tools.conf
+++ b/image/mkosi/mkosi.conf.d/tools.conf
@ -0,0 +1,8 @@
+[Content]
+Packages=
+           passwd,
+           nano,
+           nano-default-editor,
+           vim,
+           curl,
+           wget
--- a/image/mkosi/mkosi.files/mkosi.azure.conf
+++ b/image/mkosi/mkosi.files/mkosi.azure.conf
@ -0,0 +1,3 @@
+[Output]
+KernelCommandLine=constel.csp=azure
+OutputDirectory=mkosi.output.azure
--- a/image/mkosi/mkosi.files/mkosi.gcp.conf
+++ b/image/mkosi/mkosi.files/mkosi.gcp.conf
@ -0,0 +1,3 @@
+[Output]
+KernelCommandLine=constel.csp=gcp
+OutputDirectory=mkosi.output.gcp
--- a/image/mkosi/mkosi.files/mkosi.qemu.conf
+++ b/image/mkosi/mkosi.files/mkosi.qemu.conf
@ -0,0 +1,3 @@
+[Output]
+KernelCommandLine=constel.csp=qemu
+OutputDirectory=mkosi.output.qemu
--- a/image/mkosi/mkosi.finalize
+++ b/image/mkosi/mkosi.finalize
@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+set -euxo pipefail
+
+# recreate kubelet systemd unit after reboot.
+# tmpfile config has to be written late as it interferes with the systemd-nspawn build environment
+cat >"${BUILDROOT}/usr/lib/tmpfiles.d/kubelet-service.conf" <<EOF
+C     /run/systemd/system/kubelet.service                   -    -    -     -           /run/state/systemd/system/kubelet.service
+C     /run/systemd/system/kubelet.service.d/10-kubeadm.conf -    -    -     -           /run/state/systemd/system/kubelet.service.d/10-kubeadm.conf
+EOF
+
+# cleanup dracut generation files (disk-mapper) to save space
+rm -rf "${BUILDROOT}/usr/lib/dracut/modules.d/39constellation-mount/"
--- a/image/mkosi/mkosi.postinst
+++ b/image/mkosi/mkosi.postinst
@ -0,0 +1,22 @@
+#!/bin/sh
+set -euxo pipefail
+
+# This will work in sd-boot 251 to auto-enroll secure boot keys.
+# https://www.freedesktop.org/software/systemd/man/systemd-boot.html
+# > CHANGES WITH 252 in spe:
+# > [...]
+# >     * sd-boot can automatically enroll SecureBoot keys from files found on
+# >       the ESP. This enrollment can be either automatic ('force' mode) or
+# >       controlled by the user ('manual' mode).
+# > [...]
+#
+# echo "secure-boot-enroll force" >> /boot/loader/loader.conf
+
+# create mountpoints in /etc
+mkdir -p /etc/{cni,kubernetes}
+
+# move issue files away from /etc
+# to allow /run/issue and /run/issue.d to take precedence
+mv /etc/issue.d /usr/lib/issue.d || true
+rm -f /etc/issue
+rm -f /etc/issue.net
--- a/image/mkosi/mkosi.skeleton/etc/dracut.conf.d/90-networkd.conf
+++ b/image/mkosi/mkosi.skeleton/etc/dracut.conf.d/90-networkd.conf
@ -0,0 +1,4 @@
+# enable networking in initrd (initramfs) with dracut and systemd-networkd
+install_items+=" /usr/lib/systemd/network/20-wired.network "
+install_items+=" /usr/lib/systemd/network/21-azure.network "
+add_dracutmodules+=" systemd-networkd "
--- a/image/mkosi/mkosi.skeleton/etc/dracut.conf.d/azure.conf
+++ b/image/mkosi/mkosi.skeleton/etc/dracut.conf.d/azure.conf
@ -0,0 +1,3 @@
+# add hyperv drivers to initramfs
+# (important for early networking)
+force_drivers+=" hv_netvsc hv_sock hv_storvsc hv_vmbus "
--- a/image/mkosi/mkosi.skeleton/etc/dracut.conf.d/gce.conf
+++ b/image/mkosi/mkosi.skeleton/etc/dracut.conf.d/gce.conf
@ -0,0 +1,2 @@
+# Include NVMe driver in initrd to boot on NVMe devices.
+force_drivers+=" nvme "
--- a/image/mkosi/mkosi.skeleton/etc/fstab
+++ b/image/mkosi/mkosi.skeleton/etc/fstab
@ -0,0 +1,5 @@
+/dev/mapper/state        /run/state         ext4    defaults,x-systemd.makefs,x-mount.mkdir    0    0
+/run/state/var           /var               none    defaults,bind,x-mount.mkdir                0    0
+/run/state/kubernetes    /etc/kubernetes    none    defaults,bind,x-mount.mkdir                0    0
+/run/state/etccni        /etc/cni/          none    defaults,bind,x-mount.mkdir                0    0
+/run/state/opt           /opt               none    defaults,bind,x-mount.mkdir                0    0
--- a/image/mkosi/mkosi.skeleton/etc/profile.d/constellation.sh
+++ b/image/mkosi/mkosi.skeleton/etc/profile.d/constellation.sh
@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+export PATH=/run/state/bin:${PATH}
+export KUBECONFIG=/etc/kubernetes/admin.conf
+alias k=kubectl
--- a/image/mkosi/mkosi.skeleton/usr/etc/containerd/config.toml
+++ b/image/mkosi/mkosi.skeleton/usr/etc/containerd/config.toml
@ -0,0 +1,216 @@
+disabled_plugins = []
+imports = []
+oom_score = 0
+plugin_dir = ""
+required_plugins = []
+root = "/var/lib/containerd"
+state = "/run/containerd"
+version = 2
+
+[cgroup]
+  path = ""
+
+[debug]
+  address = ""
+  format = ""
+  gid = 0
+  level = ""
+  uid = 0
+
+[grpc]
+  address = "/run/containerd/containerd.sock"
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+  tcp_address = ""
+  tcp_tls_cert = ""
+  tcp_tls_key = ""
+  uid = 0
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[plugins]
+
+  [plugins."io.containerd.gc.v1.scheduler"]
+    deletion_threshold = 0
+    mutation_threshold = 100
+    pause_threshold = 0.02
+    schedule_delay = "0s"
+    startup_delay = "100ms"
+
+  [plugins."io.containerd.grpc.v1.cri"]
+    disable_apparmor = false
+    disable_cgroup = false
+    disable_hugetlb_controller = true
+    disable_proc_mount = false
+    disable_tcp_service = true
+    enable_selinux = false
+    enable_tls_streaming = false
+    ignore_image_defined_volumes = false
+    max_concurrent_downloads = 3
+    max_container_log_line_size = 16384
+    netns_mounts_under_state_dir = false
+    restrict_oom_score_adj = false
+    sandbox_image = "k8s.gcr.io/pause:3.5"
+    selinux_category_range = 1024
+    stats_collect_period = 10
+    stream_idle_timeout = "4h0m0s"
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    systemd_cgroup = false
+    tolerate_missing_hugetlb_controller = true
+    unset_seccomp_profile = ""
+
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+      max_conf_num = 1
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      disable_snapshot_annotations = true
+      discard_unpacked_layers = false
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        base_runtime_spec = ""
+        container_annotations = []
+        pod_annotations = []
+        privileged_without_host_devices = false
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = ""
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          base_runtime_spec = ""
+          container_annotations = []
+          pod_annotations = []
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v2"
+
+          [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
+            BinaryName = ""
+            CriuImagePath = ""
+            CriuPath = ""
+            CriuWorkPath = ""
+            IoGid = 0
+            IoUid = 0
+            NoNewKeyring = false
+            NoPivotRoot = false
+            Root = ""
+            ShimCgroup = ""
+            SystemdCgroup = true
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
+        base_runtime_spec = ""
+        container_annotations = []
+        pod_annotations = []
+        privileged_without_host_devices = false
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = ""
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]
+
+    [plugins."io.containerd.grpc.v1.cri".image_decryption]
+      key_model = "node"
+
+    [plugins."io.containerd.grpc.v1.cri".registry]
+      config_path = ""
+
+      [plugins."io.containerd.grpc.v1.cri".registry.auths]
+
+      [plugins."io.containerd.grpc.v1.cri".registry.configs]
+
+      [plugins."io.containerd.grpc.v1.cri".registry.headers]
+
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+
+    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+
+  [plugins."io.containerd.internal.v1.opt"]
+    path = "/opt/containerd"
+
+  [plugins."io.containerd.internal.v1.restart"]
+    interval = "10s"
+
+  [plugins."io.containerd.metadata.v1.bolt"]
+    content_sharing_policy = "shared"
+
+  [plugins."io.containerd.monitor.v1.cgroups"]
+    no_prometheus = false
+
+  [plugins."io.containerd.runtime.v1.linux"]
+    no_shim = false
+    runtime = "runc"
+    runtime_root = ""
+    shim = "containerd-shim"
+    shim_debug = false
+
+  [plugins."io.containerd.runtime.v2.task"]
+    platforms = ["linux/amd64"]
+
+  [plugins."io.containerd.service.v1.diff-service"]
+    default = ["walking"]
+
+  [plugins."io.containerd.snapshotter.v1.aufs"]
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.btrfs"]
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.devmapper"]
+    async_remove = false
+    base_image_size = ""
+    pool_name = ""
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.native"]
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.overlayfs"]
+    root_path = ""
+
+  [plugins."io.containerd.snapshotter.v1.zfs"]
+    root_path = ""
+
+[proxy_plugins]
+
+[stream_processors]
+
+  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
+    accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
+    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
+    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
+    path = "ctd-decoder"
+    returns = "application/vnd.oci.image.layer.v1.tar"
+
+  [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
+    accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
+    args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
+    env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
+    path = "ctd-decoder"
+    returns = "application/vnd.oci.image.layer.v1.tar+gzip"
+
+[timeouts]
+  "io.containerd.timeout.shim.cleanup" = "5s"
+  "io.containerd.timeout.shim.load" = "5s"
+  "io.containerd.timeout.shim.shutdown" = "3s"
+  "io.containerd.timeout.task.state" = "2s"
+
+[ttrpc]
+  address = ""
+  gid = 0
+  uid = 0
--- a/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/configure-constel-csp.service
+++ b/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/configure-constel-csp.service
@ -0,0 +1 @@
+../../../systemd/system/configure-constel-csp.service
--- a/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/constellation-state-disk-generator
+++ b/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/constellation-state-disk-generator
--- a/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/google-nvme-disk.service
+++ b/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/google-nvme-disk.service
@ -0,0 +1,15 @@
+[Unit]
+Description=Force symlink creation for GCP nvme disks
+Before=prepare-state-disk.service
+After=network-online.target
+Wants=network-online.target
+ConditionKernelCommandLine=constel.csp=gcp
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash /usr/sbin/google-nvme-disk
+RemainAfterExit=yes
+StandardOutput=tty
+StandardInput=tty
+StandardError=tty
+TimeoutSec=infinity
--- a/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/google-nvme-disk.sh
+++ b/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/google-nvme-disk.sh
@ -0,0 +1,23 @@
+#!/bin/bash
+set -euo pipefail
+shopt -s extglob nullglob
+
+GCP_STATE_DISK_SYMLINK="/dev/disk/by-id/google-state-disk"
+
+# hack: google nvme udev rules are never executed. Create symlinks for the nvme devices manually.
+while [ ! -L "${GCP_STATE_DISK_SYMLINK}" ]
+do
+    for nvmedisk in /dev/nvme0n+([0-9])
+    do
+        /usr/lib/udev/google_nvme_id -s -d "${nvmedisk}" || true
+    done
+    if [ -L "${GCP_STATE_DISK_SYMLINK}" ]; then
+        break
+    fi
+    echo "Waiting for state disk to appear.."
+    sleep 2
+done
+
+echo "Google state disk found"
+echo ${GCP_STATE_DISK_SYMLINK} → $(readlink -f "${GCP_STATE_DISK_SYMLINK}")
+sleep 2
--- a/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/module-setup.sh
+++ b/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/module-setup.sh
@ -0,0 +1,56 @@
+#!/bin/bash
+
+depends() {
+    echo systemd
+}
+
+install_and_enable_unit() {
+    unit="$1"; shift
+    target="$1"; shift
+    inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
+    mkdir -p "${initdir}${systemdsystemconfdir}/${target}.wants"
+    ln_r "${systemdsystemunitdir}/${unit}" \
+        "${systemdsystemconfdir}/${target}.wants/${unit}"
+}
+
+install() {
+    inst_multiple \
+        bash
+    inst_script "/usr/sbin/disk-mapper" \
+        "/usr/sbin/disk-mapper"
+
+    inst_script "$moddir/prepare-state-disk.sh" \
+        "/usr/sbin/prepare-state-disk"
+    install_and_enable_unit "prepare-state-disk.service" \
+        "basic.target"
+    inst_script "$moddir/google-nvme-disk.sh" \
+        "/usr/sbin/google-nvme-disk"
+    install_and_enable_unit "google-nvme-disk.service" \
+        "basic.target"
+    install_and_enable_unit "configure-constel-csp.service" \
+        "basic.target"
+
+    # azure scsi disks
+    inst_multiple \
+        cut \
+        readlink
+
+    # gcp nvme disks
+    inst_multiple \
+        date \
+        xxd \
+        grep \
+        sed \
+        ln \
+        command \
+        readlink
+
+    inst_script "/usr/sbin/nvme" \
+        "/usr/sbin/nvme"
+    inst_script "/usr/lib/udev/google_nvme_id" \
+        "/usr/lib/udev/google_nvme_id"
+    inst_simple "/usr/lib/udev/rules.d/64-gce-disk-removal.rules" \
+        "/usr/lib/udev/rules.d/64-gce-disk-removal.rules"
+    inst_simple "/usr/lib/udev/rules.d/65-gce-disk-naming.rules" \
+        "/usr/lib/udev/rules.d/65-gce-disk-naming.rules"
+}
--- a/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/prepare-state-disk.service
+++ b/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/prepare-state-disk.service
@ -0,0 +1,17 @@
+[Unit]
+Description=Prepare encrypted state disk
+Before=initrd-fs.target
+After=network-online.target configure-constel-csp.service
+Wants=network-online.target
+Requires=initrd-root-fs.target
+FailureAction=reboot-immediate
+
+[Service]
+Type=oneshot
+EnvironmentFile=/run/constellation.env
+ExecStart=/bin/bash /usr/sbin/prepare-state-disk
+RemainAfterExit=yes
+StandardOutput=tty
+StandardInput=tty
+StandardError=tty
+TimeoutSec=infinity
--- a/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/prepare-state-disk.sh
+++ b/image/mkosi/mkosi.skeleton/usr/lib/dracut/modules.d/39constellation-mount/prepare-state-disk.sh
@ -0,0 +1,11 @@
+#!/bin/bash
+set -euo pipefail
+
+# Prepare the encrypted volume by either initializing it with a random key or by aquiring the key from another bootstrapper.
+# Store encryption key (random or recovered key) in /run/cryptsetup-keys.d/state.key
+disk-mapper -csp "${CONSTEL_CSP}"
+if [[ $? -ne 0 ]]; then
+    echo "Failed to prepare state disk"
+    sleep 2 # give the serial console time to print the error message
+    exit $? # exit with the same error code as disk-mapper
+fi
--- a/image/mkosi/mkosi.skeleton/usr/lib/environment.d/99-constellation.conf
+++ b/image/mkosi/mkosi.skeleton/usr/lib/environment.d/99-constellation.conf
@ -0,0 +1,2 @@
+PATH=/run/state/bin:$PATH
+KUBECONFIG=/etc/kubernetes/admin.conf
--- a/image/mkosi/mkosi.skeleton/usr/lib/modules-load.d/k8s.conf
+++ b/image/mkosi/mkosi.skeleton/usr/lib/modules-load.d/k8s.conf
@ -0,0 +1,2 @@
+overlay
+br_netfilter
--- a/image/mkosi/mkosi.skeleton/usr/lib/sysctl.d/10-cilium.conf
+++ b/image/mkosi/mkosi.skeleton/usr/lib/sysctl.d/10-cilium.conf
@ -0,0 +1,3 @@
+# See https://github.com/cilium/cilium/issues/10645
+net.ipv4.conf.lxc*.rp_filter = 0
+net.ipv4.conf.cilium_*.rp_filter = 0
--- a/image/mkosi/mkosi.skeleton/usr/lib/sysctl.d/10-k8s.conf
+++ b/image/mkosi/mkosi.skeleton/usr/lib/sysctl.d/10-k8s.conf
@ -0,0 +1,9 @@
+net.bridge.bridge-nf-call-ip6tables = 1
+net.bridge.bridge-nf-call-iptables  = 1
+net.ipv4.ip_forward                 = 1
+fs.inotify.max_user_instances       = 8192
+fs.inotify.max_user_watches         = 524288
+# kubernetes hardening (protectKernelDefaults=true)
+vm.overcommit_memory                = 1
+kernel.panic                        = 10
+kernel.panic_on_oops                = 1
--- a/image/mkosi/mkosi.skeleton/usr/lib/systemd/network/20-wired.network
+++ b/image/mkosi/mkosi.skeleton/usr/lib/systemd/network/20-wired.network
@ -0,0 +1,5 @@
+[Match]
+Name=en*
+
+[Network]
+DHCP=yes
--- a/image/mkosi/mkosi.skeleton/usr/lib/systemd/network/21-azure.network
+++ b/image/mkosi/mkosi.skeleton/usr/lib/systemd/network/21-azure.network
@ -0,0 +1,6 @@
+# Used as a fallback rule for Azure NICs as they are not named with "en*"
+[Match]
+Driver=hv_netvsc
+
+[Network]
+DHCP=yes
--- a/image/mkosi/mkosi.skeleton/usr/lib/systemd/system-preset/30-constellation.preset
+++ b/image/mkosi/mkosi.skeleton/usr/lib/systemd/system-preset/30-constellation.preset
@ -0,0 +1,5 @@
+enable constellation-bootstrapper.service
+enable configure-constel-csp.service
+enable containerd.service
+enable tpm-pcrs.service
+enable systemd-networkd.service
--- a/image/mkosi/mkosi.skeleton/usr/lib/systemd/system/configure-constel-csp.service
+++ b/image/mkosi/mkosi.skeleton/usr/lib/systemd/system/configure-constel-csp.service
@ -0,0 +1,10 @@
+[Unit]
+Description=Configures constellation cloud service provider environment variable
+
+[Service]
+Type=oneshot
+ExecStart=/bin/bash -c "CSP=$(< /proc/cmdline tr  ' ' '\n' | grep constel.csp | sed 's/constel.csp=//'); echo CONSTEL_CSP=$CSP >> /run/constellation.env"
+RemainAfterExit=yes
+
+[Install]
+WantedBy=multi-user.target
--- a/image/mkosi/mkosi.skeleton/usr/lib/systemd/system/constellation-bootstrapper.service
+++ b/image/mkosi/mkosi.skeleton/usr/lib/systemd/system/constellation-bootstrapper.service
@ -0,0 +1,15 @@
+[Unit]
+Description=Constellation Bootstrapper
+Wants=network-online.target
+After=network-online.target configure-constel-csp.service
+
+[Service]
+Type=simple
+RemainAfterExit=yes
+Restart=on-failure
+EnvironmentFile=/run/constellation.env
+Environment=PATH=/run/state/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
+ExecStart=/usr/bin/bootstrapper
+
+[Install]
+WantedBy=multi-user.target
--- a/image/mkosi/mkosi.skeleton/usr/lib/systemd/system/containerd.service.d/local.conf
+++ b/image/mkosi/mkosi.skeleton/usr/lib/systemd/system/containerd.service.d/local.conf
@ -0,0 +1,3 @@
+[Service]
+ExecStart=
+ExecStart=/usr/bin/containerd --config /usr/etc/containerd/config.toml
--- a/image/mkosi/mkosi.skeleton/usr/lib/systemd/system/tpm-pcrs.service
+++ b/image/mkosi/mkosi.skeleton/usr/lib/systemd/system/tpm-pcrs.service
@ -0,0 +1,11 @@
+[Unit]
+Description=Print PCR state on startup
+Before=constellation-bootstrapper.service
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/libexec/constellation-pcrs
+
+[Install]
+WantedBy=multi-user.target
--- a/image/mkosi/mkosi.skeleton/usr/lib/sysusers.d/constellation.conf
+++ b/image/mkosi/mkosi.skeleton/usr/lib/sysusers.d/constellation.conf
@ -0,0 +1,2 @@
+#Type Name     ID               GECOS                      Home directory Shell
+u     etcd     998:997          "etcd user"                /var/lib/etcd
--- a/image/mkosi/mkosi.skeleton/usr/lib/tmpfiles.d/constellation.conf
+++ b/image/mkosi/mkosi.skeleton/usr/lib/tmpfiles.d/constellation.conf
@ -0,0 +1,8 @@
+#Type Path                                                  Mode User Group Age         Argument
+d     /var/lib/etcd                                         0700 998  997   -           -
+d     /var/log/kubernetes/audit/                            0700   0    0   -           -
+d     /run/state/bin                                        0755   0    0   -           -
+d     /run/issue.d                                          0755   0    0   -           -
+C     /run/issue                                            -    -    -     -           /usr/lib/issue
+# merge all CNI binaries in writable folder until containerd can use multiple CNI bins: https://github.com/containerd/containerd/issues/6600
+C     /opt/cni/bin                                          -    -    -     -           /usr/libexec/cni/
--- a/image/mkosi/mkosi.skeleton/usr/lib/udev/google_nvme_id
+++ b/image/mkosi/mkosi.skeleton/usr/lib/udev/google_nvme_id
@ -0,0 +1,245 @@
+#!/bin/bash
+# Copyright 2020 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Used to generate symlinks for PD-NVMe devices using the disk names reported by
+# the metadata server
+
+# Locations of the script's dependencies
+readonly nvme_cli_bin=/usr/sbin/nvme
+
+# Bash regex to parse device paths and controller identification
+readonly NAMESPACE_NUMBER_REGEX="/dev/nvme[[:digit:]]+n([[:digit:]]+).*"
+readonly PARTITION_NUMBER_REGEX="/dev/nvme[[:digit:]]+n[[:digit:]]+p([[:digit:]]+)"
+readonly PD_NVME_REGEX="sn[[:space:]]+:[[:space]]+nvme_card-pd"
+
+# Globals used to generate the symlinks for a PD-NVMe disk.  These are populated
+# by the identify_pd_disk function and exported for consumption by udev rules.
+ID_SERIAL=''
+ID_SERIAL_SHORT=''
+
+#######################################
+# Helper function to log an error message to stderr.
+# Globals:
+#   None
+# Arguments:
+#   String to print as the log message
+# Outputs:
+#   Writes error to STDERR
+#######################################
+function err() {
+  echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
+}
+
+#######################################
+# Retrieves the device name for an NVMe namespace using nvme-cli.
+# Globals:
+#   Uses nvme_cli_bin
+# Arguments:
+#   The path to the nvme namespace (/dev/nvme0n?)
+# Outputs:
+#   The device name parsed from the JSON in the vendor ext of the ns-id command.
+# Returns:
+#   0 if the device name for the namespace could be retrieved, 1 otherwise
+#######################################
+function get_namespace_device_name() {
+  local nvme_json
+  nvme_json="$("$nvme_cli_bin" id-ns -b "$1" | xxd -p -seek 384 | xxd -p -r)"
+  if [[ $? -ne 0 ]]; then
+    return 1
+  fi
+
+  if [[ -z "$nvme_json" ]]; then
+    err "NVMe Vendor Extension disk information not present"
+    return 1
+  fi
+
+  local device_name
+  device_name="$(echo "$nvme_json" | grep device_name | sed -e 's/.*"device_name":[  \t]*"\([a-zA-Z0-9_-]\+\)".*/\1/')"
+
+  # Error if our device name is empty
+  if [[ -z "$device_name" ]]; then
+    err "Empty name"
+    return 1
+  fi
+
+  echo "$device_name"
+  return 0
+}
+
+#######################################
+# Retrieves the nsid for an NVMe namespace
+# Globals:
+#   None
+# Arguments:
+#   The path to the nvme namespace (/dev/nvme0n*)
+# Outputs:
+#   The namespace number/id
+# Returns:
+#   0 if the namespace id could be retrieved, 1 otherwise
+#######################################
+function get_namespace_number() {
+  local dev_path="$1"
+  local namespace_number
+  if [[ "$dev_path" =~ $NAMESPACE_NUMBER_REGEX ]]; then
+    namespace_number="${BASH_REMATCH[1]}"
+  else
+    return 1
+  fi
+
+  echo "$namespace_number"
+  return 0
+}
+
+#######################################
+# Retrieves the partition number for a device path if it exists
+# Globals:
+#   None
+# Arguments:
+#   The path to the device partition (/dev/nvme0n*p*)
+# Outputs:
+#   The value after 'p' in the device path, or an empty string if the path has
+#   no partition.
+#######################################
+function get_partition_number() {
+  local dev_path="$1"
+  local partition_number
+  if [[ "$dev_path" =~ $PARTITION_NUMBER_REGEX ]]; then
+    partition_number="${BASH_REMATCH[1]}"
+    echo "$partition_number"
+  else
+    echo ''
+  fi
+  return 0
+}
+
+#######################################
+# Generates a symlink for a PD-NVMe device using the metadata's disk name.
+# Primarily used for testing but can be used if the script is directly invoked.
+# Globals:
+#   Uses ID_SERIAL_SHORT (can be populated by identify_pd_disk)
+# Arguments:
+#   The device path for the disk
+#######################################
+function gen_symlink() {
+  local dev_path="$1"
+  local partition_number="$(get_partition_number "$dev_path")"
+
+  if [[ -n "$partition_number" ]]; then
+    ln -s "$dev_path" /dev/disk/by-id/google-"$ID_SERIAL_SHORT"-part"$partition_number" > /dev/null 2>&1
+  else
+    ln -s "$dev_path" /dev/disk/by-id/google-"$ID_SERIAL_SHORT" > /dev/null 2>&1
+  fi
+
+  return 0
+}
+
+#######################################
+# Populates the ID_* global variables with a disk's device name and namespace
+# Globals:
+#   Populates ID_SERIAL_SHORT, and ID_SERIAL
+# Arguments:
+#   The device path for the disk
+# Returns:
+#   0 on success and 1 if an error occurrs
+#######################################
+function identify_pd_disk() {
+  local dev_path="$1"
+  local dev_name
+  dev_name="$(get_namespace_device_name "$dev_path")"
+  if [[ $? -ne 0 ]]; then
+    return 1
+  fi
+
+  ID_SERIAL_SHORT="$dev_name"
+  ID_SERIAL="Google_PersistentDisk_${ID_SERIAL_SHORT}"
+  return 0
+}
+
+function print_help_message() {
+  echo "Usage: google_nvme_id [-s] [-h] -d device_path"
+  echo "  -d <device_path> (Required): Specifies the path to generate a name"
+  echo "        for.  This needs to be a path to an nvme device or namespace"
+  echo "  -s: Create symbolic link for the disk under /dev/disk/by-id."
+  echo "        Otherwise, the disk name will be printed to STDOUT"
+  echo "  -h: Print this help message"
+}
+
+function main() {
+  local opt_gen_symlink='false'
+  local device_path=''
+
+  while getopts :d:sh flag; do
+    case "$flag" in
+      d) device_path="$OPTARG";;
+      s) opt_gen_symlink='true';;
+      h) print_help_message
+         return 0
+         ;;
+      :) echo "Invalid option: ${OPTARG} requires an argument" 1>&2
+         return 1
+         ;;
+      *) return 1
+    esac
+  done
+
+  if [[ -z "$device_path" ]]; then
+    echo "Device path (-d) argument required. Use -h for full usage." 1>&2
+    exit 1
+  fi
+
+  # Ensure the nvme-cli command is installed
+  command -v "$nvme_cli_bin" > /dev/null 2>&1
+  if [[ $? -ne 0 ]]; then
+    err "The nvme utility (/usr/sbin/nvme) was not found. You may need to run \
+with sudo or install nvme-cli."
+    return 1
+  fi
+
+  # Ensure the passed device is actually an NVMe device
+  "$nvme_cli_bin" id-ctrl "$device_path" &>/dev/null
+  if [[ $? -ne 0 ]]; then
+    err "Passed device was not an NVMe device.  (You may need to run this \
+script as root/with sudo)."
+    return 1
+  fi
+
+  # Detect the type of attached nvme device
+  local controller_id
+  controller_id=$("$nvme_cli_bin" id-ctrl "$device_path")
+  if [[ ! "$controller_id" =~ nvme_card-pd ]] ; then
+    err "Device is not a PD-NVMe device"
+    return 1
+  fi
+
+  # Fill the global variables for the id command for the given disk type
+  # Error messages will be printed closer to error, no need to reprint here
+  identify_pd_disk "$device_path"
+  if [[ $? -ne 0 ]]; then
+    return $?
+  fi
+
+  # Gen symlinks or print out the globals set by the identify command
+  if [[ "$opt_gen_symlink" == 'true' ]]; then
+    gen_symlink "$device_path"
+  else
+    # These will be consumed by udev
+    echo "ID_SERIAL_SHORT=${ID_SERIAL_SHORT}"
+    echo "ID_SERIAL=${ID_SERIAL}"
+  fi
+
+  return $?
+
+}
+main "$@"
--- a/image/mkosi/mkosi.skeleton/usr/lib/udev/rules.d/64-gce-disk-removal.rules
+++ b/image/mkosi/mkosi.skeleton/usr/lib/udev/rules.d/64-gce-disk-removal.rules
@ -0,0 +1,17 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# When a disk is removed, unmount any remaining attached volumes.
+
+ACTION=="remove", SUBSYSTEM=="block", KERNEL=="sd*|vd*|nvme*", RUN+="/bin/sh -c '/bin/umount -fl /dev/$name && /usr/bin/logger -p daemon.warn -s WARNING: hot-removed /dev/$name that was still mounted, data may have been corrupted'"
--- a/image/mkosi/mkosi.skeleton/usr/lib/udev/rules.d/65-gce-disk-naming.rules
+++ b/image/mkosi/mkosi.skeleton/usr/lib/udev/rules.d/65-gce-disk-naming.rules
@ -0,0 +1,37 @@
+# Copyright 2016 Google Inc. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Name the attached disks as the specified by deviceName.
+
+ACTION!="add|change", GOTO="gce_disk_naming_end"
+SUBSYSTEM!="block", GOTO="gce_disk_naming_end"
+
+# SCSI naming
+KERNEL=="sd*|vd*", IMPORT{program}="scsi_id --export --whitelisted -d $tempnode"
+
+# NVME Local SSD naming
+KERNEL=="nvme*n*", ATTRS{model}=="nvme_card", PROGRAM="/bin/sh -c 'nsid=$$(echo %k|sed -re s/nvme[0-9]+n\([0-9]+\).\*/\\1/); echo $$((nsid-1))'", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-%c"
+KERNEL=="nvme*", ATTRS{model}=="nvme_card", ENV{ID_SERIAL}="Google_EphemeralDisk_$env{ID_SERIAL_SHORT}"
+
+# NVME Persistent Disk IO Timeout
+KERNEL=="nvme*n*", ENV{DEVTYPE}=="disk", ATTRS{model}=="nvme_card-pd", ATTR{queue/io_timeout}="4294967295"
+
+# NVME Persistent Disk Naming
+KERNEL=="nvme*n*", ATTRS{model}=="nvme_card-pd", IMPORT{program}="google_nvme_id -d $tempnode"
+
+# Symlinks
+KERNEL=="sd*|vd*|nvme*", ENV{DEVTYPE}=="disk", SYMLINK+="disk/by-id/google-$env{ID_SERIAL_SHORT}"
+KERNEL=="sd*|vd*|nvme*", ENV{DEVTYPE}=="partition", SYMLINK+="disk/by-id/google-$env{ID_SERIAL_SHORT}-part%n"
+
+LABEL="gce_disk_naming_end"
--- a/image/mkosi/mkosi.skeleton/usr/lib/udev/rules.d/98-override-systemd.rules
+++ b/image/mkosi/mkosi.skeleton/usr/lib/udev/rules.d/98-override-systemd.rules
@ -0,0 +1,3 @@
+# prevent systemd udev rules from marking unformatted device mapper device as unready (SYSTEMD_READY=0)
+# this is the offending rule from systemd: SUBSYSTEM=="block", ENV{DM_UUID}=="CRYPT-*", ENV{ID_PART_TABLE_TYPE}=="", ENV{ID_FS_USAGE}=="", ENV{SYSTEMD_READY}="0"
+SUBSYSTEM=="block", ENV{DM_NAME}=="state", ENV{DM_UUID}=="CRYPT-*", ENV{ID_PART_TABLE_TYPE}=="", ENV{ID_FS_USAGE}="constellation-state"
--- a/image/mkosi/mkosi.skeleton/usr/libexec/constellation-pcrs
+++ b/image/mkosi/mkosi.skeleton/usr/libexec/constellation-pcrs
@ -0,0 +1,10 @@
+#!/usr/bin/bash
+# This script reads the PCR state of the system
+# and prints the message to the serial console
+
+main() {
+    pcr_state="$(tpm2_pcrread sha256)"
+    echo -e "PCR state:\n${pcr_state}\n" > /run/issue.d/35_constellation_pcrs.issue
+}
+
+main
--- a/image/mkosi/pki_testing/KEK.auth
+++ b/image/mkosi/pki_testing/KEK.auth
--- a/image/mkosi/pki_testing/KEK.cer
+++ b/image/mkosi/pki_testing/KEK.cer
--- a/image/mkosi/pki_testing/KEK.crt
+++ b/image/mkosi/pki_testing/KEK.crt
@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID3zCCAsegAwIBAgIUHii75K8+vo3LkCUKJjGBiTVJy/8wDQYJKoZIhvcNAQEL
+BQAwgYgxCzAJBgNVBAYTAkRFMRwwGgYDVQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVu
+MQ8wDQYDVQQHDAZCb2NodW0xHjAcBgNVBAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21i
+SDEqMCgGA1UEAwwhQ29uc3RlbGxhdGlvbiBUZXN0aW5nIEtFSyBDQSAyMDIyMB4X
+DTIyMDkyMzE0MTYwN1oXDTIyMTAyMzE0MTYwN1owgYgxCzAJBgNVBAYTAkRFMRww
+GgYDVQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVuMQ8wDQYDVQQHDAZCb2NodW0xHjAc
+BgNVBAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21iSDEqMCgGA1UEAwwhQ29uc3RlbGxh
+dGlvbiBUZXN0aW5nIEtFSyBDQSAyMDIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEA9QnVns2kVMrbV6308MDN6xVoTLcQ91ashmpj5Zj/gDiidtguggWS
+hJ8bItzH2c02AWrMuf+T5wxaPWxZPjgYI9CnoBg53rgmGWpeBeV2ZWK8wRp0iUQe
+GmhaBP+aA4h6UYN7N1120kV0O4BmJg76JwecHK2VXom7H4ILD4EKvXgpiyRdKJm7
+CZaJ422G1LIjzLI1Gm2yhTsihyma9iguyV4EFLukp8umGneiE0erRngKw9a6YTWP
+eFfTyXtwaWVcGhA5guhkp5U/KQCY1Mr6e4+Zp3ISL8QFBneYbDpVI3nWDNfWPMAH
+eLxny2rpc+zvOJ+JDCc3frkWCLV4BZD8ZQIDAQABoz8wPTAdBgNVHQ4EFgQU8iGE
+QDAs7qF7LowPBQu0Tft2fbowDwYDVR0TAQH/BAUwAwEB/zALBgNVHQ8EBAMCAYYw
+DQYJKoZIhvcNAQELBQADggEBAKF9RGT46YAFhn2CrMuMd3eOalpoLM+SllAtmq5c
+7RqUGQBOhuAdf1aiucFXH8xzi7DOV6aVhQG67kv5isISqUdUL80+RpajWYcU5YaW
+jEX+w/o2Jv0kzBDBTVcX6uKuid1oiQDXGVL/UaU30Smdk/9ni1RImuPBPupNEljU
+AjaduiqqcJArLBmXzEizCnaGhEvdezPIiZDbzARDbkvl1WQthcghh3i6iiBiN7Vp
+gGMzEecGJ4oxlxa+fycIiFbrX8h3DgVOCXeaxWtbfmIuswwFeZ3rVXHXizfOMEdr
+2qWu/xQwwUfN3Z07kXigfB3akJfqDYO5/mNGNNi6fAJV2do=
+-----END CERTIFICATE-----
--- a/image/mkosi/pki_testing/KEK.esl
+++ b/image/mkosi/pki_testing/KEK.esl
--- a/image/mkosi/pki_testing/MicCorKEKCA2011_2011-06-24.crt
+++ b/image/mkosi/pki_testing/MicCorKEKCA2011_2011-06-24.crt
--- a/image/mkosi/pki_testing/MicCorKEKCA2011_2011-06-24.esl
+++ b/image/mkosi/pki_testing/MicCorKEKCA2011_2011-06-24.esl
--- a/image/mkosi/pki_testing/MicCorUEFCA2011_2011-06-27.crt
+++ b/image/mkosi/pki_testing/MicCorUEFCA2011_2011-06-27.crt
--- a/image/mkosi/pki_testing/MicCorUEFCA2011_2011-06-27.esl
+++ b/image/mkosi/pki_testing/MicCorUEFCA2011_2011-06-27.esl
--- a/image/mkosi/pki_testing/MicWinProPCA2011_2011-10-19.crt
+++ b/image/mkosi/pki_testing/MicWinProPCA2011_2011-10-19.crt
--- a/image/mkosi/pki_testing/MicWinProPCA2011_2011-10-19.esl
+++ b/image/mkosi/pki_testing/MicWinProPCA2011_2011-10-19.esl
--- a/image/mkosi/pki_testing/PK.auth
+++ b/image/mkosi/pki_testing/PK.auth
--- a/image/mkosi/pki_testing/PK.cer
+++ b/image/mkosi/pki_testing/PK.cer
--- a/image/mkosi/pki_testing/PK.crt
+++ b/image/mkosi/pki_testing/PK.crt
@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID4TCCAsmgAwIBAgIUUsz90g+sTYH+2QEMZf7gKWDiDEwwDQYJKoZIhvcNAQEL
+BQAwgYkxCzAJBgNVBAYTAkRFMRwwGgYDVQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVu
+MQ8wDQYDVQQHDAZCb2NodW0xHjAcBgNVBAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21i
+SDErMCkGA1UEAwwiQ29uc3RlbGxhdGlvbiBUZXN0aW5nIFVFRkkgQ0EgMjAyMjAe
+Fw0yMjA5MjMxNDE2MDZaFw0yMjEwMjMxNDE2MDZaMIGJMQswCQYDVQQGEwJERTEc
+MBoGA1UECAwTTm9yZHJoZWluIFdlc3RmYWxlbjEPMA0GA1UEBwwGQm9jaHVtMR4w
+HAYDVQQKDBVFZGdlbGVzcyBTeXN0ZW1zIEdtYkgxKzApBgNVBAMMIkNvbnN0ZWxs
+YXRpb24gVGVzdGluZyBVRUZJIENBIDIwMjIwggEiMA0GCSqGSIb3DQEBAQUAA4IB
+DwAwggEKAoIBAQC4fWmh6n3qjKmJSh0zvNc9frZnEcyfK3pDSCtdhNQh/kN59Tjd
+kg79yxPfG5v6MYiQhxva2EVPiAuMmB/zrhq75UmNU6o0WCVk58g7IrXDd4xpO+s
+32+umlntT86wHWfrSAj3JduBG8ci1J1uARz0mmHl2gOzoB0dn4WOAoLLMH2bAzjS
+ICX0giXiY6Q66JQya4OPMvdwSAW78H6JgTtNIsYKn2clnA3VJZWXIzYl6Bd92zjC
+6RFrhzu7WYw9nmIA0HMtjpSeU9JDPUmedU7MPqLAK6kpiR+RrowzkfaFmIUqdxpj
+4IlGXkqWqu/I3WzKucdt7X0Run914M5iM4xzAgMBAAGjPzA9MB0GA1UdDgQWBBQ9
+/pmrY9gxhPV658NbhCfmcGWkDjAPBgNVHRMBAf8EBTADAQH/MAsGA1UdDwQEAwIB
+hjANBgkqhkiG9w0BAQsFAAOCAQEAU+4eQJXS02qof7S+vkLOGznuC4KD0yXs9+jg
+Ih6ANg6YBlxNZWDWAYZeJIIrQfINnzC36dQcb4StiUOKJu4eT5YxH4Afv39L3eoZ
+eKsVE2Dddt4tE+i8oEBA7XPKZZH8le2V0csnZ2cbsphxftwy72qkFukmkmcBn4Zq
+fXeAsdbDHoKlnTPeTNAXcgPUyLGhxqoX5vaIsNDFPGQ3BhsHfmNgKkZ4J+BSGGnc
+R4Gre/mN1eNz3LYn7RZeExOrcnwnQAvFVhtz4ZFIndxP3kSiNh2Lo/7p/ECEnnEn
+jOUCkARuA3lUZiWMzGMDo/kgTi6C6kulEkCWcp417OeXmThrjA==
+-----END CERTIFICATE-----
--- a/image/mkosi/pki_testing/PK.esl
+++ b/image/mkosi/pki_testing/PK.esl
--- a/image/mkosi/pki_testing/db.auth
+++ b/image/mkosi/pki_testing/db.auth
--- a/image/mkosi/pki_testing/db.cer
+++ b/image/mkosi/pki_testing/db.cer
--- a/image/mkosi/pki_testing/db.crt
+++ b/image/mkosi/pki_testing/db.crt
@ -0,0 +1,23 @@
+-----BEGIN CERTIFICATE-----
+MIID2TCCAsGgAwIBAgIUQZYX6ujSb/QFL2p8BY62ydJ/HEYwDQYJKoZIhvcNAQEL
+BQAwgYUxCzAJBgNVBAYTAkRFMRwwGgYDVQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVu
+MQ8wDQYDVQQHDAZCb2NodW0xHjAcBgNVBAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21i
+SDEnMCUGA1UEAwweQ29uc3RlbGxhdGlvbiBUZXN0aW5nIFBDQSAyMDIyMB4XDTIy
+MDkyMzE0MTYwN1oXDTIyMTAyMzE0MTYwN1owgYUxCzAJBgNVBAYTAkRFMRwwGgYD
+VQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVuMQ8wDQYDVQQHDAZCb2NodW0xHjAcBgNV
+BAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21iSDEnMCUGA1UEAwweQ29uc3RlbGxhdGlv
+biBUZXN0aW5nIFBDQSAyMDIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEAx4lZNSnqGy1Dk8hxQHgzzIpCAr2+zPtNpPKdXq8PLIZ1fBERYkVW66JHH5Dl
+9pzMl8KaJkkRbv5orxzpL0g9qtM+j/JQl1Ap7S9MAXgysATwyaFWfddLW+ZOi/XS
+oOhq1JXp0FFFUBQwP56JtqhUEcgaICi41L9S/XMxQaoZfRQu+9ICVlDMLurqVdI4
+Or5mz7eWUqj34Xl5bpjMOZgbyI1pYN4UxvODKFIYhVTmKZOWA0np4JtI4iCA5Dkd
+ckF9uwOyQUyGUs7Atar2tZr2hWK66NxhcdSfj9ltuBHwskJnoJYesWPz0QHz26xO
+d+a4WwIKhKfchvv5qVq9KFkRpQIDAQABoz8wPTAdBgNVHQ4EFgQUvCP4FCxcypyU
+STwBjhtgqnojgaEwDwYDVR0TAQH/BAUwAwEB/zALBgNVHQ8EBAMCAYYwDQYJKoZI
+hvcNAQELBQADggEBAAhpPWCLVl8FvWZOeZA2iOKmLen/c6W0d9WiB9qvj+mI8QBq
+kLhkqkxEcXTpaOjCxJLqLU3LsA828yUdfL0zmxusrJlz+gux+KRlRn1yTsCyWqmx
+9rYUXO6IFwZnSUV923uVZ1nkHydwqV9hqIgrKYrppzDXOsm+ugz+NP2lxFNp6q8u
+MyMsClaTdfPT+EUucp7g1lPVdtWV4RbWK/v3/rp2l5jzs7F0Roa3zc/+YquwB/AA
+ZasNpDInz6RlEzhA/GkXEO5Rssem4a1NBwGrvs9mCIm5sKLaxLUnlM7SLCAeEBdS
+qxLwUVTOtnQzWEM04I978V3zfKNPBtl2mvhuXYw=
+-----END CERTIFICATE-----
--- a/image/mkosi/pki_testing/db.esl
+++ b/image/mkosi/pki_testing/db.esl
--- a/image/mkosi/secure-boot/azure/delete.sh
+++ b/image/mkosi/secure-boot/azure/delete.sh
@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -z "${CONFIG_FILE-}" ] && [ -f "${CONFIG_FILE-}" ]; then
+    . "${CONFIG_FILE}"
+fi
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -n|--name)
+      AZURE_VM_NAME="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+    *)
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift # past argument
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
+
+AZ_VM_INFO=$(az vm show --name "${AZURE_VM_NAME}" --resource-group "${AZURE_RESOURCE_GROUP_NAME}" -o json)
+NIC=$(echo "${AZ_VM_INFO}" | jq -r '.networkProfile.networkInterfaces[0].id')
+NIC_INFO=$(az network nic show --ids "${NIC}" -o json)
+PUBIP=$(echo "${NIC_INFO}" | jq -r '.ipConfigurations[0].publicIpAddress.id')
+NSG=$(echo "${NIC_INFO}" | jq -r '.networkSecurityGroup.id')
+SUBNET=$(echo "${NIC_INFO}" | jq -r '.ipConfigurations[0].subnet.id')
+VNET=$(echo $SUBNET | sed 's#/subnets/.*##')
+DISK=$(echo "${AZ_VM_INFO}" | jq -r '.storageProfile.osDisk.managedDisk.id')
+
+
+delete_vm () {
+    az vm delete -y --name "${AZURE_VM_NAME}" \
+        --resource-group "${AZURE_RESOURCE_GROUP_NAME}" || true
+}
+
+delete_vnet () {
+    az network vnet delete --ids "${VNET}" || true
+}
+
+delete_subnet () {
+    az network vnet subnet delete --ids "${SUBNET}" || true
+}
+
+delete_nsg () {
+    az network nsg delete --ids "${NSG}" || true
+}
+
+delete_pubip () {
+    az network public-ip delete --ids "${PUBIP}" || true
+}
+
+delete_disk () {
+    az disk delete -y --ids "${DISK}" || true
+}
+
+delete_nic () {
+    az network nic delete --ids "${NIC}" || true
+}
+
+delete_vm
+delete_disk
+delete_nic
+delete_nsg
+delete_subnet
+delete_vnet
+delete_pubip
--- a/image/mkosi/secure-boot/azure/extract_vmgs.sh
+++ b/image/mkosi/secure-boot/azure/extract_vmgs.sh
@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -z "${CONFIG_FILE-}" ] && [ -f "${CONFIG_FILE-}" ]; then
+    . "${CONFIG_FILE}"
+fi
+AZURE_SUBSCRIPTION=$(az account show --query id -o tsv)
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -n|--name)
+      AZURE_VM_NAME="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+    *)
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift # past argument
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
+
+VM_DISK=$(az vm show -g "${AZURE_RESOURCE_GROUP_NAME}" --name "${AZURE_VM_NAME}" --query "storageProfile.osDisk.managedDisk.id" -o tsv)
+LOCATION=$(az disk show --ids "${VM_DISK}" --query "location" -o tsv)
+
+az snapshot create \
+    -g "${AZURE_RESOURCE_GROUP_NAME}" \
+    --source "${VM_DISK}" \
+    --name "${AZURE_SNAPSHOT_NAME}" \
+    -l "${LOCATION}"
+
+# Azure CLI does not implement getSecureVMGuestStateSAS for snapshots yet
+# az snapshot grant-access \
+#     --duration-in-seconds 3600 \
+#     --access-level Read \
+#     --name "${AZURE_SNAPSHOT_NAME}" \
+#     -g "${AZURE_RESOURCE_GROUP_NAME}"
+
+BEGIN=$(az rest \
+    --method post \
+    --url "https://management.azure.com/subscriptions/${AZURE_SUBSCRIPTION}/resourceGroups/${AZURE_RESOURCE_GROUP_NAME}/providers/Microsoft.Compute/snapshots/${AZURE_SNAPSHOT_NAME}/beginGetAccess" \
+    --uri-parameters api-version="2021-12-01" \
+    --body '{"access": "Read", "durationInSeconds": 3600, "getSecureVMGuestStateSAS": true}' \
+    --verbose 2>&1)
+ASYNC_OPERATION_URI=$(echo "${BEGIN}" | grep Azure-AsyncOperation | cut -d ' ' -f 7 | tr -d "'")
+sleep 10
+ACCESS=$(az rest --method get --url "${ASYNC_OPERATION_URI}")
+VMGS_URL=$(echo "${ACCESS}" | jq -r '.properties.output.securityDataAccessSAS')
+
+curl -L -o "${AZURE_VMGS_FILENAME}" "${VMGS_URL}"
+
+az snapshot revoke-access \
+    --name "${AZURE_SNAPSHOT_NAME}" \
+    -g "${AZURE_RESOURCE_GROUP_NAME}"
+az snapshot delete \
+    --name "${AZURE_SNAPSHOT_NAME}" \
+    -g "${AZURE_RESOURCE_GROUP_NAME}"
+echo "VMGS saved to ${AZURE_VMGS_FILENAME}"
--- a/image/mkosi/secure-boot/azure/launch.sh
+++ b/image/mkosi/secure-boot/azure/launch.sh
@ -0,0 +1,101 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+if [ -z "${CONFIG_FILE-}" ] && [ -f "${CONFIG_FILE-}" ]; then
+    . "${CONFIG_FILE}"
+fi
+POSITIONAL_ARGS=()
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -n|--name)
+      AZURE_VM_NAME="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -g|--gallery)
+      CREATE_FROM_GALLERY=YES
+      shift # past argument
+      ;;
+    -d|--disk)
+      CREATE_FROM_GALLERY=NO
+      shift # past argument
+      ;;
+    --secure-boot)
+      AZURE_SECURE_BOOT="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    --disk-name)
+      AZURE_DISK_NAME="$2"
+      shift # past argument
+      shift # past value
+      ;;
+    -*|--*)
+      echo "Unknown option $1"
+      exit 1
+      ;;
+    *)
+      POSITIONAL_ARGS+=("$1") # save positional arg
+      shift # past argument
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
+
+if [[ "${AZURE_SECURITY_TYPE}" == "ConfidentialVM" ]]; then
+  VMSIZE="Standard_DC2as_v5"
+elif [[ "${AZURE_SECURITY_TYPE}" == "TrustedLaunch" ]]; then
+  VMSIZE="standard_D2as_v5"
+else
+  echo "Unknown security type: ${AZURE_SECURITY_TYPE}"
+  exit 1
+fi
+
+create_vm_from_disk () {
+    AZURE_DISK_REFERENCE=$(az disk show --resource-group ${AZURE_RESOURCE_GROUP_NAME} --name ${AZURE_DISK_NAME} --query id -o tsv)
+    az vm create --name "${AZURE_VM_NAME}" \
+        --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \
+        -l ${AZURE_REGION} \
+        --size "${VMSIZE}" \
+        --public-ip-sku Standard \
+        --os-type Linux \
+        --attach-os-disk "${AZURE_DISK_REFERENCE}" \
+        --security-type "${AZURE_SECURITY_TYPE}" \
+        --os-disk-security-encryption-type VMGuestStateOnly \
+        --enable-vtpm true \
+        --enable-secure-boot "${AZURE_SECURE_BOOT}" \
+        --boot-diagnostics-storage "" \
+        --no-wait
+}
+
+create_vm_from_sig () {
+    AZURE_IMAGE_REFERENCE=$(az sig image-version show \
+        --gallery-image-definition "${AZURE_IMAGE_DEFINITION}" \
+        --gallery-image-version "${AZURE_IMAGE_VERSION}" \
+        --gallery-name "${AZURE_GALLERY_NAME}" \
+        -g "${AZURE_RESOURCE_GROUP_NAME}" \
+        --query id -o tsv)
+    az vm create --name "${AZURE_VM_NAME}" \
+        --resource-group "${AZURE_RESOURCE_GROUP_NAME}" \
+        -l ${AZURE_REGION} \
+        --size "${VMSIZE}" \
+        --public-ip-sku Standard \
+        --image "${AZURE_IMAGE_REFERENCE}" \
+        --security-type "${AZURE_SECURITY_TYPE}" \
+        --os-disk-security-encryption-type VMGuestStateOnly \
+        --enable-vtpm true \
+        --enable-secure-boot "${AZURE_SECURE_BOOT}" \
+        --boot-diagnostics-storage "" \
+        --no-wait
+}
+
+if [ "$CREATE_FROM_GALLERY" = "YES" ]; then
+    create_vm_from_sig
+else
+    create_vm_from_disk
+fi
+
+sleep 30
+az vm boot-diagnostics enable --name "${AZURE_VM_NAME}" --resource-group "${AZURE_RESOURCE_GROUP_NAME}"
--- a/image/mkosi/secure-boot/generate_nvram_vars.sh
+++ b/image/mkosi/secure-boot/generate_nvram_vars.sh
@ -0,0 +1,89 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+BASE_DIR=$(realpath "${SCRIPT_DIR}/..")
+
+# Set to qemu+tcp://localhost:16599/system for dockerized libvirt setup
+if [[ -z "${LIBVIRT_SOCK}" ]]; then
+    LIBVIRT_SOCK=qemu:///system
+fi
+
+libvirt_nvram_gen () {
+    local image_path="${1}"
+    if test -f "${BASE_DIR}/image.nvram.template"; then
+        echo "NVRAM template already generated: $(realpath "--relative-to=$(pwd)" ${BASE_DIR}/image.nvram.template)"
+        return
+    fi
+    if ! test -f "${image_path}"; then
+        echo "Image \"${image_path}\" does not exist yet. To generate nvram, create disk image first."
+        return
+    fi
+
+    OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.ms.fd
+    OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.ms.fd
+    if ! test -f "${OVMF_CODE}"; then
+        OVMF_CODE=/usr/share/OVMF/OVMF_CODE.secboot.fd
+    fi
+    if ! test -f "${OVMF_VARS}"; then
+        OVMF_VARS=/usr/share/OVMF/OVMF_VARS.secboot.fd
+    fi
+
+    echo "Using OVMF_CODE: ${OVMF_CODE}"
+    echo "Using OVMF_VARS: ${OVMF_VARS}"
+
+    # generate nvram file using libvirt
+    virt-install --name constell-nvram-gen \
+        --connect ${LIBVIRT_SOCK} \
+        --nonetworks \
+        --description 'Constellation' \
+        --ram 1024 \
+        --vcpus 1 \
+        --osinfo detect=on,require=off \
+        --disk "${image_path},format=raw" \
+        --boot "machine=q35,menu=on,loader=${OVMF_CODE},loader.readonly=yes,loader.type=pflash,nvram.template=${OVMF_VARS},nvram=${BASE_DIR}/image.nvram,loader_secure=yes" \
+        --features smm.state=on \
+        --noautoconsole
+    echo -e 'connect using'
+    echo -e '    \u001b[1mvirsh console constell-nvram-gen\u001b[0m'
+    echo -e ''
+    echo -e 'Load db cert with MokManager or enroll full PKI with firmware setup'
+    echo -e ''
+    echo -e '    \u001b[1mMokManager\u001b[0m'
+    echo -e '    For mokmanager, try to boot as usual. You will see this message:'
+    echo -e '    > "Verification failed: (0x1A) Security Violation"'
+    echo -e '    Press OK, then ENTER, then "Enroll key from disk"'
+    echo -e '    Select the following key:'
+    echo -e '    > \u001b[1m/EFI/loader/keys/auto/db.cer\u001b[0m'
+    echo -e '    Press Continue, then choose "Yes" to the question "Enroll the key(s)?"'
+    echo -e '    Choose reboot and continue this script.'
+    echo -e ''
+    echo -e '    \u001b[1mFirmware setup\u001b[0m'
+    echo -e '    For firmware setup, press F2.'
+    echo -e '    Go to "Device Manager">"Secure Boot Configuration">"Secure Boot Mode"'
+    echo -e '    Choose "Custom Mode"'
+    echo -e '    Go to "Custom Securee Boot Options"'
+    echo -e '    Go to "PK Options">"Enroll PK", Press "Y" if queried, "Enroll PK using File"'
+    echo -e '    Select the following cert: \u001b[1m/EFI/loader/keys/auto/PK.cer\u001b[0m'
+    echo -e '    Choose "Commit Changes and Exit"'
+    echo -e '    Go to "KEK Options">"Enroll KEK", Press "Y" if queried, "Enroll KEK using File"'
+    echo -e '    Select the following cert: \u001b[1m/EFI/loader/keys/auto/KEK.cer\u001b[0m'
+    echo -e '    Choose "Commit Changes and Exit"'
+    echo -e '    Go to "DB Options">"Enroll Signature">"Enroll Signature using File"'
+    echo -e '    Select the following cert: \u001b[1m/EFI/loader/keys/auto/db.cer\u001b[0m'
+    echo -e '    Choose "Commit Changes and Exit"'
+    echo -e '    Repeat the last step for the following certs:'
+    echo -e '    > \u001b[1m/EFI/loader/keys/auto/MicWinProPCA2011_2011-10-19.crt\u001b[0m'
+    echo -e '    > \u001b[1m/EFI/loader/keys/auto/MicCorUEFCA2011_2011-06-27.crt\u001b[0m'
+    echo -e '    Reboot and continue this script.'
+    echo -e ''
+    echo -e 'Press ENTER to continue after you followed one of the guides from above.'
+    read
+    sudo cp "${BASE_DIR}/image.nvram" "${BASE_DIR}/image.nvram.template"
+    virsh --connect "${LIBVIRT_SOCK}" destroy --domain constell-nvram-gen
+    virsh --connect "${LIBVIRT_SOCK}" undefine --nvram constell-nvram-gen
+    rm -f "${BASE_DIR}/image.nvram"
+
+    echo "NVRAM template generated: $(realpath "--relative-to=$(pwd)" ${BASE_DIR}/image.nvram.template)"
+}
+
+libvirt_nvram_gen $1
--- a/image/mkosi/secure-boot/genkeys.sh
+++ b/image/mkosi/secure-boot/genkeys.sh
@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# This script generates a PKI for secure boot.
+# It is based on the example from https://github.com/systemd/systemd/blob/main/man/loader.conf.xml
+# This is meant to be used for development purposes only.
+# Release images are signed using a different set of keys.
+# Set PKI to an empty folder and PKI_SET to "dev".
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+TEMPLATES=${SCRIPT_DIR}/templates
+BASE_DIR=$(realpath "${SCRIPT_DIR}/..")
+if [ -z "${PKI}" ]; then
+    PKI=${BASE_DIR}/pki
+fi
+if [ -z "${PKI_SET}" ]; then
+    PKI_SET=dev
+fi
+
+gen_pki () {
+    # Only use for non-production images.
+    # Use real PKI for production images instead.
+    count=$(ls -1 ${PKI}/*.{key,crt,cer,esl,auth} 2>/dev/null | wc -l)
+    if [ $count != 0 ]
+    then
+        echo PKI files $(ls -1 $(realpath "--relative-to=$(pwd)" ${PKI})/*.{key,crt,cer,esl,auth}) already exist
+        return
+    fi
+    mkdir -p "${PKI}"
+    pushd "${PKI}"
+
+    uuid=$(systemd-id128 new --uuid)
+    for key in PK KEK db; do
+        openssl req -new -x509 -config "${TEMPLATES}/${PKI_SET}_${key}.conf" -keyout "${key}.key" -out "${key}.crt" -nodes
+        openssl x509 -outform DER -in "${key}.crt" -out "${key}.cer"
+        cert-to-efi-sig-list -g "${uuid}" "${key}.crt" "${key}.esl"
+    done
+
+    for key in MicWinProPCA2011_2011-10-19.crt MicCorUEFCA2011_2011-06-27.crt MicCorKEKCA2011_2011-06-24.crt; do
+        curl -sL "https://www.microsoft.com/pkiops/certs/${key}" --output "${key}"
+        sbsiglist --owner 77fa9abd-0359-4d32-bd60-28f4e78f784b --type x509 --output "${key%crt}esl" "${key}"
+    done
+
+    # Optionally add Microsoft Windows Production CA 2011 (needed to boot into Windows).
+    cat MicWinProPCA2011_2011-10-19.esl >> db.esl
+
+    # Optionally add Microsoft Corporation UEFI CA 2011 (for firmware drivers / option ROMs
+    # and third-party boot loaders (including shim). This is highly recommended on real
+    # hardware as not including this may soft-brick your device (see next paragraph).
+    cat MicCorUEFCA2011_2011-06-27.esl >> db.esl
+
+    # Optionally add Microsoft Corporation KEK CA 2011. Recommended if either of the
+    # Microsoft keys is used as the official UEFI revocation database is signed with this
+    # key. The revocation database can be updated with [fwupdmgr(1)](https://www.freedesktop.org/software/systemd/man/fwupdmgr.html#).
+    cat MicCorKEKCA2011_2011-06-24.esl >> KEK.esl
+
+    sign-efi-sig-list -c PK.crt -k PK.key PK PK.esl PK.auth
+    sign-efi-sig-list -c PK.crt -k PK.key KEK KEK.esl KEK.auth
+    sign-efi-sig-list -c KEK.crt -k KEK.key db db.esl db.auth
+
+    popd
+}
+
+# gen_pki generates a PKI for testing purposes only.
+# if keys/certs are already present in the pki folder, they are not regenerated.
+gen_pki
--- a/image/mkosi/secure-boot/signed-shim.sh
+++ b/image/mkosi/secure-boot/signed-shim.sh
@ -0,0 +1,45 @@
+#!/bin/env bash
+set -euo pipefail
+# This script is used to add a signed shim to the image.raw file EFI partition after running `mkosi build`.
+
+if (( $# != 1 ))
+then
+    echo "Usage: $0 <image.raw>"
+    exit 1
+fi
+
+# SOURCE is the URL used to download the signed shim RPM
+SOURCE=https://kojipkgs.fedoraproject.org/packages/shim/15.6/2/x86_64/shim-x64-15.6-2.x86_64.rpm
+# EXPECTED_SHA512 is the SHA512 checksum of the signed shim RPM
+EXPECTED_SHA512=971978bddee95a6a134ef05c4d88cf5df41926e631de863b74ef772307f3e106c82c8f6889c18280d47187986abd774d8671c5be4b85b1b0bb3d1858b65d02cf
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+BASE_DIR=$(realpath "${SCRIPT_DIR}/..")
+TMPDIR=$(mktemp -d)
+
+pushd "${TMPDIR}"
+
+curl -sL -o shim.rpm "${SOURCE}"
+echo "Checking SHA512 checksum of signed shim..."
+sha512sum -c <<< "${EXPECTED_SHA512}  shim.rpm"
+rpm2cpio shim.rpm | cpio -idmv
+echo $TMPDIR
+
+popd
+
+MOUNTPOINT=$(mktemp -d)
+sectoroffset=$(sfdisk -J "${1}" | jq -r '.partitiontable.partitions[0].start')
+byteoffset=$((sectoroffset * 512))
+mount -o offset="${byteoffset}" "${1}" "${MOUNTPOINT}"
+
+mkdir -p "${MOUNTPOINT}/EFI/BOOT/"
+cp "${TMPDIR}/boot/efi/EFI/BOOT/BOOTX64.EFI" "${MOUNTPOINT}/EFI/BOOT/"
+cp "${TMPDIR}/boot/efi/EFI/fedora/mmx64.efi" "${MOUNTPOINT}/EFI/BOOT/"
+cp "${MOUNTPOINT}/EFI/systemd/systemd-bootx64.efi" "${MOUNTPOINT}/EFI/BOOT/grubx64.efi"
+
+# Remove unused kernel and initramfs from EFI to save space
+# We boot from unified kernel image anyway
+rm -f "${MOUNTPOINT}"/*/*/{linux,initrd}
+
+umount "${MOUNTPOINT}"
+rm -rf ${MOUNTPOINT}
+rm -rf "${TMPDIR}"
--- a/image/mkosi/secure-boot/templates/dev_KEK.conf
+++ b/image/mkosi/secure-boot/templates/dev_KEK.conf
@ -0,0 +1,20 @@
+ [ req ]
+ default_bits           = 2048
+ distinguished_name     = req_distinguished_name
+ x509_extensions        = v3_req
+ req_extensions         = v3_req
+ prompt                 = no
+
+ dirstring_type = nobmp
+
+ [ req_distinguished_name ]
+ C                      = DE
+ ST                     = Nordrhein Westfalen
+ L                      = Bochum
+ O                      = Edgeless Systems GmbH
+ CN                     = Constellation Development KEK CA 2022
+
+ [ v3_req ]
+ subjectKeyIdentifier = hash
+ basicConstraints     = critical,CA:true
+ keyUsage             = digitalSignature,keyCertSign,cRLSign
--- a/image/mkosi/secure-boot/templates/dev_PK.conf
+++ b/image/mkosi/secure-boot/templates/dev_PK.conf
@ -0,0 +1,20 @@
+ [ req ]
+ default_bits           = 2048
+ distinguished_name     = req_distinguished_name
+ x509_extensions        = v3_req
+ req_extensions         = v3_req
+ prompt                 = no
+
+ dirstring_type = nobmp
+
+ [ req_distinguished_name ]
+ C                      = DE
+ ST                     = Nordrhein Westfalen
+ L                      = Bochum
+ O                      = Edgeless Systems GmbH
+ CN                     = Constellation Development UEFI CA 2022
+
+ [ v3_req ]
+ subjectKeyIdentifier = hash
+ basicConstraints     = critical,CA:true
+ keyUsage             = digitalSignature,keyCertSign,cRLSign
--- a/image/mkosi/secure-boot/templates/dev_db.conf
+++ b/image/mkosi/secure-boot/templates/dev_db.conf
@ -0,0 +1,20 @@
+ [ req ]
+ default_bits           = 2048
+ distinguished_name     = req_distinguished_name
+ x509_extensions        = v3_req
+ req_extensions         = v3_req
+ prompt                 = no
+
+ dirstring_type = nobmp
+
+ [ req_distinguished_name ]
+ C                      = DE
+ ST                     = Nordrhein Westfalen
+ L                      = Bochum
+ O                      = Edgeless Systems GmbH
+ CN                     = Constellation Development PCA 2022
+
+ [ v3_req ]
+ subjectKeyIdentifier = hash
+ basicConstraints     = critical,CA:true
+ keyUsage             = digitalSignature,keyCertSign,cRLSign
--- a/image/mkosi/secure-boot/templates/prod_KEK.conf
+++ b/image/mkosi/secure-boot/templates/prod_KEK.conf
@ -0,0 +1,20 @@
+ [ req ]
+ default_bits           = 2048
+ distinguished_name     = req_distinguished_name
+ x509_extensions        = v3_req
+ req_extensions         = v3_req
+ prompt                 = no
+
+ dirstring_type = nobmp
+
+ [ req_distinguished_name ]
+ C                      = DE
+ ST                     = Nordrhein Westfalen
+ L                      = Bochum
+ O                      = Edgeless Systems GmbH
+ CN                     = Constellation KEK CA 2022
+
+ [ v3_req ]
+ subjectKeyIdentifier = hash
+ basicConstraints     = critical,CA:true
+ keyUsage             = digitalSignature,keyCertSign,cRLSign
--- a/image/mkosi/secure-boot/templates/prod_PK.conf
+++ b/image/mkosi/secure-boot/templates/prod_PK.conf
@ -0,0 +1,20 @@
+ [ req ]
+ default_bits           = 2048
+ distinguished_name     = req_distinguished_name
+ x509_extensions        = v3_req
+ req_extensions         = v3_req
+ prompt                 = no
+
+ dirstring_type = nobmp
+
+ [ req_distinguished_name ]
+ C                      = DE
+ ST                     = Nordrhein Westfalen
+ L                      = Bochum
+ O                      = Edgeless Systems GmbH
+ CN                     = Constellation UEFI CA 2022
+
+ [ v3_req ]
+ subjectKeyIdentifier = hash
+ basicConstraints     = critical,CA:true
+ keyUsage             = digitalSignature,keyCertSign,cRLSign
--- a/image/mkosi/secure-boot/templates/prod_db.conf
+++ b/image/mkosi/secure-boot/templates/prod_db.conf
@ -0,0 +1,20 @@
+ [ req ]
+ default_bits           = 2048
+ distinguished_name     = req_distinguished_name
+ x509_extensions        = v3_req
+ req_extensions         = v3_req
+ prompt                 = no
+
+ dirstring_type = nobmp
+
+ [ req_distinguished_name ]
+ C                      = DE
+ ST                     = Nordrhein Westfalen
+ L                      = Bochum
+ O                      = Edgeless Systems GmbH
+ CN                     = Constellation Production PCA 2022
+
+ [ v3_req ]
+ subjectKeyIdentifier = hash
+ basicConstraints     = critical,CA:true
+ keyUsage             = digitalSignature,keyCertSign,cRLSign
--- a/image/mkosi/secure-boot/templates/testing_KEK.conf
+++ b/image/mkosi/secure-boot/templates/testing_KEK.conf
@ -0,0 +1,20 @@
+ [ req ]
+ default_bits           = 2048
+ distinguished_name     = req_distinguished_name
+ x509_extensions        = v3_req
+ req_extensions         = v3_req
+ prompt                 = no
+
+ dirstring_type = nobmp
+
+ [ req_distinguished_name ]
+ C                      = DE
+ ST                     = Nordrhein Westfalen
+ L                      = Bochum
+ O                      = Edgeless Systems GmbH
+ CN                     = Constellation Testing KEK CA 2022
+
+ [ v3_req ]
+ subjectKeyIdentifier = hash
+ basicConstraints     = critical,CA:true
+ keyUsage             = digitalSignature,keyCertSign,cRLSign
--- a/Show More
+++ b/Show More
				`@ -0,0 +1 @@`
				`../../../systemd/system/configure-constel-csp.service`