Create mkosi image build pipeline

This commit is contained in:
Malte Poll 2022-10-19 13:10:15 +02:00 committed by Malte Poll
parent e5aaf0a42f
commit 34367ea3cc
107 changed files with 2733 additions and 105 deletions

42
.github/actions/setup_mkosi/action.yaml vendored Normal file
View File

@ -0,0 +1,42 @@
name: Setup mkosi
description: Install mkosi and all its dependencies
inputs:
version:
description: "Version (commit hash) of mkosi to install."
required: true
runs:
using: composite
steps:
- name: Dependencies
shell: bash
run: |
echo "::group::Dependencies"
echo "deb-src http://archive.ubuntu.com/ubuntu/ $(lsb_release -cs) main restricted universe multiverse" | sudo tee -a /etc/apt/sources.list
sudo add-apt-repository ppa:michel-slm/kernel-utils
sudo apt-get update
sudo apt-get install --assume-yes --no-install-recommends \
dnf \
makepkg \
systemd-container \
qemu-system-x86 \
ovmf \
e2fsprogs \
squashfs-tools
echo "::endgroup::"
# Try to eliminate "Failed to dissect image: Connection timed out" errors from nspawn by compiling
# systemd-nspawn from v251 from source.
- name: Update systemd-nspawn
shell: bash
working-directory: ${{ github.action_path }}
run: |
echo "::group::Update systemd-nspawn"
sudo apt-get build-dep systemd
git clone https://github.com/systemd/systemd-stable --branch v251.2 --depth=1
meson systemd-stable/build systemd-stable
ninja -C systemd-stable/build systemd-nspawn
sudo ln -svf $PWD/systemd-stable/build/systemd-nspawn $(which systemd-nspawn)
systemd-nspawn --version
echo "::endgroup::"
- name: Install
shell: bash
run: sudo python3 -m pip install git+https://github.com/systemd/mkosi.git@${{ inputs.version }}

327
.github/workflows/build-os-image.yml vendored Normal file
View File

@ -0,0 +1,327 @@
name: Build and Upload OS image
on:
workflow_dispatch:
inputs:
imageVersion:
description: "Semantic version including patch e.g. v<major>.<minor>.<patch> (only used for releases)"
required: false
debug:
description: "Build debug image"
type: boolean
default: false
required: false
jobs:
build-dependencies:
name: "Build binaries for embedding in the OS"
runs-on: ubuntu-22.04
permissions:
contents: read
packages: read
steps:
- name: Checkout
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791
- name: Build bootstrapper
if: ${{ inputs.debug == false }}
uses: ./.github/actions/build_bootstrapper
with:
outputPath: ${{ github.workspace }}/build/bootstrapper
- name: Build debugd
if: ${{ inputs.debug == true }}
uses: ./.github/actions/build_debugd
with:
outputPath: ${{ github.workspace }}/build/bootstrapper
- name: Build disk-mapper
uses: ./.github/actions/build_disk_mapper
with:
outputPath: ${{ github.workspace }}/build/disk-mapper
- name: Upload dependencies
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
with:
name: dependencies
path: |
${{ github.workspace }}/build/bootstrapper
${{ github.workspace }}/build/disk-mapper
- name: Collect hashes
run: |
echo "::set-output name=bootstrapper-sha256::$(sha256sum bootstrapper)"
echo "::set-output name=disk-mapper-sha256::$(sha256sum disk-mapper)"
working-directory: ${{ github.workspace }}/build
make-os-image:
name: "Build OS using mkosi"
needs: build-dependencies
runs-on: ubuntu-22.04
strategy:
matrix:
csp: [azure, gcp, qemu]
steps:
- name: Checkout
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791
- name: Download build dependencies
uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
with:
name: dependencies
path: ${{ github.workspace }}/build
- name: Mark bootstrapper and disk-mapper as executable
run: |
chmod +x ${{ github.workspace }}/build/bootstrapper
chmod +x ${{ github.workspace }}/build/disk-mapper
- name: Setup mkosi
uses: ./.github/actions/setup_mkosi
with:
version: 1750fc261a071f39c54f8b64ce2424195f750ee0
- name: Prepare PKI for secure boot signing
shell: bash
run: |
ln -s pki_testing pki
echo "${DB_KEY}" > pki/db.key
working-directory: ${{ github.workspace }}/image/mkosi
env:
DB_KEY: ${{ secrets.SECURE_BOOT_TESTING_DB_KEY }}
- name: Build
shell: bash
run: |
echo "::group::Build"
sudo make "${CSP}"
echo "::endgroup::"
working-directory: ${{ github.workspace }}/image/mkosi
env:
BOOTSTRAPPER_BINARY: ${{ github.workspace }}/build/bootstrapper
DISK_MAPPER_BINARY: ${{ github.workspace }}/build/disk-mapper
CSP: ${{ matrix.csp }}
- name: Upload raw OS image as artifact
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
with:
name: image-${{ matrix.csp }}
path: ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.raw
if: always()
continue-on-error: true
- name: Upload individual OS parts as artifacts
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
with:
name: parts-${{ matrix.csp }}
path: |
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.cmdline
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.efi
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.initrd
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.root.raw
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.root.roothash
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.root.verity
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.vmlinuz
if: always()
continue-on-error: true
- name: Upload manifest as artifact
uses: actions/upload-artifact@3cea5372237819ed00197afe530f5a7ea3e805c8
with:
name: manifest-${{ matrix.csp }}
path: |
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.raw.changelog
${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36/image.raw.manifest
if: always()
continue-on-error: true
upload-os-image:
name: "Upload OS image to CSP"
needs: make-os-image
runs-on: ubuntu-22.04
strategy:
matrix:
csp: [azure, gcp]
upload-variant: [""]
include:
- csp: azure
upload-variant: TrustedLaunch
steps:
- name: Checkout
uses: actions/checkout@e2f20e631ae6d7dd3b768f56a5d2af784dd54791
- name: Download OS image artifact
uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
with:
name: image-${{ matrix.csp }}
path: ${{ github.workspace }}/image/mkosi/mkosi.output.${{ matrix.csp }}/fedora~36
- name: Install tools
shell: bash
run: |
echo "::group::Install tools"
sudo apt-get update
sudo apt-get install -y pigz qemu-utils
echo "::endgroup::"
- name: Login to Azure
uses: azure/login@92a5484dfaf04ca78a94597f4f19fea633851fa2
if: ${{ matrix.csp == 'azure' }}
with:
creds: ${{ secrets.AZURE_CREDENTIALS }}
- name: Login to GCP
uses: ./.github/actions/gcp_login
if: ${{ matrix.csp == 'gcp' }}
with:
gcp_service_account_json: ${{ secrets.GCP_IMAGE_UPLOAD_SERVICE_ACCOUNT }}
- name: Prepare PKI for image upload
shell: bash
run: ln -s pki_testing pki
working-directory: ${{ github.workspace }}/image/mkosi
- name: Determine version
id: version
uses: ./.github/actions/pseudo_version
# Make sure to set valid names for GCP and Azure
# Azure
# gallery name may include alphanumeric characters, dots and underscores. Must end and begin with an alphanumeric character
# image definition may include alphanumeric characters, dots, dashes and underscores. Must end and begin with an alphanumeric character
# image version has to be semantic version in the form <uint>.<uint>.<uint> . uint may not be larger than 2,147,483,647
#
# GCP
# image family and image name may include lowercase alphanumeric characters and dashes. Must not end or begin with a dash
- name: Configure input variables
shell: bash
run: |
timestamp=${{ steps.version.outputs.timestamp }}
semver=${{ steps.version.outputs.semanticVersion }}
imageVersion=${{ inputs.imageVersion }}
pseudover=${{ steps.version.outputs.pseudoVersion }}
echo "PKI=${{ github.workspace }}/image/mkosi/pki" >> $GITHUB_ENV
echo "GCP_PROJECT=constellation-images" >> $GITHUB_ENV
echo "GCP_BUCKET=constellation-images" >> $GITHUB_ENV
echo "GCP_REGION=europe-west3" >> $GITHUB_ENV
echo "GCP_RAW_IMAGE_PATH=${{ github.workspace }}/image/mkosi/mkosi.output.gcp/fedora~36/image.raw" >> $GITHUB_ENV
echo "GCP_IMAGE_PATH=${{ github.workspace }}/image/mkosi/mkosi.output.gcp/fedora~36/image.tar.gz" >> $GITHUB_ENV
echo "AZURE_RESOURCE_GROUP_NAME=constellation-images" >> $GITHUB_ENV
echo "AZURE_REGION=northeurope" >> $GITHUB_ENV
echo "AZURE_REPLICATION_REGIONS=northeurope eastus westeurope westus" >> $GITHUB_ENV
echo "AZURE_SKU=constellation" >> $GITHUB_ENV
echo "AZURE_PUBLISHER=edgelesssys" >> $GITHUB_ENV
echo "AZURE_RAW_IMAGE_PATH=${{ github.workspace }}/image/mkosi/mkosi.output.azure/fedora~36/image.raw" >> $GITHUB_ENV
echo "AZURE_IMAGE_PATH=${{ github.workspace }}/image/mkosi/mkosi.output.azure/fedora~36/image.vhd" >> $GITHUB_ENV
# TODO: set default security type to "ConfidentialVM" once replication is possible
AZURE_SECURITY_TYPE=${{ matrix.upload-variant }}
if [ -z "${AZURE_SECURITY_TYPE}" ]; then
AZURE_SECURITY_TYPE=ConfidentialVMSupported
fi
echo "AZURE_SECURITY_TYPE=${AZURE_SECURITY_TYPE}" >> $GITHUB_ENV
echo "AZURE_DISK_NAME=constellation-${pseudover//./-}-${AZURE_SECURITY_TYPE,,}" >> $GITHUB_ENV
if [ "${{ startsWith(github.ref, 'refs/heads/release/') && (inputs.debug == false) }}" = true ]
then
GCP_IMAGE_NAME=constellation-${imageVersion//./-}
echo "GCP_IMAGE_FAMILY=constellation" >> $GITHUB_ENV
AZURE_IMAGE_DEFINITION=constellation
echo "AZURE_IMAGE_VERSION=${imageVersion:1}" >> $GITHUB_ENV
AZURE_GALLERY_NAME=Constellation
elif [ "${{ ((github.ref == 'refs/heads/main') || startsWith(github.ref, 'refs/heads/release/')) && (inputs.debug == true) }}" = true ]
then
GCP_IMAGE_NAME=constellation-${{ steps.version.outputs.timestamp }}
echo "GCP_IMAGE_FAMILY=constellation-debug-${semver//./-}" >> $GITHUB_ENV
AZURE_IMAGE_DEFINITION=${semver}
echo "AZURE_IMAGE_VERSION=${timestamp:0:4}.${timestamp:4:4}.${timestamp:8}" >> $GITHUB_ENV
AZURE_GALLERY_NAME=Constellation_Debug
else
GCP_IMAGE_NAME=constellation-${{ steps.version.outputs.timestamp }}
echo "GCP_IMAGE_FAMILY=constellation-${{ steps.version.outputs.branchName }}" >> $GITHUB_ENV
AZURE_IMAGE_DEFINITION=${{ steps.version.outputs.branchName }}
echo "AZURE_IMAGE_VERSION=${timestamp:0:4}.${timestamp:4:4}.${timestamp:8}" >> $GITHUB_ENV
AZURE_GALLERY_NAME=Constellation_Testing
fi
# TODO: enable VMGS upload for ConfidentialVM images once replication is possible
if [ "${AZURE_SECURITY_TYPE}" == "ConfidentialVMSupported" ]; then
echo "AZURE_GALLERY_NAME=${AZURE_GALLERY_NAME}_CVM" >> $GITHUB_ENV
echo "AZURE_VMGS_PATH=" >> $GITHUB_ENV
else
echo "AZURE_GALLERY_NAME=${AZURE_GALLERY_NAME}" >> $GITHUB_ENV
echo "AZURE_VMGS_PATH=${{ github.workspace }}/image/mkosi/pki/${AZURE_SECURITY_TYPE}.vmgs" >> $GITHUB_ENV
fi
echo "AZURE_IMAGE_DEFINITION=${AZURE_IMAGE_DEFINITION}" >> $GITHUB_ENV
echo "AZURE_IMAGE_OFFER=${AZURE_IMAGE_DEFINITION}" >> $GITHUB_ENV
echo "GCP_IMAGE_NAME=${GCP_IMAGE_NAME}" >> $GITHUB_ENV
echo "GCP_IMAGE_FILENAME=${GCP_IMAGE_NAME}.tar.gz" >> $GITHUB_ENV
- name: Download VMGS blob
run: aws s3 cp \
s3://constellation-secure-boot/pki_testing/${AZURE_SECURITY_TYPE}.vmgs \
pki_testing/${AZURE_SECURITY_TYPE}.vmgs \
--no-progress
working-directory: ${{ github.workspace }}/image/mkosi
if: ${{ matrix.csp == 'azure' }}
- name: Upload GCP image
shell: bash
run: |
echo "::group::Upload GCP image"
upload/pack.sh gcp "${GCP_RAW_IMAGE_PATH}" "${GCP_IMAGE_PATH}"
upload/upload_gcp.sh
echo -e "Uploaded GCP image: \`projects/${GCP_PROJECT}/global/images/${GCP_IMAGE_NAME}\`" >> $GITHUB_STEP_SUMMARY
echo "::endgroup::"
working-directory: ${{ github.workspace }}/image/mkosi
if: ${{ matrix.csp == 'gcp' }}
- name: Upload Azure image
shell: bash
run: |
echo "::group::Upload Azure image"
upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
upload/upload_azure.sh -g --disk-name "${AZURE_DISK_NAME}" "${AZURE_VMGS_PATH}"
echo -e "Uploaded Azure ${AZURE_SECURITY_TYPE} image: \`/subscriptions/0d202bbb-4fa7-4af8-8125-58c269a05435/resourceGroups/${AZURE_RESOURCE_GROUP_NAME^^}/providers/Microsoft.Compute/galleries/${AZURE_GALLERY_NAME}/images/${AZURE_IMAGE_DEFINITION}/versions/${AZURE_IMAGE_VERSION}\`" >> $GITHUB_STEP_SUMMARY
echo "::endgroup::"
working-directory: ${{ github.workspace }}/image/mkosi
if: ${{ matrix.csp == 'azure' }}
generate-sbom:
name: "Generate SBOM"
needs: make-os-image
runs-on: ubuntu-22.04
steps:
- name: Install squashfs tools
run: |
echo "::group::Install squashfs tools"
sudo apt-get update
sudo apt-get install -y squashfs-tools
echo "::endgroup::"
- name: Download rootfs
uses: actions/download-artifact@fb598a63ae348fa914e94cd0ff38f362e927b741
with:
# downloading / using only the QEMU rootfs is fine
# since the images only differ in the ESP partition
name: parts-qemu
- name: Unpack squashfs
run: |
echo "::group::Unpack squashfs"
unsquashfs -user-xattrs -d image.root.tree image.root.raw
echo "::endgroup::"
- uses: anchore/sbom-action@eda59434a8e5ce9bda93a202dfe50f6a2e637bb6
with:
path: image.root.tree
artifact-name: sbom.spdx.json
format: spdx-json
- uses: anchore/sbom-action@eda59434a8e5ce9bda93a202dfe50f6a2e637bb6
with:
path: image.root.tree
artifact-name: sbom.cyclonedx.json
format: cyclonedx-json
- uses: anchore/sbom-action@eda59434a8e5ce9bda93a202dfe50f6a2e637bb6
with:
path: image.root.tree
artifact-name: sbom.syft.json
format: syft-json

View File

@ -12,9 +12,10 @@ const (
binDir = "/run/state/bin" binDir = "/run/state/bin"
kubeadmPath = "/run/state/bin/kubeadm" kubeadmPath = "/run/state/bin/kubeadm"
kubeletPath = "/run/state/bin/kubelet" kubeletPath = "/run/state/bin/kubelet"
kubeletServiceEtcPath = "/etc/systemd/system/kubelet.service" kubectlPath = "/run/state/bin/kubectl"
kubeletServiceEtcPath = "/run/systemd/system/kubelet.service"
kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service" kubeletServiceStatePath = "/run/state/systemd/system/kubelet.service"
kubeadmConfEtcPath = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" kubeadmConfEtcPath = "/run/systemd/system/kubelet.service.d/10-kubeadm.conf"
kubeadmConfStatePath = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf" kubeadmConfStatePath = "/run/state/systemd/system/kubelet.service.d/10-kubeadm.conf"
executablePerm = 0o544 executablePerm = 0o544
systemdUnitPerm = 0o644 systemdUnitPerm = 0o644

View File

@ -128,6 +128,7 @@ func (c *Creator) createAzure(ctx context.Context, cl terraformClient, config *c
StateDiskType: config.Provider.Azure.StateDiskType, StateDiskType: config.Provider.Azure.StateDiskType,
ImageID: config.Provider.Azure.Image, ImageID: config.Provider.Azure.Image,
ConfidentialVM: *config.Provider.Azure.ConfidentialVM, ConfidentialVM: *config.Provider.Azure.ConfidentialVM,
SecureBoot: *config.Provider.Azure.SecureBoot,
Debug: config.IsDebugCluster(), Debug: config.IsDebugCluster(),
} }
@ -210,6 +211,8 @@ func (c *Creator) createQEMU(ctx context.Context, cl terraformClient, lv libvirt
MemorySizeMiB: config.Provider.QEMU.Memory, MemorySizeMiB: config.Provider.QEMU.Memory,
MetadataAPIImage: config.Provider.QEMU.MetadataAPIImage, MetadataAPIImage: config.Provider.QEMU.MetadataAPIImage,
MetadataLibvirtURI: metadataLibvirtURI, MetadataLibvirtURI: metadataLibvirtURI,
NVRAM: config.Provider.QEMU.NVRAM,
Firmware: config.Provider.QEMU.Firmware,
} }
ip, err := cl.CreateCluster(ctx, name, &vars) ip, err := cl.CreateCluster(ctx, name, &vars)

View File

@ -17,6 +17,11 @@ RUN echo "cgroup_controllers = []" >> /etc/libvirt/qemu.conf && \
echo "listen_addr = \"localhost\"" >> /etc/libvirt/libvirtd.conf && \ echo "listen_addr = \"localhost\"" >> /etc/libvirt/libvirtd.conf && \
echo "auth_tcp = \"none\"" >> /etc/libvirt/libvirtd.conf echo "auth_tcp = \"none\"" >> /etc/libvirt/libvirtd.conf
# Copy nvram templates
COPY ./cli/internal/libvirt/nvram/constellation_vars.testing.fd /usr/share/OVMF/constellation_vars.testing.fd
# TODO: Uncomment this line when we have a production template
# COPY ./cli/internal/libvirt/nvram/constellation_vars.production.fd /usr/share/OVMF/constellation_vars.production.fd
COPY --chmod=755 ./cli/internal/libvirt/start.sh /start.sh COPY --chmod=755 ./cli/internal/libvirt/start.sh /start.sh
ENTRYPOINT ["/start.sh"] ENTRYPOINT ["/start.sh"]

View File

@ -193,6 +193,7 @@ module "scale_set_control_plane" {
location = var.location location = var.location
instance_type = var.instance_type instance_type = var.instance_type
confidential_vm = var.confidential_vm confidential_vm = var.confidential_vm
secure_boot = var.secure_boot
tags = merge(local.tags, { role = "control-plane" }) tags = merge(local.tags, { role = "control-plane" })
image_id = var.image_id image_id = var.image_id
user_assigned_identity = var.user_assigned_identity user_assigned_identity = var.user_assigned_identity
@ -215,6 +216,7 @@ module "scale_set_worker" {
location = var.location location = var.location
instance_type = var.instance_type instance_type = var.instance_type
confidential_vm = var.confidential_vm confidential_vm = var.confidential_vm
secure_boot = var.secure_boot
tags = merge(local.tags, { role = "worker" }) tags = merge(local.tags, { role = "worker" })
image_id = var.image_id image_id = var.image_id
user_assigned_identity = var.user_assigned_identity user_assigned_identity = var.user_assigned_identity

View File

@ -27,7 +27,7 @@ resource "azurerm_linux_virtual_machine_scale_set" "scale_set" {
vtpm_enabled = true vtpm_enabled = true
disable_password_authentication = false disable_password_authentication = false
upgrade_mode = "Manual" upgrade_mode = "Manual"
secure_boot_enabled = true secure_boot_enabled = var.secure_boot
source_image_id = var.image_id source_image_id = var.image_id
tags = var.tags tags = var.tags

View File

@ -71,3 +71,9 @@ variable "confidential_vm" {
default = true default = true
description = "Whether to deploy the cluster nodes as confidential VMs." description = "Whether to deploy the cluster nodes as confidential VMs."
} }
variable "secure_boot" {
type = bool
default = false
description = "Whether to deploy the cluster nodes with secure boot."
}

View File

@ -57,6 +57,12 @@ variable "confidential_vm" {
description = "Whether to deploy the cluster nodes as confidential VMs." description = "Whether to deploy the cluster nodes as confidential VMs."
} }
variable "secure_boot" {
type = bool
default = false
description = "Whether to deploy the cluster nodes with secure boot."
}
variable "debug" { variable "debug" {
type = bool type = bool
default = false default = false

View File

@ -59,6 +59,8 @@ module "control_plane" {
pool = libvirt_pool.cluster.name pool = libvirt_pool.cluster.name
boot_volume_id = libvirt_volume.constellation_coreos_image.id boot_volume_id = libvirt_volume.constellation_coreos_image.id
machine = var.machine machine = var.machine
firmware = var.firmware
nvram = var.nvram
name = var.name name = var.name
} }
@ -74,6 +76,8 @@ module "worker" {
pool = libvirt_pool.cluster.name pool = libvirt_pool.cluster.name
boot_volume_id = libvirt_volume.constellation_coreos_image.id boot_volume_id = libvirt_volume.constellation_coreos_image.id
machine = var.machine machine = var.machine
firmware = var.firmware
nvram = var.nvram
name = var.name name = var.name
} }

View File

@ -6,10 +6,31 @@
</xsl:copy> </xsl:copy>
</xsl:template> </xsl:template>
<xsl:template match="os"> <xsl:template match="os">
<os firmware="efi"> <os>
<xsl:apply-templates select="@*|node()"/> <xsl:apply-templates select="@*|node()"/>
</os> </os>
</xsl:template> </xsl:template>
<xsl:template match="/domain/os/loader">
<xsl:copy>
<!--<xsl:apply-templates select="node()|@*"/>-->
<xsl:attribute name="secure">
<xsl:value-of select="'yes'"/>
</xsl:attribute>
<xsl:attribute name="readonly">
<xsl:value-of select="'yes'"/>
</xsl:attribute>
<xsl:attribute name="type">
<xsl:value-of select="'pflash'"/>
</xsl:attribute>
<xsl:value-of select="."/>
</xsl:copy>
</xsl:template>
<xsl:template match="/domain/features">
<xsl:copy>
<xsl:apply-templates select="node()|@*"/>
<xsl:element name ="smm" />
</xsl:copy>
</xsl:template>
<xsl:template match="/domain/devices/tpm/backend"> <xsl:template match="/domain/devices/tpm/backend">
<xsl:copy> <xsl:copy>
<xsl:apply-templates select="node()|@*"/> <xsl:apply-templates select="node()|@*"/>

View File

@ -13,11 +13,16 @@ locals {
} }
resource "libvirt_domain" "instance_group" { resource "libvirt_domain" "instance_group" {
name = "${var.name}-${var.role}-${count.index}" name = "${var.name}-${var.role}-${count.index}"
count = var.amount count = var.amount
memory = var.memory memory = var.memory
vcpu = var.vcpus vcpu = var.vcpus
machine = var.machine machine = var.machine
firmware = var.firmware
nvram {
file = "/var/lib/libvirt/qemu/nvram/${var.role}-${count.index}_VARS.fd"
template = var.nvram
}
tpm { tpm {
backend_type = "emulator" backend_type = "emulator"
backend_version = "2.0" backend_version = "2.0"

View File

@ -48,7 +48,18 @@ variable "machine" {
description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'" description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
} }
variable "firmware" {
type = string
description = "path to UEFI firmware file."
}
variable "nvram" {
type = string
description = "path to UEFI NVRAM template file. Used for secure boot."
}
variable "name" { variable "name" {
type = string type = string
description = "name prefix of the cluster VMs" description = "name prefix of the cluster VMs"
} }

View File

@ -45,6 +45,17 @@ variable "machine" {
description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'" description = "machine type. use 'q35' for secure boot and 'pc' for non secure boot. See 'qemu-system-x86_64 -machine help'"
} }
variable "firmware" {
type = string
default = "/usr/share/OVMF/OVMF_CODE.secboot.fd"
description = "path to UEFI firmware file. Use \"OVMF_CODE_4M.ms.fd\" on Ubuntu and \"OVMF_CODE.secboot.fd\" on Fedora."
}
variable "nvram" {
type = string
description = "path to UEFI NVRAM template file. Used for secure boot."
}
variable "metadata_api_image" { variable "metadata_api_image" {
type = string type = string
description = "container image of the QEMU metadata api server" description = "container image of the QEMU metadata api server"

View File

@ -97,6 +97,8 @@ type AzureVariables struct {
ImageID string ImageID string
// ConfidentialVM sets the VM to be confidential. // ConfidentialVM sets the VM to be confidential.
ConfidentialVM bool ConfidentialVM bool
// SecureBoot sets the VM to use secure boot.
SecureBoot bool
// Debug is true if debug mode is enabled. // Debug is true if debug mode is enabled.
Debug bool Debug bool
} }
@ -112,6 +114,7 @@ func (v *AzureVariables) String() string {
writeLinef(b, "state_disk_type = %q", v.StateDiskType) writeLinef(b, "state_disk_type = %q", v.StateDiskType)
writeLinef(b, "image_id = %q", v.ImageID) writeLinef(b, "image_id = %q", v.ImageID)
writeLinef(b, "confidential_vm = %t", v.ConfidentialVM) writeLinef(b, "confidential_vm = %t", v.ConfidentialVM)
writeLinef(b, "secure_boot = %t", v.SecureBoot)
writeLinef(b, "debug = %t", v.Debug) writeLinef(b, "debug = %t", v.Debug)
return b.String() return b.String()
@ -140,6 +143,10 @@ type QEMUVariables struct {
// In case of unix socket, this should be "qemu:///system". // In case of unix socket, this should be "qemu:///system".
// Other wise it should be the same as LibvirtURI. // Other wise it should be the same as LibvirtURI.
MetadataLibvirtURI string MetadataLibvirtURI string
// NVRAM is the path to the NVRAM template.
NVRAM string
// Firmware is the path to the firmware.
Firmware string
} }
// String returns a string representation of the variables, formatted as Terraform variables. // String returns a string representation of the variables, formatted as Terraform variables.
@ -154,6 +161,17 @@ func (v *QEMUVariables) String() string {
writeLinef(b, "memory = %d", v.MemorySizeMiB) writeLinef(b, "memory = %d", v.MemorySizeMiB)
writeLinef(b, "metadata_api_image = %q", v.MetadataAPIImage) writeLinef(b, "metadata_api_image = %q", v.MetadataAPIImage)
writeLinef(b, "metadata_libvirt_uri = %q", v.MetadataLibvirtURI) writeLinef(b, "metadata_libvirt_uri = %q", v.MetadataLibvirtURI)
switch v.NVRAM {
case "production":
b.WriteString("nvram = \"/usr/share/OVMF/constellation_vars.production.fd\"\n")
case "testing":
b.WriteString("nvram = \"/usr/share/OVMF/constellation_vars.testing.fd\"\n")
default:
writeLinef(b, "nvram = %q", v.NVRAM)
}
if v.Firmware != "" {
writeLinef(b, "firmware = %q", v.Firmware)
}
return b.String() return b.String()
} }

View File

@ -14,7 +14,7 @@ const (
SSHCheckInterval = 30 * time.Second SSHCheckInterval = 30 * time.Second
DiscoverDebugdInterval = 30 * time.Second DiscoverDebugdInterval = 30 * time.Second
BootstrapperDownloadRetryBackoff = 1 * time.Minute BootstrapperDownloadRetryBackoff = 1 * time.Minute
BootstrapperDeployFilename = "/opt/bootstrapper" BootstrapperDeployFilename = "/run/state/bin/bootstrapper"
Chunksize = 1024 Chunksize = 1024
BootstrapperSystemdUnitName = "bootstrapper.service" BootstrapperSystemdUnitName = "bootstrapper.service"
BootstrapperSystemdUnitContents = `[Unit] BootstrapperSystemdUnitContents = `[Unit]
@ -25,12 +25,10 @@ After=network-online.target
Type=simple Type=simple
RemainAfterExit=yes RemainAfterExit=yes
Restart=on-failure Restart=on-failure
EnvironmentFile=/etc/constellation.env EnvironmentFile=/run/constellation.env
Environment=PATH=/run/state/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
ExecStartPre=-setenforce Permissive ExecStartPre=-setenforce Permissive
ExecStartPre=/usr/bin/mkdir -p /opt/cni/bin/ ExecStart=/run/state/bin/bootstrapper
# merge all CNI binaries in writable folder until containerd can use multiple CNI bins: https://github.com/containerd/containerd/issues/6600
ExecStartPre=/bin/sh -c "/usr/bin/cp /usr/libexec/cni/* /opt/cni/bin/"
ExecStart=/opt/bootstrapper
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target
` `

View File

@ -36,7 +36,7 @@ func TestMain(m *testing.M) {
} }
func TestDownloadBootstrapper(t *testing.T) { func TestDownloadBootstrapper(t *testing.T) {
filename := "/opt/bootstrapper" filename := "/run/state/bin/bootstrapper"
someErr := errors.New("failed") someErr := errors.New("failed")
testCases := map[string]struct { testCases := map[string]struct {

View File

@ -18,7 +18,7 @@ import (
) )
const ( const (
systemdUnitFolder = "/etc/systemd/system" systemdUnitFolder = "/run/systemd/system"
) )
//go:generate stringer -type=SystemdAction //go:generate stringer -type=SystemdAction

View File

@ -188,7 +188,7 @@ func TestUploadBootstrapper(t *testing.T) {
assert.Equal(tc.wantResponseStatus, resp.Status) assert.Equal(tc.wantResponseStatus, resp.Status)
if tc.wantFile { if tc.wantFile {
assert.Equal(tc.wantChunks, tc.streamer.writeStreamChunks) assert.Equal(tc.wantChunks, tc.streamer.writeStreamChunks)
assert.Equal("/opt/bootstrapper", tc.streamer.writeStreamFilename) assert.Equal("/run/state/bin/bootstrapper", tc.streamer.writeStreamFilename)
} else { } else {
assert.Empty(tc.streamer.writeStreamChunks) assert.Empty(tc.streamer.writeStreamChunks)
assert.Empty(tc.streamer.writeStreamFilename) assert.Empty(tc.streamer.writeStreamFilename)
@ -256,7 +256,7 @@ func TestDownloadBootstrapper(t *testing.T) {
} }
require.NoError(err) require.NoError(err)
assert.Equal(tc.wantChunks, chunks) assert.Equal(tc.wantChunks, chunks)
assert.Equal("/opt/bootstrapper", tc.streamer.readStreamFilename) assert.Equal("/run/state/bin/bootstrapper", tc.streamer.readStreamFilename)
}) })
} }
} }

View File

@ -1,113 +1,193 @@
# Constellation images
We use the [Fedora CoreOS Assembler](https://coreos.github.io/coreos-assembler/) to build the base image for Constellation nodes.
## Setup ## Setup
1. Install prerequisites: - Install mkosi (from git):
- [Docker](https://docs.docker.com/engine/install/) or [Podman](https://podman.io/getting-started/installation)
- [Azure CLI](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli-linux)
- [azcopy](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10)
- [Google Cloud CLI](https://cloud.google.com/sdk/docs/install)
- [gsutil](https://cloud.google.com/storage/docs/gsutil_install#linux)
- Ubuntu:
```shell-session ```sh
sudo apt install -y bash coreutils cryptsetup-bin grep libguestfs-tools make parted pv qemu-system qemu-utils sed tar util-linux wget cd /tmp/
``` git clone https://github.com/systemd/mkosi
cd mkosi
2. Log in to GCP and Azure tools/generate-zipapp.sh
cp builddir/mkosi /usr/local/bin/
```shell-session
gcloud auth login
az login
```
3. [Log in to the ghcr.io package registry](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry#authenticating-to-the-container-registry)
4. Ensure read and write access to `/dev/kvm` (and repeat after every reboot)
```shell-session
sudo chmod 666 /dev/kvm
``` ```
## Configuration - Install tools:
Create a configuration file in `image/config.mk` to override any of the variables found at the top of the [Makefile](Makefile). <details>
Important settings are: <summary>Ubuntu / Debian</summary>
- `BOOTSTRAPPER_BINARY`: path to a bootstrapper binary. Can be substituted with a path to a `debugd` binary if a debug image should be built. The binary has to be built before! ```sh
- `CONTAINER_ENGINE`: container engine used to run COSA. either `podman` or `docker`. sudo apt-get update
- `COSA_INIT_REPO`: Git repository containing CoreOS config. Cloned in `cosa-init` target. sudo apt-get install --assume-yes --no-install-recommends \
- `COSA_INIT_BRANCH`: Git branch checked out from `COSA_INIT_REPO`. Can be used to test out changes on another branch before merging. dnf \
- `NETRC` path to a netrc file containing a GitHub PAT. Used to authenticate to GitHub from within the COSA container. systemd-container \
- `GCP_IMAGE_NAME`: Image name for the GCP image. Set to include a timestamp when using the build pipeline. Can be set to a custom value if you want to upload a custom image for testing on GCP. qemu-system-x86 \
- `AZURE_IMAGE_NAME`: Image name for the Azure image. Can be set to a custom value if you want to upload a custom image for testing on Azure. qemu-utils \
ovmf \
e2fsprogs \
squashfs-tools \
efitools \
sbsigntool \
coreutils \
curl \
jq \
util-linux \
virt-manager
```
Example `config.mk` to create a debug image with docker and name it `my-custom-image`: </details>
```Makefile <details>
BOOTSTRAPPER_BINARY = ../build/debugd <summary>Fedora</summary>
CONTAINER_ENGINE = docker
GCP_IMAGE_NAME = my-custom-image ```sh
AZURE_IMAGE_NAME = my-custom-image sudo dnf install -y \
edk2-ovmf \
systemd-container \
qemu \
e2fsprogs \
squashfs-tools \
efitools \
sbsigntools \
coreutils \
curl \
jq \
util-linux \
virt-manager
```
</details>
- Prepare secure boot PKI (see `secure-boot/genkeys.sh`)
## Build
When building your first image, prepare the secure boot PKI (see `secure-boot/genkeys.sh`) for self-signed, locally built images.
After that, you can build the image with:
```sh
# OPTIONAL: to create a debug image, export the following line
# export BOOTSTRAPPER_BINARY=$(realpath ${PWD}/../../build/debugd)
# OPTIONAL: symlink custom path to secure boot PKI to ./pki
# ln -s /path/to/pki/folder ./pki
sudo make -j $(nproc)
``` ```
## Build an image Raw images will be placed in `mkosi.output.<CSP>/fedora~36/image.raw`.
Ensure you have the modified cosa container image installed: ## Prepare Secure Boot
```shell-session The generated images are partially signed by Microsoft ([shim loader](https://github.com/rhboot/shim)), and partially signed by Edgeless Systems (systemd-boot and unified kernel images consisting of the linux kernel, initramfs and kernel commandline).
docker image ls | grep localhost/coreos-assembler
For QEMU and Azure, you can pre-generate the NVRAM variables for secure boot. This is not necessary for GCP, as you can specify secure boot parameters via the GCP API on image creation.
<details>
<summary>libvirt / QEMU / KVM</summary>
```sh
secure-boot/generate_nvram_vars.sh mkosi.output.qemu/fedora~36/image.raw
``` ```
or </details>
```shell-session <details>
podman image ls | grep localhost/coreos-assembler <summary><a id="azure-secure-boot">Azure</a></summary>
These steps only have to performed once for a fresh set of secure boot certificates.
VMGS blobs for testing and release images already exist.
First, create a disk without embedded MOK EFI variables.
```sh
# set these variables
export AZURE_SECURITY_TYPE=ConfidentialVM # or TrustedLaunch
export AZURE_RESOURCE_GROUP_NAME= # e.g. "constellation-images"
export AZURE_REGION=northeurope
export AZURE_DISK_NAME=constellation-$(date +%s)
export AZURE_SNAPSHOT_NAME=${AZURE_DISK_NAME}
export AZURE_RAW_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.raw
export AZURE_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.vhd
export AZURE_VMGS_FILENAME=${AZURE_SECURITY_TYPE}.vmgs
export BLOBS_DIR=${PWD}/blobs
upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
upload/upload_azure.sh --disk-name "${AZURE_DISK_NAME}-setup-secure-boot" ""
secure-boot/azure/launch.sh -n "${AZURE_DISK_NAME}-setup-secure-boot" -d --secure-boot true --disk-name "${AZURE_DISK_NAME}-setup-secure-boot"
``` ```
If not present, install with Ignore the running launch script and connect to the serial console once available.
The console shows the message "Verification failed: (0x1A) Security Violation". You can import the MOK certificate via the UEFI shell:
```shell-session Press OK, then ENTER, then "Enroll key from disk".
make cosa-image Select the following key: `/EFI/loader/keys/auto/db.cer`.
Press Continue, then choose "Yes" to the question "Enroll the key(s)?".
Choose reboot.
Extract the VMGS from the running VM (this includes the MOK EFI variables) and delete the VM:
```sh
secure-boot/azure/extract_vmgs.sh --name "${AZURE_DISK_NAME}-setup-secure-boot"
secure-boot/azure/delete.sh --name "${AZURE_DISK_NAME}-setup-secure-boot"
``` ```
> It is always advisable to create an image from a clean `build` dir. </details>
Clean up the `build` dir and remove old images (⚠ this will undo any local changes to the CoreOS configuration!): ## Upload to CSP
```shell-session <details>
sudo make clean <summary>GCP</summary>
- Install `gcloud` and `gsutil` (see [here](https://cloud.google.com/sdk/docs/install))
- Login to GCP (see [here](https://cloud.google.com/sdk/docs/authorizing))
- Choose secure boot PKI public keys (one of `pki_dev`, `pki_test`, `pki_prod`)
- `pki_dev` can be used for local image builds
- `pki_test` is used by the CI for non-release images
- `pki_prod` is used for release images
```sh
# set these variables
export GCP_IMAGE_FAMILY= # e.g. "constellation"
export GCP_IMAGE_NAME= # e.g. "constellation-v1.0.0"
export PKI=${PWD}/pki
export GCP_PROJECT=constellation-images
export GCP_REGION=europe-west3
export GCP_BUCKET=constellation-images
export GCP_RAW_IMAGE_PATH=${PWD}/mkosi.output.gcp/fedora~36/image.raw
export GCP_IMAGE_FILENAME=$(date +%s).tar.gz
export GCP_IMAGE_PATH=${PWD}/mkosi.output.gcp/fedora~36/image.tar.gz
upload/pack.sh gcp ${GCP_RAW_IMAGE_PATH} ${GCP_IMAGE_PATH}
upload/upload_gcp.sh
``` ```
- Build QEMU image (for local testing only) </details>
```shell-session <details>
make coreos <summary>Azure</summary>
```
- Build Azure image (without upload) - Install `az` and `azcopy` (see [here](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli))
- Login to Azure (see [here](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli))
- [Prepare virtual machine guest state (VMGS) with customized NVRAM or use existing VMGS blob](#azure-secure-boot)
```shell-session ```sh
make image-azure # set these variables
``` export AZURE_GALLERY_NAME= # e.g. "Constellation"
export AZURE_IMAGE_DEFINITION= # e.g. "constellation"
export AZURE_IMAGE_VERSION= # e.g. "1.0.0"
export AZURE_VMGS_PATH= # e.g. "path/to/ConfidentialVM.vmgs"
export AZURE_SECURITY_TYPE=ConfidentialVM # or TrustedLaunch
- Build Azure image (with upload) export AZURE_RESOURCE_GROUP_NAME=constellation-images
export AZURE_REGION=northeurope
export AZURE_REPLICATION_REGIONS="northeurope eastus westeurope westus"
export AZURE_IMAGE_OFFER=constellation
export AZURE_SKU=constellation
export AZURE_PUBLISHER=edgelesssys
export AZURE_DISK_NAME=constellation-$(date +%s)
export AZURE_RAW_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.raw
export AZURE_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.vhd
upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
upload/upload_azure.sh -g --disk-name "${AZURE_DISK_NAME}" "${AZURE_VMGS_PATH}"
```
```shell-session </details>
make image-azure upload-azure
```
- Build GCP image (without upload)
```shell-session
make image-gcp
```
- Build GCP image (with upload)
```shell-session
make image-gcp upload-gcp
```
Resulting images for the CSPs can be found under [images](images/). QEMU images are stored at `build/builds/latest/` with a name ending in `.qcow2`.

0
image/mkosi/.gitattributes vendored Normal file
View File

5
image/mkosi/.gitignore vendored Normal file
View File

@ -0,0 +1,5 @@
mkosi.cache
mkosi.extra
pki
image.*
mkosi.output.*

54
image/mkosi/Makefile Normal file
View File

@ -0,0 +1,54 @@
SHELL = /bin/bash
SRC_PATH = $(CURDIR)
BASE_PATH ?= $(SRC_PATH)
BOOTSTRAPPER_BINARY ?= $(BASE_PATH)/../../build/bootstrapper
DISK_MAPPER_BINARY ?= $(BASE_PATH)/../../build/disk-mapper
PKI ?= $(BASE_PATH)/pki
MKOSI_EXTRA ?= $(BASE_PATH)/mkosi.extra
-include $(CURDIR)/config.mk
csps := qemu gcp azure
certs := $(PKI)/PK.cer $(PKI)/KEK.cer $(PKI)/db.cer
.PHONY: all clean inject-bins $(csps)
all: $(csps)
$(csps): %: mkosi.output.%/fedora~36/image.raw
mkosi.output.%/fedora~36/image.raw: mkosi.files/mkosi.%.conf inject-bins inject-certs
mkosi --config mkosi.files/mkosi.$*.conf build
secure-boot/signed-shim.sh $@
@if [ -n $(SUDO_UID) ] && [ -n $(SUDO_GID) ]; then \
chown -R $(SUDO_UID):$(SUDO_GID) mkosi.output.$*; \
fi
@echo "Image is ready: $@"
inject-bins:
mkdir -p $(MKOSI_EXTRA)/usr/bin
mkdir -p $(MKOSI_EXTRA)/usr/sbin
cp $(BOOTSTRAPPER_BINARY) $(MKOSI_EXTRA)/usr/bin/bootstrapper
cp $(DISK_MAPPER_BINARY) $(MKOSI_EXTRA)/usr/sbin/disk-mapper
inject-certs: $(certs)
# for auto enrollment using systemd-boot (not working yet)
mkdir -p "$(MKOSI_EXTRA)/boot/loader/keys/auto"
cp $(PKI)/{PK,KEK,db}.cer "$(MKOSI_EXTRA)/boot/loader/keys/auto"
cp $(PKI)/{MicWinProPCA2011_2011-10-19,MicCorUEFCA2011_2011-06-27,MicCorKEKCA2011_2011-06-24}.crt "$(MKOSI_EXTRA)/boot/loader/keys/auto"
cp $(PKI)/{PK,KEK,db}.esl "$(MKOSI_EXTRA)/boot/loader/keys/auto"
cp $(PKI)/{PK,KEK,db}.auth "$(MKOSI_EXTRA)/boot/loader/keys/auto"
# for manual enrollment using sbkeysync
mkdir -p $(MKOSI_EXTRA)/etc/secureboot/keys/{db,dbx,KEK,PK}
cp $(PKI)/db.auth "$(MKOSI_EXTRA)/etc/secureboot/keys/db/"
cp $(PKI)/KEK.auth "$(MKOSI_EXTRA)/etc/secureboot/keys/KEK/"
cp $(PKI)/PK.auth "$(MKOSI_EXTRA)/etc/secureboot/keys/PK/"
clean-cache:
rm -rf mkosi.cache/*
clean-%:
mkosi --config mkosi.files/mkosi.$*.conf clean
clean:
rm -rf mkosi.output.*
rm -rf $(MKOSI_EXTRA)
mkdir -p $(MKOSI_EXTRA)

187
image/mkosi/README.md Normal file
View File

@ -0,0 +1,187 @@
## Setup
- Install mkosi (from git):
```sh
cd /tmp/
git clone https://github.com/systemd/mkosi
cd mkosi
tools/generate-zipapp.sh
cp builddir/mkosi /usr/local/bin/
```
- Install tools:
<details>
<summary>Ubuntu / Debian</summary>
```sh
sudo apt-get update
sudo apt-get install --assume-yes --no-install-recommends \
dnf \
systemd-container \
qemu-system-x86 \
qemu-utils \
ovmf \
e2fsprogs \
squashfs-tools \
efitools \
sbsigntool \
coreutils \
curl \
jq \
util-linux \
virt-manager
```
</details>
<details>
<summary>Fedora</summary>
```sh
sudo dnf install -y \
edk2-ovmf \
systemd-container \
qemu \
e2fsprogs \
squashfs-tools \
efitools \
sbsigntools \
coreutils \
curl \
jq \
util-linux \
virt-manager
```
</details>
- Prepare secure boot PKI (see `secure-boot/genkeys.sh`)
## Build
```sh
# OPTIONAL: to create a debug image, export the following line
# export BOOTSTRAPPER_BINARY=$(realpath ${PWD}/../../build/debugd)
# OPTIONAL: specify path to secure boot PKI
# export PKI=/path/to/pki/folder
sudo make -j $(nproc)
```
Raw images will be placed in `mkosi.output.<CSP>/fedora~36/image.raw`.
## Prepare Secure Boot
The generated images are partially signed by Microsoft ([shim loader](https://github.com/rhboot/shim)), and partially signed by Edgeless Systems (systemd-boot and unified kernel images consisting of the linux kernel, initramfs and kernel commandline).
For QEMU and Azure, you can pre-generate the NVRAM variables for secure boot. This is not necessary for GCP, as you can specify secure boot parameters via the GCP API on image creation.
<details>
<summary>libvirt / QEMU / KVM</summary>
```sh
secure-boot/generate_nvram_vars.sh mkosi.output.qemu/fedora~36/image.raw
```
</details>
<details>
<summary><a id="azure-secure-boot">Azure</a></summary>
These steps only have to performed once for a fresh set of secure boot certificates.
VMGS blobs for testing and release images already exist.
First, create a disk without embedded MOK EFI variables.
```sh
# set these variables
export AZURE_SECURITY_TYPE=ConfidentialVM # or TrustedLaunch
export AZURE_RESOURCE_GROUP_NAME= # e.g. "constellation-images"
export AZURE_REGION=northeurope
export AZURE_DISK_NAME=constellation-$(date +%s)
export AZURE_SNAPSHOT_NAME=${AZURE_DISK_NAME}
export AZURE_RAW_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.raw
export AZURE_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.vhd
export AZURE_VMGS_FILENAME=${AZURE_SECURITY_TYPE}.vmgs
export BLOBS_DIR=${PWD}/blobs
upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
upload/upload_azure.sh --disk-name "${AZURE_DISK_NAME}-setup-secure-boot" ""
secure-boot/azure/launch.sh -n "${AZURE_DISK_NAME}-setup-secure-boot" -d --secure-boot true --disk-name "${AZURE_DISK_NAME}-setup-secure-boot"
```
Ignore the running launch script and connect to the serial console once available.
The console shows the message "Verification failed: (0x1A) Security Violation". You can import the MOK certificate via the UEFI shell:
Press OK, then ENTER, then "Enroll key from disk".
Select the following key: `/EFI/loader/keys/auto/db.cer`.
Press Continue, then choose "Yes" to the question "Enroll the key(s)?".
Choose reboot.
Extract the VMGS from the running VM (this includes the MOK EFI variables) and delete the VM:
```sh
secure-boot/azure/extract_vmgs.sh --name "${AZURE_DISK_NAME}-setup-secure-boot"
secure-boot/azure/delete.sh --name "${AZURE_DISK_NAME}-setup-secure-boot"
```
</details>
## Upload to CSP
<details>
<summary>GCP</summary>
- Install `gcloud` and `gsutil` (see [here](https://cloud.google.com/sdk/docs/install))
- Login to GCP (see [here](https://cloud.google.com/sdk/docs/authorizing))
- Prepare secure boot PKI (see `secure-boot/genkeys.sh`)
```sh
# set these variables
export GCP_IMAGE_FAMILY= # e.g. "constellation"
export GCP_IMAGE_NAME= # e.g. "constellation-v1.0.0"
export PKI=${PWD}/pki
export GCP_PROJECT=constellation-images
export GCP_REGION=europe-west3
export GCP_BUCKET=constellation-images
export GCP_RAW_IMAGE_PATH=${PWD}/mkosi.output.gcp/fedora~36/image.raw
export GCP_IMAGE_FILENAME=$(date +%s).tar.gz
export GCP_IMAGE_PATH=${PWD}/mkosi.output.gcp/fedora~36/image.tar.gz
upload/pack.sh gcp ${GCP_RAW_IMAGE_PATH} ${GCP_IMAGE_PATH}
upload/upload_gcp.sh
```
</details>
<details>
<summary>Azure</summary>
- Install `az` and `azcopy` (see [here](https://docs.microsoft.com/en-us/cli/azure/install-azure-cli))
- Login to Azure (see [here](https://docs.microsoft.com/en-us/cli/azure/authenticate-azure-cli))
- Prepare secure boot PKI (see `secure-boot/genkeys.sh`)
- [Prepare virtual machine guest state (VMGS) with customized NVRAM or use existing VMGS blob](#azure-secure-boot)
```sh
# set these variables
export AZURE_GALLERY_NAME= # e.g. "Constellation"
export AZURE_IMAGE_DEFINITION= # e.g. "constellation"
export AZURE_IMAGE_VERSION= # e.g. "1.0.0"
export AZURE_VMGS_PATH= # e.g. "path/to/ConfidentialVM.vmgs"
export AZURE_SECURITY_TYPE=ConfidentialVM # or TrustedLaunch
export AZURE_RESOURCE_GROUP_NAME=constellation-images
export AZURE_REGION=northeurope
export AZURE_REPLICATION_REGIONS="northeurope eastus westeurope westus"
export AZURE_IMAGE_OFFER=constellation
export AZURE_SKU=constellation
export AZURE_PUBLISHER=edgelesssys
export AZURE_DISK_NAME=constellation-$(date +%s)
export AZURE_RAW_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.raw
export AZURE_IMAGE_PATH=${PWD}/mkosi.output.azure/fedora~36/image.vhd
upload/pack.sh azure "${AZURE_RAW_IMAGE_PATH}" "${AZURE_IMAGE_PATH}"
upload/upload_azure.sh -g --disk-name "${AZURE_DISK_NAME}" "${AZURE_VMGS_PATH}"
```
</details>

View File

View File

@ -0,0 +1,3 @@
[Content]
Packages=
WALinuxAgent-udev

View File

@ -0,0 +1,9 @@
[Content]
Packages=
containerd,
containernetworking-plugins,
iptables-nft,
ethtool,
socat,
iproute-tc,
conntrack-tools

View File

@ -0,0 +1,3 @@
[Content]
Packages=
nvme-cli

View File

@ -0,0 +1,22 @@
[Distribution]
Distribution=fedora
Release=36
[Output]
Format=gpt_squashfs
ManifestFormat=json,changelog
Bootable=yes
KernelCommandLine=mitigations=auto,nosmt preempt=full
WithUnifiedKernelImages=yes
Verity=yes
CompressFs=zstd
SplitArtifacts=yes
# Enable Secure Boot with own PKI
SecureBoot=yes
SecureBootKey=pki/db.key
SecureBootCertificate=pki/db.crt
# TODO: Wait for systemd 252 to bring systemd-measure
# Measure=yes
[Host]
QemuHeadless=yes

View File

@ -0,0 +1,8 @@
[Content]
Packages=
iproute,
dbus-broker,
systemd-networkd,
systemd-resolved,
dracut-network,
dhclient, # prevent NetworkManager from being pulled in by dracut-network

View File

@ -0,0 +1,8 @@
[Content]
# Secure Boot / EFI related packages for manual enrollment / verification of Secure Boot
Packages=
e2fsprogs,
sbsigntools,
efitools,
mokutil,
tpm2-tools

View File

@ -0,0 +1,8 @@
[Content]
Packages=
passwd,
nano,
nano-default-editor,
vim,
curl,
wget

View File

@ -0,0 +1,3 @@
[Output]
KernelCommandLine=constel.csp=azure
OutputDirectory=mkosi.output.azure

View File

@ -0,0 +1,3 @@
[Output]
KernelCommandLine=constel.csp=gcp
OutputDirectory=mkosi.output.gcp

View File

@ -0,0 +1,3 @@
[Output]
KernelCommandLine=constel.csp=qemu
OutputDirectory=mkosi.output.qemu

12
image/mkosi/mkosi.finalize Executable file
View File

@ -0,0 +1,12 @@
#!/usr/bin/env bash
set -euxo pipefail
# recreate kubelet systemd unit after reboot.
# tmpfile config has to be written late as it interferes with the systemd-nspawn build environment
cat >"${BUILDROOT}/usr/lib/tmpfiles.d/kubelet-service.conf" <<EOF
C /run/systemd/system/kubelet.service - - - - /run/state/systemd/system/kubelet.service
C /run/systemd/system/kubelet.service.d/10-kubeadm.conf - - - - /run/state/systemd/system/kubelet.service.d/10-kubeadm.conf
EOF
# cleanup dracut generation files (disk-mapper) to save space
rm -rf "${BUILDROOT}/usr/lib/dracut/modules.d/39constellation-mount/"

22
image/mkosi/mkosi.postinst Executable file
View File

@ -0,0 +1,22 @@
#!/bin/sh
set -euxo pipefail
# This will work in sd-boot 251 to auto-enroll secure boot keys.
# https://www.freedesktop.org/software/systemd/man/systemd-boot.html
# > CHANGES WITH 252 in spe:
# > [...]
# > * sd-boot can automatically enroll SecureBoot keys from files found on
# > the ESP. This enrollment can be either automatic ('force' mode) or
# > controlled by the user ('manual' mode).
# > [...]
#
# echo "secure-boot-enroll force" >> /boot/loader/loader.conf
# create mountpoints in /etc
mkdir -p /etc/{cni,kubernetes}
# move issue files away from /etc
# to allow /run/issue and /run/issue.d to take precedence
mv /etc/issue.d /usr/lib/issue.d || true
rm -f /etc/issue
rm -f /etc/issue.net

View File

@ -0,0 +1,4 @@
# enable networking in initrd (initramfs) with dracut and systemd-networkd
install_items+=" /usr/lib/systemd/network/20-wired.network "
install_items+=" /usr/lib/systemd/network/21-azure.network "
add_dracutmodules+=" systemd-networkd "

View File

@ -0,0 +1,3 @@
# add hyperv drivers to initramfs
# (important for early networking)
force_drivers+=" hv_netvsc hv_sock hv_storvsc hv_vmbus "

View File

@ -0,0 +1,2 @@
# Include NVMe driver in initrd to boot on NVMe devices.
force_drivers+=" nvme "

View File

@ -0,0 +1,5 @@
/dev/mapper/state /run/state ext4 defaults,x-systemd.makefs,x-mount.mkdir 0 0
/run/state/var /var none defaults,bind,x-mount.mkdir 0 0
/run/state/kubernetes /etc/kubernetes none defaults,bind,x-mount.mkdir 0 0
/run/state/etccni /etc/cni/ none defaults,bind,x-mount.mkdir 0 0
/run/state/opt /opt none defaults,bind,x-mount.mkdir 0 0

View File

@ -0,0 +1,4 @@
#!/usr/bin/env bash
export PATH=/run/state/bin:${PATH}
export KUBECONFIG=/etc/kubernetes/admin.conf
alias k=kubectl

View File

@ -0,0 +1,216 @@
disabled_plugins = []
imports = []
oom_score = 0
plugin_dir = ""
required_plugins = []
root = "/var/lib/containerd"
state = "/run/containerd"
version = 2
[cgroup]
path = ""
[debug]
address = ""
format = ""
gid = 0
level = ""
uid = 0
[grpc]
address = "/run/containerd/containerd.sock"
gid = 0
max_recv_message_size = 16777216
max_send_message_size = 16777216
tcp_address = ""
tcp_tls_cert = ""
tcp_tls_key = ""
uid = 0
[metrics]
address = ""
grpc_histogram = false
[plugins]
[plugins."io.containerd.gc.v1.scheduler"]
deletion_threshold = 0
mutation_threshold = 100
pause_threshold = 0.02
schedule_delay = "0s"
startup_delay = "100ms"
[plugins."io.containerd.grpc.v1.cri"]
disable_apparmor = false
disable_cgroup = false
disable_hugetlb_controller = true
disable_proc_mount = false
disable_tcp_service = true
enable_selinux = false
enable_tls_streaming = false
ignore_image_defined_volumes = false
max_concurrent_downloads = 3
max_container_log_line_size = 16384
netns_mounts_under_state_dir = false
restrict_oom_score_adj = false
sandbox_image = "k8s.gcr.io/pause:3.5"
selinux_category_range = 1024
stats_collect_period = 10
stream_idle_timeout = "4h0m0s"
stream_server_address = "127.0.0.1"
stream_server_port = "0"
systemd_cgroup = false
tolerate_missing_hugetlb_controller = true
unset_seccomp_profile = ""
[plugins."io.containerd.grpc.v1.cri".cni]
bin_dir = "/opt/cni/bin"
conf_dir = "/etc/cni/net.d"
conf_template = ""
max_conf_num = 1
[plugins."io.containerd.grpc.v1.cri".containerd]
default_runtime_name = "runc"
disable_snapshot_annotations = true
discard_unpacked_layers = false
no_pivot = false
snapshotter = "overlayfs"
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
base_runtime_spec = ""
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
base_runtime_spec = ""
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_root = ""
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
BinaryName = ""
CriuImagePath = ""
CriuPath = ""
CriuWorkPath = ""
IoGid = 0
IoUid = 0
NoNewKeyring = false
NoPivotRoot = false
Root = ""
ShimCgroup = ""
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
base_runtime_spec = ""
container_annotations = []
pod_annotations = []
privileged_without_host_devices = false
runtime_engine = ""
runtime_root = ""
runtime_type = ""
[plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options]
[plugins."io.containerd.grpc.v1.cri".image_decryption]
key_model = "node"
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = ""
[plugins."io.containerd.grpc.v1.cri".registry.auths]
[plugins."io.containerd.grpc.v1.cri".registry.configs]
[plugins."io.containerd.grpc.v1.cri".registry.headers]
[plugins."io.containerd.grpc.v1.cri".registry.mirrors]
[plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
tls_cert_file = ""
tls_key_file = ""
[plugins."io.containerd.internal.v1.opt"]
path = "/opt/containerd"
[plugins."io.containerd.internal.v1.restart"]
interval = "10s"
[plugins."io.containerd.metadata.v1.bolt"]
content_sharing_policy = "shared"
[plugins."io.containerd.monitor.v1.cgroups"]
no_prometheus = false
[plugins."io.containerd.runtime.v1.linux"]
no_shim = false
runtime = "runc"
runtime_root = ""
shim = "containerd-shim"
shim_debug = false
[plugins."io.containerd.runtime.v2.task"]
platforms = ["linux/amd64"]
[plugins."io.containerd.service.v1.diff-service"]
default = ["walking"]
[plugins."io.containerd.snapshotter.v1.aufs"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.btrfs"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.devmapper"]
async_remove = false
base_image_size = ""
pool_name = ""
root_path = ""
[plugins."io.containerd.snapshotter.v1.native"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.overlayfs"]
root_path = ""
[plugins."io.containerd.snapshotter.v1.zfs"]
root_path = ""
[proxy_plugins]
[stream_processors]
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar"]
accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"]
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar"
[stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"]
accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"]
args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"]
env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"]
path = "ctd-decoder"
returns = "application/vnd.oci.image.layer.v1.tar+gzip"
[timeouts]
"io.containerd.timeout.shim.cleanup" = "5s"
"io.containerd.timeout.shim.load" = "5s"
"io.containerd.timeout.shim.shutdown" = "3s"
"io.containerd.timeout.task.state" = "2s"
[ttrpc]
address = ""
gid = 0
uid = 0

View File

@ -0,0 +1 @@
../../../systemd/system/configure-constel-csp.service

View File

@ -0,0 +1,15 @@
[Unit]
Description=Force symlink creation for GCP nvme disks
Before=prepare-state-disk.service
After=network-online.target
Wants=network-online.target
ConditionKernelCommandLine=constel.csp=gcp
[Service]
Type=oneshot
ExecStart=/bin/bash /usr/sbin/google-nvme-disk
RemainAfterExit=yes
StandardOutput=tty
StandardInput=tty
StandardError=tty
TimeoutSec=infinity

View File

@ -0,0 +1,23 @@
#!/bin/bash
set -euo pipefail
shopt -s extglob nullglob
GCP_STATE_DISK_SYMLINK="/dev/disk/by-id/google-state-disk"
# hack: google nvme udev rules are never executed. Create symlinks for the nvme devices manually.
while [ ! -L "${GCP_STATE_DISK_SYMLINK}" ]
do
for nvmedisk in /dev/nvme0n+([0-9])
do
/usr/lib/udev/google_nvme_id -s -d "${nvmedisk}" || true
done
if [ -L "${GCP_STATE_DISK_SYMLINK}" ]; then
break
fi
echo "Waiting for state disk to appear.."
sleep 2
done
echo "Google state disk found"
echo ${GCP_STATE_DISK_SYMLINK}$(readlink -f "${GCP_STATE_DISK_SYMLINK}")
sleep 2

View File

@ -0,0 +1,56 @@
#!/bin/bash
depends() {
echo systemd
}
install_and_enable_unit() {
unit="$1"; shift
target="$1"; shift
inst_simple "$moddir/$unit" "$systemdsystemunitdir/$unit"
mkdir -p "${initdir}${systemdsystemconfdir}/${target}.wants"
ln_r "${systemdsystemunitdir}/${unit}" \
"${systemdsystemconfdir}/${target}.wants/${unit}"
}
install() {
inst_multiple \
bash
inst_script "/usr/sbin/disk-mapper" \
"/usr/sbin/disk-mapper"
inst_script "$moddir/prepare-state-disk.sh" \
"/usr/sbin/prepare-state-disk"
install_and_enable_unit "prepare-state-disk.service" \
"basic.target"
inst_script "$moddir/google-nvme-disk.sh" \
"/usr/sbin/google-nvme-disk"
install_and_enable_unit "google-nvme-disk.service" \
"basic.target"
install_and_enable_unit "configure-constel-csp.service" \
"basic.target"
# azure scsi disks
inst_multiple \
cut \
readlink
# gcp nvme disks
inst_multiple \
date \
xxd \
grep \
sed \
ln \
command \
readlink
inst_script "/usr/sbin/nvme" \
"/usr/sbin/nvme"
inst_script "/usr/lib/udev/google_nvme_id" \
"/usr/lib/udev/google_nvme_id"
inst_simple "/usr/lib/udev/rules.d/64-gce-disk-removal.rules" \
"/usr/lib/udev/rules.d/64-gce-disk-removal.rules"
inst_simple "/usr/lib/udev/rules.d/65-gce-disk-naming.rules" \
"/usr/lib/udev/rules.d/65-gce-disk-naming.rules"
}

View File

@ -0,0 +1,17 @@
[Unit]
Description=Prepare encrypted state disk
Before=initrd-fs.target
After=network-online.target configure-constel-csp.service
Wants=network-online.target
Requires=initrd-root-fs.target
FailureAction=reboot-immediate
[Service]
Type=oneshot
EnvironmentFile=/run/constellation.env
ExecStart=/bin/bash /usr/sbin/prepare-state-disk
RemainAfterExit=yes
StandardOutput=tty
StandardInput=tty
StandardError=tty
TimeoutSec=infinity

View File

@ -0,0 +1,11 @@
#!/bin/bash
set -euo pipefail
# Prepare the encrypted volume by either initializing it with a random key or by aquiring the key from another bootstrapper.
# Store encryption key (random or recovered key) in /run/cryptsetup-keys.d/state.key
disk-mapper -csp "${CONSTEL_CSP}"
if [[ $? -ne 0 ]]; then
echo "Failed to prepare state disk"
sleep 2 # give the serial console time to print the error message
exit $? # exit with the same error code as disk-mapper
fi

View File

@ -0,0 +1,2 @@
PATH=/run/state/bin:$PATH
KUBECONFIG=/etc/kubernetes/admin.conf

View File

@ -0,0 +1,2 @@
overlay
br_netfilter

View File

@ -0,0 +1,3 @@
# See https://github.com/cilium/cilium/issues/10645
net.ipv4.conf.lxc*.rp_filter = 0
net.ipv4.conf.cilium_*.rp_filter = 0

View File

@ -0,0 +1,9 @@
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
fs.inotify.max_user_instances = 8192
fs.inotify.max_user_watches = 524288
# kubernetes hardening (protectKernelDefaults=true)
vm.overcommit_memory = 1
kernel.panic = 10
kernel.panic_on_oops = 1

View File

@ -0,0 +1,5 @@
[Match]
Name=en*
[Network]
DHCP=yes

View File

@ -0,0 +1,6 @@
# Used as a fallback rule for Azure NICs as they are not named with "en*"
[Match]
Driver=hv_netvsc
[Network]
DHCP=yes

View File

@ -0,0 +1,5 @@
enable constellation-bootstrapper.service
enable configure-constel-csp.service
enable containerd.service
enable tpm-pcrs.service
enable systemd-networkd.service

View File

@ -0,0 +1,10 @@
[Unit]
Description=Configures constellation cloud service provider environment variable
[Service]
Type=oneshot
ExecStart=/bin/bash -c "CSP=$(< /proc/cmdline tr ' ' '\n' | grep constel.csp | sed 's/constel.csp=//'); echo CONSTEL_CSP=$CSP >> /run/constellation.env"
RemainAfterExit=yes
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,15 @@
[Unit]
Description=Constellation Bootstrapper
Wants=network-online.target
After=network-online.target configure-constel-csp.service
[Service]
Type=simple
RemainAfterExit=yes
Restart=on-failure
EnvironmentFile=/run/constellation.env
Environment=PATH=/run/state/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin
ExecStart=/usr/bin/bootstrapper
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,3 @@
[Service]
ExecStart=
ExecStart=/usr/bin/containerd --config /usr/etc/containerd/config.toml

View File

@ -0,0 +1,11 @@
[Unit]
Description=Print PCR state on startup
Before=constellation-bootstrapper.service
[Service]
Type=oneshot
RemainAfterExit=yes
ExecStart=/usr/libexec/constellation-pcrs
[Install]
WantedBy=multi-user.target

View File

@ -0,0 +1,2 @@
#Type Name ID GECOS Home directory Shell
u etcd 998:997 "etcd user" /var/lib/etcd

View File

@ -0,0 +1,8 @@
#Type Path Mode User Group Age Argument
d /var/lib/etcd 0700 998 997 - -
d /var/log/kubernetes/audit/ 0700 0 0 - -
d /run/state/bin 0755 0 0 - -
d /run/issue.d 0755 0 0 - -
C /run/issue - - - - /usr/lib/issue
# merge all CNI binaries in writable folder until containerd can use multiple CNI bins: https://github.com/containerd/containerd/issues/6600
C /opt/cni/bin - - - - /usr/libexec/cni/

View File

@ -0,0 +1,245 @@
#!/bin/bash
# Copyright 2020 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Used to generate symlinks for PD-NVMe devices using the disk names reported by
# the metadata server
# Locations of the script's dependencies
readonly nvme_cli_bin=/usr/sbin/nvme
# Bash regex to parse device paths and controller identification
readonly NAMESPACE_NUMBER_REGEX="/dev/nvme[[:digit:]]+n([[:digit:]]+).*"
readonly PARTITION_NUMBER_REGEX="/dev/nvme[[:digit:]]+n[[:digit:]]+p([[:digit:]]+)"
readonly PD_NVME_REGEX="sn[[:space:]]+:[[:space]]+nvme_card-pd"
# Globals used to generate the symlinks for a PD-NVMe disk. These are populated
# by the identify_pd_disk function and exported for consumption by udev rules.
ID_SERIAL=''
ID_SERIAL_SHORT=''
#######################################
# Helper function to log an error message to stderr.
# Globals:
# None
# Arguments:
# String to print as the log message
# Outputs:
# Writes error to STDERR
#######################################
function err() {
echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
}
#######################################
# Retrieves the device name for an NVMe namespace using nvme-cli.
# Globals:
# Uses nvme_cli_bin
# Arguments:
# The path to the nvme namespace (/dev/nvme0n?)
# Outputs:
# The device name parsed from the JSON in the vendor ext of the ns-id command.
# Returns:
# 0 if the device name for the namespace could be retrieved, 1 otherwise
#######################################
function get_namespace_device_name() {
local nvme_json
nvme_json="$("$nvme_cli_bin" id-ns -b "$1" | xxd -p -seek 384 | xxd -p -r)"
if [[ $? -ne 0 ]]; then
return 1
fi
if [[ -z "$nvme_json" ]]; then
err "NVMe Vendor Extension disk information not present"
return 1
fi
local device_name
device_name="$(echo "$nvme_json" | grep device_name | sed -e 's/.*"device_name":[ \t]*"\([a-zA-Z0-9_-]\+\)".*/\1/')"
# Error if our device name is empty
if [[ -z "$device_name" ]]; then
err "Empty name"
return 1
fi
echo "$device_name"
return 0
}
#######################################
# Retrieves the nsid for an NVMe namespace
# Globals:
# None
# Arguments:
# The path to the nvme namespace (/dev/nvme0n*)
# Outputs:
# The namespace number/id
# Returns:
# 0 if the namespace id could be retrieved, 1 otherwise
#######################################
function get_namespace_number() {
local dev_path="$1"
local namespace_number
if [[ "$dev_path" =~ $NAMESPACE_NUMBER_REGEX ]]; then
namespace_number="${BASH_REMATCH[1]}"
else
return 1
fi
echo "$namespace_number"
return 0
}
#######################################
# Retrieves the partition number for a device path if it exists
# Globals:
# None
# Arguments:
# The path to the device partition (/dev/nvme0n*p*)
# Outputs:
# The value after 'p' in the device path, or an empty string if the path has
# no partition.
#######################################
function get_partition_number() {
local dev_path="$1"
local partition_number
if [[ "$dev_path" =~ $PARTITION_NUMBER_REGEX ]]; then
partition_number="${BASH_REMATCH[1]}"
echo "$partition_number"
else
echo ''
fi
return 0
}
#######################################
# Generates a symlink for a PD-NVMe device using the metadata's disk name.
# Primarily used for testing but can be used if the script is directly invoked.
# Globals:
# Uses ID_SERIAL_SHORT (can be populated by identify_pd_disk)
# Arguments:
# The device path for the disk
#######################################
function gen_symlink() {
local dev_path="$1"
local partition_number="$(get_partition_number "$dev_path")"
if [[ -n "$partition_number" ]]; then
ln -s "$dev_path" /dev/disk/by-id/google-"$ID_SERIAL_SHORT"-part"$partition_number" > /dev/null 2>&1
else
ln -s "$dev_path" /dev/disk/by-id/google-"$ID_SERIAL_SHORT" > /dev/null 2>&1
fi
return 0
}
#######################################
# Populates the ID_* global variables with a disk's device name and namespace
# Globals:
# Populates ID_SERIAL_SHORT, and ID_SERIAL
# Arguments:
# The device path for the disk
# Returns:
# 0 on success and 1 if an error occurrs
#######################################
function identify_pd_disk() {
local dev_path="$1"
local dev_name
dev_name="$(get_namespace_device_name "$dev_path")"
if [[ $? -ne 0 ]]; then
return 1
fi
ID_SERIAL_SHORT="$dev_name"
ID_SERIAL="Google_PersistentDisk_${ID_SERIAL_SHORT}"
return 0
}
function print_help_message() {
echo "Usage: google_nvme_id [-s] [-h] -d device_path"
echo " -d <device_path> (Required): Specifies the path to generate a name"
echo " for. This needs to be a path to an nvme device or namespace"
echo " -s: Create symbolic link for the disk under /dev/disk/by-id."
echo " Otherwise, the disk name will be printed to STDOUT"
echo " -h: Print this help message"
}
function main() {
local opt_gen_symlink='false'
local device_path=''
while getopts :d:sh flag; do
case "$flag" in
d) device_path="$OPTARG";;
s) opt_gen_symlink='true';;
h) print_help_message
return 0
;;
:) echo "Invalid option: ${OPTARG} requires an argument" 1>&2
return 1
;;
*) return 1
esac
done
if [[ -z "$device_path" ]]; then
echo "Device path (-d) argument required. Use -h for full usage." 1>&2
exit 1
fi
# Ensure the nvme-cli command is installed
command -v "$nvme_cli_bin" > /dev/null 2>&1
if [[ $? -ne 0 ]]; then
err "The nvme utility (/usr/sbin/nvme) was not found. You may need to run \
with sudo or install nvme-cli."
return 1
fi
# Ensure the passed device is actually an NVMe device
"$nvme_cli_bin" id-ctrl "$device_path" &>/dev/null
if [[ $? -ne 0 ]]; then
err "Passed device was not an NVMe device. (You may need to run this \
script as root/with sudo)."
return 1
fi
# Detect the type of attached nvme device
local controller_id
controller_id=$("$nvme_cli_bin" id-ctrl "$device_path")
if [[ ! "$controller_id" =~ nvme_card-pd ]] ; then
err "Device is not a PD-NVMe device"
return 1
fi
# Fill the global variables for the id command for the given disk type
# Error messages will be printed closer to error, no need to reprint here
identify_pd_disk "$device_path"
if [[ $? -ne 0 ]]; then
return $?
fi
# Gen symlinks or print out the globals set by the identify command
if [[ "$opt_gen_symlink" == 'true' ]]; then
gen_symlink "$device_path"
else
# These will be consumed by udev
echo "ID_SERIAL_SHORT=${ID_SERIAL_SHORT}"
echo "ID_SERIAL=${ID_SERIAL}"
fi
return $?
}
main "$@"

View File

@ -0,0 +1,17 @@
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# When a disk is removed, unmount any remaining attached volumes.
ACTION=="remove", SUBSYSTEM=="block", KERNEL=="sd*|vd*|nvme*", RUN+="/bin/sh -c '/bin/umount -fl /dev/$name && /usr/bin/logger -p daemon.warn -s WARNING: hot-removed /dev/$name that was still mounted, data may have been corrupted'"

View File

@ -0,0 +1,37 @@
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Name the attached disks as the specified by deviceName.
ACTION!="add|change", GOTO="gce_disk_naming_end"
SUBSYSTEM!="block", GOTO="gce_disk_naming_end"
# SCSI naming
KERNEL=="sd*|vd*", IMPORT{program}="scsi_id --export --whitelisted -d $tempnode"
# NVME Local SSD naming
KERNEL=="nvme*n*", ATTRS{model}=="nvme_card", PROGRAM="/bin/sh -c 'nsid=$$(echo %k|sed -re s/nvme[0-9]+n\([0-9]+\).\*/\\1/); echo $$((nsid-1))'", ENV{ID_SERIAL_SHORT}="local-nvme-ssd-%c"
KERNEL=="nvme*", ATTRS{model}=="nvme_card", ENV{ID_SERIAL}="Google_EphemeralDisk_$env{ID_SERIAL_SHORT}"
# NVME Persistent Disk IO Timeout
KERNEL=="nvme*n*", ENV{DEVTYPE}=="disk", ATTRS{model}=="nvme_card-pd", ATTR{queue/io_timeout}="4294967295"
# NVME Persistent Disk Naming
KERNEL=="nvme*n*", ATTRS{model}=="nvme_card-pd", IMPORT{program}="google_nvme_id -d $tempnode"
# Symlinks
KERNEL=="sd*|vd*|nvme*", ENV{DEVTYPE}=="disk", SYMLINK+="disk/by-id/google-$env{ID_SERIAL_SHORT}"
KERNEL=="sd*|vd*|nvme*", ENV{DEVTYPE}=="partition", SYMLINK+="disk/by-id/google-$env{ID_SERIAL_SHORT}-part%n"
LABEL="gce_disk_naming_end"

View File

@ -0,0 +1,3 @@
# prevent systemd udev rules from marking unformatted device mapper device as unready (SYSTEMD_READY=0)
# this is the offending rule from systemd: SUBSYSTEM=="block", ENV{DM_UUID}=="CRYPT-*", ENV{ID_PART_TABLE_TYPE}=="", ENV{ID_FS_USAGE}=="", ENV{SYSTEMD_READY}="0"
SUBSYSTEM=="block", ENV{DM_NAME}=="state", ENV{DM_UUID}=="CRYPT-*", ENV{ID_PART_TABLE_TYPE}=="", ENV{ID_FS_USAGE}="constellation-state"

View File

@ -0,0 +1,10 @@
#!/usr/bin/bash
# This script reads the PCR state of the system
# and prints the message to the serial console
main() {
pcr_state="$(tpm2_pcrread sha256)"
echo -e "PCR state:\n${pcr_state}\n" > /run/issue.d/35_constellation_pcrs.issue
}
main

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1,23 @@
-----BEGIN CERTIFICATE-----
MIID3zCCAsegAwIBAgIUHii75K8+vo3LkCUKJjGBiTVJy/8wDQYJKoZIhvcNAQEL
BQAwgYgxCzAJBgNVBAYTAkRFMRwwGgYDVQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVu
MQ8wDQYDVQQHDAZCb2NodW0xHjAcBgNVBAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21i
SDEqMCgGA1UEAwwhQ29uc3RlbGxhdGlvbiBUZXN0aW5nIEtFSyBDQSAyMDIyMB4X
DTIyMDkyMzE0MTYwN1oXDTIyMTAyMzE0MTYwN1owgYgxCzAJBgNVBAYTAkRFMRww
GgYDVQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVuMQ8wDQYDVQQHDAZCb2NodW0xHjAc
BgNVBAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21iSDEqMCgGA1UEAwwhQ29uc3RlbGxh
dGlvbiBUZXN0aW5nIEtFSyBDQSAyMDIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
MIIBCgKCAQEA9QnVns2kVMrbV6308MDN6xVoTLcQ91ashmpj5Zj/gDiidtguggWS
hJ8bItzH2c02AWrMuf+T5wxaPWxZPjgYI9CnoBg53rgmGWpeBeV2ZWK8wRp0iUQe
GmhaBP+aA4h6UYN7N1120kV0O4BmJg76JwecHK2VXom7H4ILD4EKvXgpiyRdKJm7
CZaJ422G1LIjzLI1Gm2yhTsihyma9iguyV4EFLukp8umGneiE0erRngKw9a6YTWP
eFfTyXtwaWVcGhA5guhkp5U/KQCY1Mr6e4+Zp3ISL8QFBneYbDpVI3nWDNfWPMAH
eLxny2rpc+zvOJ+JDCc3frkWCLV4BZD8ZQIDAQABoz8wPTAdBgNVHQ4EFgQU8iGE
QDAs7qF7LowPBQu0Tft2fbowDwYDVR0TAQH/BAUwAwEB/zALBgNVHQ8EBAMCAYYw
DQYJKoZIhvcNAQELBQADggEBAKF9RGT46YAFhn2CrMuMd3eOalpoLM+SllAtmq5c
7RqUGQBOhuAdf1aiucFXH8xzi7DOV6aVhQG67kv5isISqUdUL80+RpajWYcU5YaW
jEX+w/o2Jv0kzBDBTVcX6uKuid1oiQDXGVL/UaU30Smdk/9ni1RImuPBPupNEljU
AjaduiqqcJArLBmXzEizCnaGhEvdezPIiZDbzARDbkvl1WQthcghh3i6iiBiN7Vp
gGMzEecGJ4oxlxa+fycIiFbrX8h3DgVOCXeaxWtbfmIuswwFeZ3rVXHXizfOMEdr
2qWu/xQwwUfN3Z07kXigfB3akJfqDYO5/mNGNNi6fAJV2do=
-----END CERTIFICATE-----

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1,23 @@
-----BEGIN CERTIFICATE-----
MIID4TCCAsmgAwIBAgIUUsz90g+sTYH+2QEMZf7gKWDiDEwwDQYJKoZIhvcNAQEL
BQAwgYkxCzAJBgNVBAYTAkRFMRwwGgYDVQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVu
MQ8wDQYDVQQHDAZCb2NodW0xHjAcBgNVBAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21i
SDErMCkGA1UEAwwiQ29uc3RlbGxhdGlvbiBUZXN0aW5nIFVFRkkgQ0EgMjAyMjAe
Fw0yMjA5MjMxNDE2MDZaFw0yMjEwMjMxNDE2MDZaMIGJMQswCQYDVQQGEwJERTEc
MBoGA1UECAwTTm9yZHJoZWluIFdlc3RmYWxlbjEPMA0GA1UEBwwGQm9jaHVtMR4w
HAYDVQQKDBVFZGdlbGVzcyBTeXN0ZW1zIEdtYkgxKzApBgNVBAMMIkNvbnN0ZWxs
YXRpb24gVGVzdGluZyBVRUZJIENBIDIwMjIwggEiMA0GCSqGSIb3DQEBAQUAA4IB
DwAwggEKAoIBAQC4fWmh6n3qjKmJSh0zvNc9frZnEcyfK3pDSCtdhNQh/kN59Tjd
+kg79yxPfG5v6MYiQhxva2EVPiAuMmB/zrhq75UmNU6o0WCVk58g7IrXDd4xpO+s
32+umlntT86wHWfrSAj3JduBG8ci1J1uARz0mmHl2gOzoB0dn4WOAoLLMH2bAzjS
ICX0giXiY6Q66JQya4OPMvdwSAW78H6JgTtNIsYKn2clnA3VJZWXIzYl6Bd92zjC
6RFrhzu7WYw9nmIA0HMtjpSeU9JDPUmedU7MPqLAK6kpiR+RrowzkfaFmIUqdxpj
4IlGXkqWqu/I3WzKucdt7X0Run914M5iM4xzAgMBAAGjPzA9MB0GA1UdDgQWBBQ9
/pmrY9gxhPV658NbhCfmcGWkDjAPBgNVHRMBAf8EBTADAQH/MAsGA1UdDwQEAwIB
hjANBgkqhkiG9w0BAQsFAAOCAQEAU+4eQJXS02qof7S+vkLOGznuC4KD0yXs9+jg
Ih6ANg6YBlxNZWDWAYZeJIIrQfINnzC36dQcb4StiUOKJu4eT5YxH4Afv39L3eoZ
eKsVE2Dddt4tE+i8oEBA7XPKZZH8le2V0csnZ2cbsphxftwy72qkFukmkmcBn4Zq
fXeAsdbDHoKlnTPeTNAXcgPUyLGhxqoX5vaIsNDFPGQ3BhsHfmNgKkZ4J+BSGGnc
R4Gre/mN1eNz3LYn7RZeExOrcnwnQAvFVhtz4ZFIndxP3kSiNh2Lo/7p/ECEnnEn
jOUCkARuA3lUZiWMzGMDo/kgTi6C6kulEkCWcp417OeXmThrjA==
-----END CERTIFICATE-----

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1,23 @@
-----BEGIN CERTIFICATE-----
MIID2TCCAsGgAwIBAgIUQZYX6ujSb/QFL2p8BY62ydJ/HEYwDQYJKoZIhvcNAQEL
BQAwgYUxCzAJBgNVBAYTAkRFMRwwGgYDVQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVu
MQ8wDQYDVQQHDAZCb2NodW0xHjAcBgNVBAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21i
SDEnMCUGA1UEAwweQ29uc3RlbGxhdGlvbiBUZXN0aW5nIFBDQSAyMDIyMB4XDTIy
MDkyMzE0MTYwN1oXDTIyMTAyMzE0MTYwN1owgYUxCzAJBgNVBAYTAkRFMRwwGgYD
VQQIDBNOb3JkcmhlaW4gV2VzdGZhbGVuMQ8wDQYDVQQHDAZCb2NodW0xHjAcBgNV
BAoMFUVkZ2VsZXNzIFN5c3RlbXMgR21iSDEnMCUGA1UEAwweQ29uc3RlbGxhdGlv
biBUZXN0aW5nIFBDQSAyMDIyMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
AQEAx4lZNSnqGy1Dk8hxQHgzzIpCAr2+zPtNpPKdXq8PLIZ1fBERYkVW66JHH5Dl
9pzMl8KaJkkRbv5orxzpL0g9qtM+j/JQl1Ap7S9MAXgysATwyaFWfddLW+ZOi/XS
oOhq1JXp0FFFUBQwP56JtqhUEcgaICi41L9S/XMxQaoZfRQu+9ICVlDMLurqVdI4
Or5mz7eWUqj34Xl5bpjMOZgbyI1pYN4UxvODKFIYhVTmKZOWA0np4JtI4iCA5Dkd
ckF9uwOyQUyGUs7Atar2tZr2hWK66NxhcdSfj9ltuBHwskJnoJYesWPz0QHz26xO
d+a4WwIKhKfchvv5qVq9KFkRpQIDAQABoz8wPTAdBgNVHQ4EFgQUvCP4FCxcypyU
STwBjhtgqnojgaEwDwYDVR0TAQH/BAUwAwEB/zALBgNVHQ8EBAMCAYYwDQYJKoZI
hvcNAQELBQADggEBAAhpPWCLVl8FvWZOeZA2iOKmLen/c6W0d9WiB9qvj+mI8QBq
kLhkqkxEcXTpaOjCxJLqLU3LsA828yUdfL0zmxusrJlz+gux+KRlRn1yTsCyWqmx
9rYUXO6IFwZnSUV923uVZ1nkHydwqV9hqIgrKYrppzDXOsm+ugz+NP2lxFNp6q8u
MyMsClaTdfPT+EUucp7g1lPVdtWV4RbWK/v3/rp2l5jzs7F0Roa3zc/+YquwB/AA
ZasNpDInz6RlEzhA/GkXEO5Rssem4a1NBwGrvs9mCIm5sKLaxLUnlM7SLCAeEBdS
qxLwUVTOtnQzWEM04I978V3zfKNPBtl2mvhuXYw=
-----END CERTIFICATE-----

Binary file not shown.

View File

@ -0,0 +1,74 @@
#!/usr/bin/env bash
set -euo pipefail
if [ -z "${CONFIG_FILE-}" ] && [ -f "${CONFIG_FILE-}" ]; then
. "${CONFIG_FILE}"
fi
POSITIONAL_ARGS=()
while [[ $# -gt 0 ]]; do
case $1 in
-n|--name)
AZURE_VM_NAME="$2"
shift # past argument
shift # past value
;;
-*|--*)
echo "Unknown option $1"
exit 1
;;
*)
POSITIONAL_ARGS+=("$1") # save positional arg
shift # past argument
;;
esac
done
set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
AZ_VM_INFO=$(az vm show --name "${AZURE_VM_NAME}" --resource-group "${AZURE_RESOURCE_GROUP_NAME}" -o json)
NIC=$(echo "${AZ_VM_INFO}" | jq -r '.networkProfile.networkInterfaces[0].id')
NIC_INFO=$(az network nic show --ids "${NIC}" -o json)
PUBIP=$(echo "${NIC_INFO}" | jq -r '.ipConfigurations[0].publicIpAddress.id')
NSG=$(echo "${NIC_INFO}" | jq -r '.networkSecurityGroup.id')
SUBNET=$(echo "${NIC_INFO}" | jq -r '.ipConfigurations[0].subnet.id')
VNET=$(echo $SUBNET | sed 's#/subnets/.*##')
DISK=$(echo "${AZ_VM_INFO}" | jq -r '.storageProfile.osDisk.managedDisk.id')
delete_vm () {
az vm delete -y --name "${AZURE_VM_NAME}" \
--resource-group "${AZURE_RESOURCE_GROUP_NAME}" || true
}
delete_vnet () {
az network vnet delete --ids "${VNET}" || true
}
delete_subnet () {
az network vnet subnet delete --ids "${SUBNET}" || true
}
delete_nsg () {
az network nsg delete --ids "${NSG}" || true
}
delete_pubip () {
az network public-ip delete --ids "${PUBIP}" || true
}
delete_disk () {
az disk delete -y --ids "${DISK}" || true
}
delete_nic () {
az network nic delete --ids "${NIC}" || true
}
delete_vm
delete_disk
delete_nic
delete_nsg
delete_subnet
delete_vnet
delete_pubip

View File

@ -0,0 +1,65 @@
#!/usr/bin/env bash
set -euo pipefail
if [ -z "${CONFIG_FILE-}" ] && [ -f "${CONFIG_FILE-}" ]; then
. "${CONFIG_FILE}"
fi
AZURE_SUBSCRIPTION=$(az account show --query id -o tsv)
POSITIONAL_ARGS=()
while [[ $# -gt 0 ]]; do
case $1 in
-n|--name)
AZURE_VM_NAME="$2"
shift # past argument
shift # past value
;;
-*|--*)
echo "Unknown option $1"
exit 1
;;
*)
POSITIONAL_ARGS+=("$1") # save positional arg
shift # past argument
;;
esac
done
set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
VM_DISK=$(az vm show -g "${AZURE_RESOURCE_GROUP_NAME}" --name "${AZURE_VM_NAME}" --query "storageProfile.osDisk.managedDisk.id" -o tsv)
LOCATION=$(az disk show --ids "${VM_DISK}" --query "location" -o tsv)
az snapshot create \
-g "${AZURE_RESOURCE_GROUP_NAME}" \
--source "${VM_DISK}" \
--name "${AZURE_SNAPSHOT_NAME}" \
-l "${LOCATION}"
# Azure CLI does not implement getSecureVMGuestStateSAS for snapshots yet
# az snapshot grant-access \
# --duration-in-seconds 3600 \
# --access-level Read \
# --name "${AZURE_SNAPSHOT_NAME}" \
# -g "${AZURE_RESOURCE_GROUP_NAME}"
BEGIN=$(az rest \
--method post \
--url "https://management.azure.com/subscriptions/${AZURE_SUBSCRIPTION}/resourceGroups/${AZURE_RESOURCE_GROUP_NAME}/providers/Microsoft.Compute/snapshots/${AZURE_SNAPSHOT_NAME}/beginGetAccess" \
--uri-parameters api-version="2021-12-01" \
--body '{"access": "Read", "durationInSeconds": 3600, "getSecureVMGuestStateSAS": true}' \
--verbose 2>&1)
ASYNC_OPERATION_URI=$(echo "${BEGIN}" | grep Azure-AsyncOperation | cut -d ' ' -f 7 | tr -d "'")
sleep 10
ACCESS=$(az rest --method get --url "${ASYNC_OPERATION_URI}")
VMGS_URL=$(echo "${ACCESS}" | jq -r '.properties.output.securityDataAccessSAS')
curl -L -o "${AZURE_VMGS_FILENAME}" "${VMGS_URL}"
az snapshot revoke-access \
--name "${AZURE_SNAPSHOT_NAME}" \
-g "${AZURE_RESOURCE_GROUP_NAME}"
az snapshot delete \
--name "${AZURE_SNAPSHOT_NAME}" \
-g "${AZURE_RESOURCE_GROUP_NAME}"
echo "VMGS saved to ${AZURE_VMGS_FILENAME}"

View File

@ -0,0 +1,101 @@
#!/usr/bin/env bash
set -euo pipefail
if [ -z "${CONFIG_FILE-}" ] && [ -f "${CONFIG_FILE-}" ]; then
. "${CONFIG_FILE}"
fi
POSITIONAL_ARGS=()
while [[ $# -gt 0 ]]; do
case $1 in
-n|--name)
AZURE_VM_NAME="$2"
shift # past argument
shift # past value
;;
-g|--gallery)
CREATE_FROM_GALLERY=YES
shift # past argument
;;
-d|--disk)
CREATE_FROM_GALLERY=NO
shift # past argument
;;
--secure-boot)
AZURE_SECURE_BOOT="$2"
shift # past argument
shift # past value
;;
--disk-name)
AZURE_DISK_NAME="$2"
shift # past argument
shift # past value
;;
-*|--*)
echo "Unknown option $1"
exit 1
;;
*)
POSITIONAL_ARGS+=("$1") # save positional arg
shift # past argument
;;
esac
done
set -- "${POSITIONAL_ARGS[@]}" # restore positional parameters
if [[ "${AZURE_SECURITY_TYPE}" == "ConfidentialVM" ]]; then
VMSIZE="Standard_DC2as_v5"
elif [[ "${AZURE_SECURITY_TYPE}" == "TrustedLaunch" ]]; then
VMSIZE="standard_D2as_v5"
else
echo "Unknown security type: ${AZURE_SECURITY_TYPE}"
exit 1
fi
create_vm_from_disk () {
AZURE_DISK_REFERENCE=$(az disk show --resource-group ${AZURE_RESOURCE_GROUP_NAME} --name ${AZURE_DISK_NAME} --query id -o tsv)
az vm create --name "${AZURE_VM_NAME}" \
--resource-group "${AZURE_RESOURCE_GROUP_NAME}" \
-l ${AZURE_REGION} \
--size "${VMSIZE}" \
--public-ip-sku Standard \
--os-type Linux \
--attach-os-disk "${AZURE_DISK_REFERENCE}" \
--security-type "${AZURE_SECURITY_TYPE}" \
--os-disk-security-encryption-type VMGuestStateOnly \
--enable-vtpm true \
--enable-secure-boot "${AZURE_SECURE_BOOT}" \
--boot-diagnostics-storage "" \
--no-wait
}
create_vm_from_sig () {
AZURE_IMAGE_REFERENCE=$(az sig image-version show \
--gallery-image-definition "${AZURE_IMAGE_DEFINITION}" \
--gallery-image-version "${AZURE_IMAGE_VERSION}" \
--gallery-name "${AZURE_GALLERY_NAME}" \
-g "${AZURE_RESOURCE_GROUP_NAME}" \
--query id -o tsv)
az vm create --name "${AZURE_VM_NAME}" \
--resource-group "${AZURE_RESOURCE_GROUP_NAME}" \
-l ${AZURE_REGION} \
--size "${VMSIZE}" \
--public-ip-sku Standard \
--image "${AZURE_IMAGE_REFERENCE}" \
--security-type "${AZURE_SECURITY_TYPE}" \
--os-disk-security-encryption-type VMGuestStateOnly \
--enable-vtpm true \
--enable-secure-boot "${AZURE_SECURE_BOOT}" \
--boot-diagnostics-storage "" \
--no-wait
}
if [ "$CREATE_FROM_GALLERY" = "YES" ]; then
create_vm_from_sig
else
create_vm_from_disk
fi
sleep 30
az vm boot-diagnostics enable --name "${AZURE_VM_NAME}" --resource-group "${AZURE_RESOURCE_GROUP_NAME}"

View File

@ -0,0 +1,89 @@
#!/usr/bin/env bash
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
BASE_DIR=$(realpath "${SCRIPT_DIR}/..")
# Set to qemu+tcp://localhost:16599/system for dockerized libvirt setup
if [[ -z "${LIBVIRT_SOCK}" ]]; then
LIBVIRT_SOCK=qemu:///system
fi
libvirt_nvram_gen () {
local image_path="${1}"
if test -f "${BASE_DIR}/image.nvram.template"; then
echo "NVRAM template already generated: $(realpath "--relative-to=$(pwd)" ${BASE_DIR}/image.nvram.template)"
return
fi
if ! test -f "${image_path}"; then
echo "Image \"${image_path}\" does not exist yet. To generate nvram, create disk image first."
return
fi
OVMF_CODE=/usr/share/OVMF/OVMF_CODE_4M.ms.fd
OVMF_VARS=/usr/share/OVMF/OVMF_VARS_4M.ms.fd
if ! test -f "${OVMF_CODE}"; then
OVMF_CODE=/usr/share/OVMF/OVMF_CODE.secboot.fd
fi
if ! test -f "${OVMF_VARS}"; then
OVMF_VARS=/usr/share/OVMF/OVMF_VARS.secboot.fd
fi
echo "Using OVMF_CODE: ${OVMF_CODE}"
echo "Using OVMF_VARS: ${OVMF_VARS}"
# generate nvram file using libvirt
virt-install --name constell-nvram-gen \
--connect ${LIBVIRT_SOCK} \
--nonetworks \
--description 'Constellation' \
--ram 1024 \
--vcpus 1 \
--osinfo detect=on,require=off \
--disk "${image_path},format=raw" \
--boot "machine=q35,menu=on,loader=${OVMF_CODE},loader.readonly=yes,loader.type=pflash,nvram.template=${OVMF_VARS},nvram=${BASE_DIR}/image.nvram,loader_secure=yes" \
--features smm.state=on \
--noautoconsole
echo -e 'connect using'
echo -e ' \u001b[1mvirsh console constell-nvram-gen\u001b[0m'
echo -e ''
echo -e 'Load db cert with MokManager or enroll full PKI with firmware setup'
echo -e ''
echo -e ' \u001b[1mMokManager\u001b[0m'
echo -e ' For mokmanager, try to boot as usual. You will see this message:'
echo -e ' > "Verification failed: (0x1A) Security Violation"'
echo -e ' Press OK, then ENTER, then "Enroll key from disk"'
echo -e ' Select the following key:'
echo -e ' > \u001b[1m/EFI/loader/keys/auto/db.cer\u001b[0m'
echo -e ' Press Continue, then choose "Yes" to the question "Enroll the key(s)?"'
echo -e ' Choose reboot and continue this script.'
echo -e ''
echo -e ' \u001b[1mFirmware setup\u001b[0m'
echo -e ' For firmware setup, press F2.'
echo -e ' Go to "Device Manager">"Secure Boot Configuration">"Secure Boot Mode"'
echo -e ' Choose "Custom Mode"'
echo -e ' Go to "Custom Securee Boot Options"'
echo -e ' Go to "PK Options">"Enroll PK", Press "Y" if queried, "Enroll PK using File"'
echo -e ' Select the following cert: \u001b[1m/EFI/loader/keys/auto/PK.cer\u001b[0m'
echo -e ' Choose "Commit Changes and Exit"'
echo -e ' Go to "KEK Options">"Enroll KEK", Press "Y" if queried, "Enroll KEK using File"'
echo -e ' Select the following cert: \u001b[1m/EFI/loader/keys/auto/KEK.cer\u001b[0m'
echo -e ' Choose "Commit Changes and Exit"'
echo -e ' Go to "DB Options">"Enroll Signature">"Enroll Signature using File"'
echo -e ' Select the following cert: \u001b[1m/EFI/loader/keys/auto/db.cer\u001b[0m'
echo -e ' Choose "Commit Changes and Exit"'
echo -e ' Repeat the last step for the following certs:'
echo -e ' > \u001b[1m/EFI/loader/keys/auto/MicWinProPCA2011_2011-10-19.crt\u001b[0m'
echo -e ' > \u001b[1m/EFI/loader/keys/auto/MicCorUEFCA2011_2011-06-27.crt\u001b[0m'
echo -e ' Reboot and continue this script.'
echo -e ''
echo -e 'Press ENTER to continue after you followed one of the guides from above.'
read
sudo cp "${BASE_DIR}/image.nvram" "${BASE_DIR}/image.nvram.template"
virsh --connect "${LIBVIRT_SOCK}" destroy --domain constell-nvram-gen
virsh --connect "${LIBVIRT_SOCK}" undefine --nvram constell-nvram-gen
rm -f "${BASE_DIR}/image.nvram"
echo "NVRAM template generated: $(realpath "--relative-to=$(pwd)" ${BASE_DIR}/image.nvram.template)"
}
libvirt_nvram_gen $1

View File

@ -0,0 +1,65 @@
#!/usr/bin/env bash
# This script generates a PKI for secure boot.
# It is based on the example from https://github.com/systemd/systemd/blob/main/man/loader.conf.xml
# This is meant to be used for development purposes only.
# Release images are signed using a different set of keys.
# Set PKI to an empty folder and PKI_SET to "dev".
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
TEMPLATES=${SCRIPT_DIR}/templates
BASE_DIR=$(realpath "${SCRIPT_DIR}/..")
if [ -z "${PKI}" ]; then
PKI=${BASE_DIR}/pki
fi
if [ -z "${PKI_SET}" ]; then
PKI_SET=dev
fi
gen_pki () {
# Only use for non-production images.
# Use real PKI for production images instead.
count=$(ls -1 ${PKI}/*.{key,crt,cer,esl,auth} 2>/dev/null | wc -l)
if [ $count != 0 ]
then
echo PKI files $(ls -1 $(realpath "--relative-to=$(pwd)" ${PKI})/*.{key,crt,cer,esl,auth}) already exist
return
fi
mkdir -p "${PKI}"
pushd "${PKI}"
uuid=$(systemd-id128 new --uuid)
for key in PK KEK db; do
openssl req -new -x509 -config "${TEMPLATES}/${PKI_SET}_${key}.conf" -keyout "${key}.key" -out "${key}.crt" -nodes
openssl x509 -outform DER -in "${key}.crt" -out "${key}.cer"
cert-to-efi-sig-list -g "${uuid}" "${key}.crt" "${key}.esl"
done
for key in MicWinProPCA2011_2011-10-19.crt MicCorUEFCA2011_2011-06-27.crt MicCorKEKCA2011_2011-06-24.crt; do
curl -sL "https://www.microsoft.com/pkiops/certs/${key}" --output "${key}"
sbsiglist --owner 77fa9abd-0359-4d32-bd60-28f4e78f784b --type x509 --output "${key%crt}esl" "${key}"
done
# Optionally add Microsoft Windows Production CA 2011 (needed to boot into Windows).
cat MicWinProPCA2011_2011-10-19.esl >> db.esl
# Optionally add Microsoft Corporation UEFI CA 2011 (for firmware drivers / option ROMs
# and third-party boot loaders (including shim). This is highly recommended on real
# hardware as not including this may soft-brick your device (see next paragraph).
cat MicCorUEFCA2011_2011-06-27.esl >> db.esl
# Optionally add Microsoft Corporation KEK CA 2011. Recommended if either of the
# Microsoft keys is used as the official UEFI revocation database is signed with this
# key. The revocation database can be updated with [fwupdmgr(1)](https://www.freedesktop.org/software/systemd/man/fwupdmgr.html#).
cat MicCorKEKCA2011_2011-06-24.esl >> KEK.esl
sign-efi-sig-list -c PK.crt -k PK.key PK PK.esl PK.auth
sign-efi-sig-list -c PK.crt -k PK.key KEK KEK.esl KEK.auth
sign-efi-sig-list -c KEK.crt -k KEK.key db db.esl db.auth
popd
}
# gen_pki generates a PKI for testing purposes only.
# if keys/certs are already present in the pki folder, they are not regenerated.
gen_pki

View File

@ -0,0 +1,45 @@
#!/bin/env bash
set -euo pipefail
# This script is used to add a signed shim to the image.raw file EFI partition after running `mkosi build`.
if (( $# != 1 ))
then
echo "Usage: $0 <image.raw>"
exit 1
fi
# SOURCE is the URL used to download the signed shim RPM
SOURCE=https://kojipkgs.fedoraproject.org/packages/shim/15.6/2/x86_64/shim-x64-15.6-2.x86_64.rpm
# EXPECTED_SHA512 is the SHA512 checksum of the signed shim RPM
EXPECTED_SHA512=971978bddee95a6a134ef05c4d88cf5df41926e631de863b74ef772307f3e106c82c8f6889c18280d47187986abd774d8671c5be4b85b1b0bb3d1858b65d02cf
SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
BASE_DIR=$(realpath "${SCRIPT_DIR}/..")
TMPDIR=$(mktemp -d)
pushd "${TMPDIR}"
curl -sL -o shim.rpm "${SOURCE}"
echo "Checking SHA512 checksum of signed shim..."
sha512sum -c <<< "${EXPECTED_SHA512} shim.rpm"
rpm2cpio shim.rpm | cpio -idmv
echo $TMPDIR
popd
MOUNTPOINT=$(mktemp -d)
sectoroffset=$(sfdisk -J "${1}" | jq -r '.partitiontable.partitions[0].start')
byteoffset=$((sectoroffset * 512))
mount -o offset="${byteoffset}" "${1}" "${MOUNTPOINT}"
mkdir -p "${MOUNTPOINT}/EFI/BOOT/"
cp "${TMPDIR}/boot/efi/EFI/BOOT/BOOTX64.EFI" "${MOUNTPOINT}/EFI/BOOT/"
cp "${TMPDIR}/boot/efi/EFI/fedora/mmx64.efi" "${MOUNTPOINT}/EFI/BOOT/"
cp "${MOUNTPOINT}/EFI/systemd/systemd-bootx64.efi" "${MOUNTPOINT}/EFI/BOOT/grubx64.efi"
# Remove unused kernel and initramfs from EFI to save space
# We boot from unified kernel image anyway
rm -f "${MOUNTPOINT}"/*/*/{linux,initrd}
umount "${MOUNTPOINT}"
rm -rf ${MOUNTPOINT}
rm -rf "${TMPDIR}"

View File

@ -0,0 +1,20 @@
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
x509_extensions = v3_req
req_extensions = v3_req
prompt = no
dirstring_type = nobmp
[ req_distinguished_name ]
C = DE
ST = Nordrhein Westfalen
L = Bochum
O = Edgeless Systems GmbH
CN = Constellation Development KEK CA 2022
[ v3_req ]
subjectKeyIdentifier = hash
basicConstraints = critical,CA:true
keyUsage = digitalSignature,keyCertSign,cRLSign

View File

@ -0,0 +1,20 @@
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
x509_extensions = v3_req
req_extensions = v3_req
prompt = no
dirstring_type = nobmp
[ req_distinguished_name ]
C = DE
ST = Nordrhein Westfalen
L = Bochum
O = Edgeless Systems GmbH
CN = Constellation Development UEFI CA 2022
[ v3_req ]
subjectKeyIdentifier = hash
basicConstraints = critical,CA:true
keyUsage = digitalSignature,keyCertSign,cRLSign

View File

@ -0,0 +1,20 @@
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
x509_extensions = v3_req
req_extensions = v3_req
prompt = no
dirstring_type = nobmp
[ req_distinguished_name ]
C = DE
ST = Nordrhein Westfalen
L = Bochum
O = Edgeless Systems GmbH
CN = Constellation Development PCA 2022
[ v3_req ]
subjectKeyIdentifier = hash
basicConstraints = critical,CA:true
keyUsage = digitalSignature,keyCertSign,cRLSign

View File

@ -0,0 +1,20 @@
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
x509_extensions = v3_req
req_extensions = v3_req
prompt = no
dirstring_type = nobmp
[ req_distinguished_name ]
C = DE
ST = Nordrhein Westfalen
L = Bochum
O = Edgeless Systems GmbH
CN = Constellation KEK CA 2022
[ v3_req ]
subjectKeyIdentifier = hash
basicConstraints = critical,CA:true
keyUsage = digitalSignature,keyCertSign,cRLSign

View File

@ -0,0 +1,20 @@
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
x509_extensions = v3_req
req_extensions = v3_req
prompt = no
dirstring_type = nobmp
[ req_distinguished_name ]
C = DE
ST = Nordrhein Westfalen
L = Bochum
O = Edgeless Systems GmbH
CN = Constellation UEFI CA 2022
[ v3_req ]
subjectKeyIdentifier = hash
basicConstraints = critical,CA:true
keyUsage = digitalSignature,keyCertSign,cRLSign

View File

@ -0,0 +1,20 @@
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
x509_extensions = v3_req
req_extensions = v3_req
prompt = no
dirstring_type = nobmp
[ req_distinguished_name ]
C = DE
ST = Nordrhein Westfalen
L = Bochum
O = Edgeless Systems GmbH
CN = Constellation Production PCA 2022
[ v3_req ]
subjectKeyIdentifier = hash
basicConstraints = critical,CA:true
keyUsage = digitalSignature,keyCertSign,cRLSign

View File

@ -0,0 +1,20 @@
[ req ]
default_bits = 2048
distinguished_name = req_distinguished_name
x509_extensions = v3_req
req_extensions = v3_req
prompt = no
dirstring_type = nobmp
[ req_distinguished_name ]
C = DE
ST = Nordrhein Westfalen
L = Bochum
O = Edgeless Systems GmbH
CN = Constellation Testing KEK CA 2022
[ v3_req ]
subjectKeyIdentifier = hash
basicConstraints = critical,CA:true
keyUsage = digitalSignature,keyCertSign,cRLSign

Some files were not shown because too many files have changed in this diff Show More