From 4ef3d10be3074cffc97e2069a4fee8a11690c1e1 Mon Sep 17 00:00:00 2001 From: Malte Poll Date: Mon, 11 Sep 2023 15:52:05 +0200 Subject: [PATCH] image: initrd layer --- image/initrd/BUILD.bazel | 25 ++++++++ image/initrd/mkosi.conf | 43 +++++++++++++ image/initrd/mkosi.postinst | 16 +++++ .../20-constellation-initrd.preset | 13 ++++ .../lib/systemd/system/aws-nvme-disk.service | 18 ++++++ .../systemd/system/azure-provisioning.service | 16 +++++ .../lib/systemd/system/gcp-nvme-disk.service | 18 ++++++ .../systemd/system/prepare-state-disk.service | 21 ++++++ .../01-enable-in-initd.conf | 2 + .../usr/local/bin/azure-provisioning | 64 +++++++++++++++++++ .../mkosi.skeleton/usr/sbin/aws-nvme-disk | 35 ++++++++++ .../mkosi.skeleton/usr/sbin/gcp-nvme-disk | 31 +++++++++ .../usr/sbin/prepare-state-disk | 34 ++++++++++ image/initrd/reposdir/amzn2-core.repo | 39 +++++++++++ 14 files changed, 375 insertions(+) create mode 100644 image/initrd/BUILD.bazel create mode 100644 image/initrd/mkosi.conf create mode 100755 image/initrd/mkosi.postinst create mode 100644 image/initrd/mkosi.skeleton/usr/lib/systemd/system-preset/20-constellation-initrd.preset create mode 100644 image/initrd/mkosi.skeleton/usr/lib/systemd/system/aws-nvme-disk.service create mode 100644 image/initrd/mkosi.skeleton/usr/lib/systemd/system/azure-provisioning.service create mode 100644 image/initrd/mkosi.skeleton/usr/lib/systemd/system/gcp-nvme-disk.service create mode 100644 image/initrd/mkosi.skeleton/usr/lib/systemd/system/prepare-state-disk.service create mode 100644 image/initrd/mkosi.skeleton/usr/lib/systemd/system/systemd-networkd.service.d/01-enable-in-initd.conf create mode 100755 image/initrd/mkosi.skeleton/usr/local/bin/azure-provisioning create mode 100755 image/initrd/mkosi.skeleton/usr/sbin/aws-nvme-disk create mode 100755 image/initrd/mkosi.skeleton/usr/sbin/gcp-nvme-disk create mode 100755 image/initrd/mkosi.skeleton/usr/sbin/prepare-state-disk create mode 100644 image/initrd/reposdir/amzn2-core.repo diff --git a/image/initrd/BUILD.bazel b/image/initrd/BUILD.bazel new file mode 100644 index 000000000..3b7b57975 --- /dev/null +++ b/image/initrd/BUILD.bazel @@ -0,0 +1,25 @@ +load("//bazel/mkosi:mkosi_image.bzl", "mkosi_image") + +mkosi_image( + name = "initrd", + srcs = [ + "mkosi.postinst", + ] + glob([ + "mkosi.skeleton/**", + "reposdir/**", + ]), + outs = [ + "image", + "image.cpio.zst", + ], + extra_trees = [ + "//image:sysroot_tar", + "//disk-mapper/cmd:disk-mapper-package.tar", + ], + mkosi_conf = "mkosi.conf", + tags = [ + "manual", + "no-cache", + ], + visibility = ["//visibility:public"], +) diff --git a/image/initrd/mkosi.conf b/image/initrd/mkosi.conf new file mode 100644 index 000000000..38304bdf3 --- /dev/null +++ b/image/initrd/mkosi.conf @@ -0,0 +1,43 @@ +[Distribution] +Distribution=fedora +Release=38 + +[Output] +Format=cpio +Output=image +SourceDateEpoch=0 + +[Content] +MakeInitrd=yes +Bootable=no +Packages=systemd + systemd-networkd + systemd-resolved + systemd-udev + dbus + util-linux + gzip + kmod + tpm2-tools + curl # for azure provision service +# nvme / disk / udev tools +Packages=udev + nvme-cli + ec2-utils + WALinuxAgent-udev + xxd + google-compute-engine-guest-configs-udev # google_nvme_id + device-mapper + cryptsetup + +PackageManagerTrees=reposdir:/etc/yum.repos.d + +RemoveFiles=/var/log +RemoveFiles=/var/cache +RemoveFiles=/etc/pki/ca-trust/extracted/java/cacerts + /usr/lib/sysimage/libdnf5/transaction_history.sqlite* + /var/cache/ldconfig/aux-cache +# https://github.com/authselect/authselect/pull/348 +# RemoveFiles=/etc/authselect/* +CleanPackageMetadata=true +Seed=b04a9a33-4559-4af4-8b38-9249cf933229 diff --git a/image/initrd/mkosi.postinst b/image/initrd/mkosi.postinst new file mode 100755 index 000000000..5869ee4af --- /dev/null +++ b/image/initrd/mkosi.postinst @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euxo pipefail + +# ensure google_nvme_id is executable +chmod o+x "${BUILDROOT}/usr/lib/udev/google_nvme_id" +chmod g+x "${BUILDROOT}/usr/lib/udev/google_nvme_id" +chmod u+x "${BUILDROOT}/usr/lib/udev/google_nvme_id" + +# mask unwanted services +ln -s /dev/null "${BUILDROOT}/etc/systemd/system/systemd-pcrmachine.service" +ln -s /dev/null "${BUILDROOT}/etc/systemd/system/systemd-pcrfs-root.service" +ln -s /dev/null "${BUILDROOT}/etc/systemd/system/systemd-pcrfs@.service" +ln -s /dev/null "${BUILDROOT}/etc/systemd/system/systemd-pcrphase@.service" +ln -s /dev/null "${BUILDROOT}/etc/systemd/system/systemd-pcrphase-initrd.service" +ln -s /dev/null "${BUILDROOT}/etc/systemd/system/systemd-pcrphase-sysinit.service" +ln -s /dev/null "${BUILDROOT}/etc/systemd/system/systemd-pcrphase.service" diff --git a/image/initrd/mkosi.skeleton/usr/lib/systemd/system-preset/20-constellation-initrd.preset b/image/initrd/mkosi.skeleton/usr/lib/systemd/system-preset/20-constellation-initrd.preset new file mode 100644 index 000000000..ff18a6df9 --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/lib/systemd/system-preset/20-constellation-initrd.preset @@ -0,0 +1,13 @@ +enable prepare-state-disk.service +enable aws-nvme-disk.service +enable gcp-nvme-disk.service +enable azure-provisioning.service +enable dbus.service +enable dbus-broker.service +enable dbus-daemon.service +enable systemd-timesyncd.service +enable systemd-resolved.service +enable systemd-networkd.service +enable systemd-networkd-wait-online.service +enable systemd-udev.service +enable configure-constel-csp.service diff --git a/image/initrd/mkosi.skeleton/usr/lib/systemd/system/aws-nvme-disk.service b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/aws-nvme-disk.service new file mode 100644 index 000000000..605888929 --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/aws-nvme-disk.service @@ -0,0 +1,18 @@ +[Unit] +Description=Force symlink creation for AWS nvme disks +Before=prepare-state-disk.service +After=network-online.target +Wants=network-online.target +ConditionKernelCommandLine=constel.csp=aws + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/sbin/aws-nvme-disk +RemainAfterExit=yes +StandardOutput=tty +StandardInput=tty +StandardError=tty +TimeoutSec=infinity + +[Install] +WantedBy=basic.target multi-user.target diff --git a/image/initrd/mkosi.skeleton/usr/lib/systemd/system/azure-provisioning.service b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/azure-provisioning.service new file mode 100644 index 000000000..0e15129e9 --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/azure-provisioning.service @@ -0,0 +1,16 @@ +[Unit] +Description=Azure Provisioning +After=network-online.target +Wants=network-online.target +ConditionKernelCommandLine=constel.csp=azure + +[Service] +Type=oneshot +ExecStart=/usr/local/bin/azure-provisioning +RemainAfterExit=yes +StandardOutput=tty +StandardInput=tty +StandardError=tty + +[Install] +WantedBy=basic.target multi-user.target diff --git a/image/initrd/mkosi.skeleton/usr/lib/systemd/system/gcp-nvme-disk.service b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/gcp-nvme-disk.service new file mode 100644 index 000000000..ad8357c73 --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/gcp-nvme-disk.service @@ -0,0 +1,18 @@ +[Unit] +Description=Force symlink creation for GCP nvme disks +Before=prepare-state-disk.service +After=network-online.target +Wants=network-online.target +ConditionKernelCommandLine=constel.csp=gcp + +[Service] +Type=oneshot +ExecStart=/bin/bash /usr/sbin/gcp-nvme-disk +RemainAfterExit=yes +StandardOutput=tty +StandardInput=tty +StandardError=tty +TimeoutSec=infinity + +[Install] +WantedBy=basic.target multi-user.target diff --git a/image/initrd/mkosi.skeleton/usr/lib/systemd/system/prepare-state-disk.service b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/prepare-state-disk.service new file mode 100644 index 000000000..7b490d5a9 --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/prepare-state-disk.service @@ -0,0 +1,21 @@ +[Unit] +Description=Prepare encrypted state disk +Before=initrd-fs.target +After=network-online.target nss-lookup.target configure-constel-csp.service +After=export_constellation_debug.service +Wants=network-online.target +Requires=initrd-root-fs.target configure-constel-csp.service +FailureAction=reboot-immediate + +[Service] +Type=oneshot +EnvironmentFile=/run/constellation.env +ExecStart=/bin/bash /usr/sbin/prepare-state-disk $CONSTELLATION_DEBUG_FLAGS +RemainAfterExit=yes +StandardOutput=tty +StandardInput=tty +StandardError=tty +TimeoutSec=infinity + +[Install] +WantedBy=basic.target diff --git a/image/initrd/mkosi.skeleton/usr/lib/systemd/system/systemd-networkd.service.d/01-enable-in-initd.conf b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/systemd-networkd.service.d/01-enable-in-initd.conf new file mode 100644 index 000000000..75c0ff968 --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/lib/systemd/system/systemd-networkd.service.d/01-enable-in-initd.conf @@ -0,0 +1,2 @@ +[Install] +WantedBy=initrd.target diff --git a/image/initrd/mkosi.skeleton/usr/local/bin/azure-provisioning b/image/initrd/mkosi.skeleton/usr/local/bin/azure-provisioning new file mode 100755 index 000000000..c2f2ad0c6 --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/local/bin/azure-provisioning @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# source https://learn.microsoft.com/en-us/azure/virtual-machines/linux/no-agent + +set -euo pipefail +shopt -s inherit_errexit + +attempts=1 +until [[ ${attempts} -gt 5 ]]; do + echo "obtaining goal state - attempt ${attempts}" + goalstate=$(curl --fail -v -X 'GET' -H "x-ms-agent-name: azure-vm-register" \ + -H "Content-Type: text/xml;charset=utf-8" \ + -H "x-ms-version: 2012-11-30" \ + "http://168.63.129.16/machine/?comp=goalstate") + if [[ $? -eq 0 ]]; then + echo "successfully retrieved goal state" + retrieved_goal_state=true + break + fi + sleep 5 + attempts=$((attempts + 1)) +done + +if [[ ${retrieved_goal_state} != "true" ]]; then + echo "failed to obtain goal state - cannot register this VM" + exit 1 +fi + +container_id=$(grep ContainerId <<< "${goalstate}" | sed 's/\s*<\/*ContainerId>//g' | sed 's/\r$//') +instance_id=$(grep InstanceId <<< "${goalstate}" | sed 's/\s*<\/*InstanceId>//g' | sed 's/\r$//') + +ready_doc=$( + cat << EOF + + + 1 + + ${container_id} + + + ${instance_id} + + Ready + + + + + +EOF +) + +attempts=1 +until [[ ${attempts} -gt 5 ]]; do + echo "registering with Azure - attempt ${attempts}" + curl --fail -v -X 'POST' -H "x-ms-agent-name: azure-vm-register" \ + -H "Content-Type: text/xml;charset=utf-8" \ + -H "x-ms-version: 2012-11-30" \ + -d "${ready_doc}" \ + "http://168.63.129.16/machine?comp=health" + if [[ $? -eq 0 ]]; then + echo "successfully register with Azure" + break + fi + sleep 5 # sleep to prevent throttling from wire server +done diff --git a/image/initrd/mkosi.skeleton/usr/sbin/aws-nvme-disk b/image/initrd/mkosi.skeleton/usr/sbin/aws-nvme-disk new file mode 100755 index 000000000..df3557f1b --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/sbin/aws-nvme-disk @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Copyright (c) Edgeless Systems GmbH +# +# SPDX-License-Identifier: AGPL-3.0-only + +set -euo pipefail +shopt -s extglob nullglob inherit_errexit + +AWS_STATE_DISK_DEVICENAME="sdb" +AWS_STATE_DISK_SYMLINK="/dev/${AWS_STATE_DISK_DEVICENAME}" + +function onError { + echo "Failed to symlink state disk" + sleep 2 # give the serial console time to print the error message +} + +trap onError ERR + +# hack: aws nvme udev rules are never executed. Create symlinks for the nvme devices manually. +while [[ ! -L ${AWS_STATE_DISK_SYMLINK} ]]; do + for nvmedisk in /dev/nvme*n1; do + linkname=$(nvme amzn id-ctrl -b "${nvmedisk}" | tail -c +3073 | head -c 32 | tr -d ' ') || true + if [[ -n ${linkname} ]] && [[ ${linkname} == "${AWS_STATE_DISK_DEVICENAME}" ]]; then + ln -s "${nvmedisk}" "${AWS_STATE_DISK_SYMLINK}" + fi + done + if [[ -L ${AWS_STATE_DISK_SYMLINK} ]]; then + break + fi + echo "Waiting for state disk to appear.." + sleep 2 +done + +echo "AWS state disk found" +echo "${AWS_STATE_DISK_SYMLINK}" → "$(readlink -f "${AWS_STATE_DISK_SYMLINK}")" diff --git a/image/initrd/mkosi.skeleton/usr/sbin/gcp-nvme-disk b/image/initrd/mkosi.skeleton/usr/sbin/gcp-nvme-disk new file mode 100755 index 000000000..77a93d661 --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/sbin/gcp-nvme-disk @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Copyright (c) Edgeless Systems GmbH +# +# SPDX-License-Identifier: AGPL-3.0-only + +set -euo pipefail +shopt -s extglob nullglob inherit_errexit + +GCP_STATE_DISK_SYMLINK="/dev/disk/by-id/google-state-disk" + +function onError { + echo "Failed to symlink state disk" + sleep 2 # give the serial console time to print the error message +} + +trap onError ERR + +# hack: gcp nvme udev rules are never executed. Create symlinks for the nvme devices manually. +while [[ ! -L ${GCP_STATE_DISK_SYMLINK} ]]; do + for nvmedisk in /dev/nvme?n?; do + /usr/lib/udev/google_nvme_id -s -d "${nvmedisk}" + done + if [[ -L ${GCP_STATE_DISK_SYMLINK} ]]; then + break + fi + echo "Waiting for state disk to appear.." + sleep 2 +done + +echo "GCP state disk found" +echo "${GCP_STATE_DISK_SYMLINK}" → "$(readlink -f "${GCP_STATE_DISK_SYMLINK}")" diff --git a/image/initrd/mkosi.skeleton/usr/sbin/prepare-state-disk b/image/initrd/mkosi.skeleton/usr/sbin/prepare-state-disk new file mode 100755 index 000000000..b39999e7e --- /dev/null +++ b/image/initrd/mkosi.skeleton/usr/sbin/prepare-state-disk @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Copyright (c) Edgeless Systems GmbH +# +# SPDX-License-Identifier: AGPL-3.0-only + +set -euo pipefail +shopt -s inherit_errexit + +function onError { + echo "Failed to prepare state disk" + sleep 2 # give the serial console time to print the error message +} + +trap onError ERR + +# parsing of the command line arguments. check if argv[1] is --debug +verbosity=0 +if [[ $# -gt 0 ]]; then + if [[ $1 == "--debug" ]]; then + verbosity=-1 + echo "[Constellation] Debug mode enabled" + else + echo "[Constellation] Unknown argument: $1" + exit 1 + fi +else + echo "[Constellation] Debug mode disabled" +fi + +# Prepare the encrypted volume by either initializing it with a random key or by aquiring the key from another bootstrapper. +# Store encryption key (random or recovered key) in /run/cryptsetup-keys.d/state.key +disk-mapper \ + -csp "${CONSTEL_CSP}" \ + -v "${verbosity}" diff --git a/image/initrd/reposdir/amzn2-core.repo b/image/initrd/reposdir/amzn2-core.repo new file mode 100644 index 000000000..32b6472b4 --- /dev/null +++ b/image/initrd/reposdir/amzn2-core.repo @@ -0,0 +1,39 @@ +[amzn2-core] +name=Amazon Linux 2 core repository +#mirrorlist=$awsproto://$amazonlinux.$awsregion.$awsdomain/$releasever/$product/$target/$basearch/mirror.list +mirrorlist=https://amazonlinux-2-repos-us-east-2.s3.dualstack.us-east-2.amazonaws.com/2/core/latest/x86_64/mirror.list +priority=10 +gpgcheck=1 +#gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-amazon-linux-2 +gpgkey=https://cdn.amazonlinux.com/_assets/11CF1F95C87F5B1A.asc +enabled=1 +metadata_expire=300 +mirrorlist_expire=300 +report_instanceid=yes +includepkgs=ec2-utils + +# [amzn2-core-source] +# name=Amazon Linux 2 core repository - source packages +# mirrorlist=$awsproto://$amazonlinux.$awsregion.$awsdomain/$releasever/$product/$target/SRPMS/mirror.list +# priority=10 +# gpgcheck=1 +# #gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-amazon-linux-2 +# gpgkey=https://cdn.amazonlinux.com/_assets/11CF1F95C87F5B1A.asc +# enabled=0 +# metadata_expire=300 +# mirrorlist_expire=300 +# report_instanceid=yes +# includepkgs=ec2-utils + +# [amzn2-core-debuginfo] +# name=Amazon Linux 2 core repository - debuginfo packages +# mirrorlist=$awsproto://$amazonlinux.$awsregion.$awsdomain/$releasever/$product/$target/debuginfo/$basearch/mirror.list +# priority=10 +# gpgcheck=1 +# #gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-amazon-linux-2 +# gpgkey=https://cdn.amazonlinux.com/_assets/11CF1F95C87F5B1A.asc +# enabled=0 +# metadata_expire=300 +# mirrorlist_expire=300 +# report_instanceid=yes +# includepkgs=ec2-utils