constellation/dev-docs/howto/bare-metal/launch-constellation.sh

#!/usr/bin/env bash

set -euo pipefail

set -x

function cleanup {
  kill -SIGTERM "$(cat "${PWD}"/qemu-dnsmasq-br0.pid)" || true
  rm "${PWD}"/qemu-dnsmasq-br0.pid || true

  kill -SIGTERM "$(cat "${PWD}"/swtpm0.pid)" || true
  kill -SIGTERM "$(cat "${PWD}"/swtpm1.pid)" || true

  ip l delete br0 || true
  ip l delete tap0 || true
  ip l delete tap1 || true

  rm -r "${PWD}"/tpm0 || true
  rm -r "${PWD}"/tpm1 || true

  rm OVMF_VARS_0.fd || true
  rm OVMF_VARS_1.fd || true

  rm dnsmasq.leases || true
  rm dnsmasq.log || true

  rm constellation-mastersecret.json || true
  rm constellation-admin.conf || true
  rm constellation-cluster.log || true
  rm constellation-debug.log || true
  rm constellation-state.yaml || true
  rm -r constellation-upgrade || true

  docker stop metadata-server || true
}

trap cleanup EXIT

get_mac() {
  printf '52:54:%02X:%02X:%02X:%02X' $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256)) $((RANDOM % 256))
}

mac_0=$(get_mac)
mac_1=$(get_mac)

# Regarding network setup see: https://bbs.archlinux.org/viewtopic.php?id=207907

dd if=/dev/zero of=disk0.img iflag=fullblock bs=1M count=10000 && sync
dd if=/dev/zero of=disk1.img iflag=fullblock bs=1M count=10000 && sync

DEFAULT_INTERFACE=$(ip r show default | cut -d' ' -f5)

ip link add name br0 type bridge || true
ip addr add 10.42.0.1/16 dev br0 || true
ip link set br0 up

dnsmasq \
  --pid-file="${PWD}"/qemu-dnsmasq-br0.pid \
  --interface=br0 \
  --bind-interfaces \
  --log-facility="${PWD}"/dnsmasq.log \
  --dhcp-range=10.42.0.2,10.42.255.254 \
  --dhcp-leasefile="${PWD}"/dnsmasq.leases \
  --dhcp-host="${mac_0}",10.42.1.1,control-plane0 \
  --dhcp-host="${mac_1}",10.42.2.1,worker0

password=$(tr -dc 'A-Za-z0-9!?%=' < /dev/urandom | head -c 32) || true
password_hex=$(echo -n "${password}" | xxd -p -u -c 256)
echo "${password_hex}"

# htpasswd from apache2-utils
password_bcrypt=$(htpasswd -bnBC 10 "" "${password}" | tr -d ':\n')

docker run \
  -dit \
  --rm \
  --name metadata-server \
  --net=host \
  --mount type=bind,source="$(pwd)"/dnsmasq.leases,target=/dnsmasq.leases \
  ghcr.io/edgelesssys/constellation/qemu-metadata-api:v2.17.0-pre.0.20240603111213-d7ce6af383f2 \
  --dnsmasq-leases /dnsmasq.leases --initsecrethash "${password_bcrypt}"

cat > ./constellation-state.yaml <<- EOM
version: v1 # Schema version of this state file.
# State of the cluster's cloud resources. These values are retrieved during
infrastructure:
    uid: qemu # Unique identifier the cluster's cloud resources are tagged with.
    clusterEndpoint: 10.42.1.1 # Endpoint the cluster can be reached at. This is the endpoint that is being used by the CLI.
    inClusterEndpoint: 10.42.1.1 # The Cluster uses to reach itself. This might differ from the ClusterEndpoint in case e.g.,
    initSecret: "${password_hex}" # Secret used to authenticate the bootstrapping node.
    # List of Subject Alternative Names (SANs) to add to the Kubernetes API server certificate.
    apiServerCertSANs:
        - 10.42.1.1
    name: mini-qemu # Name used in the cluster's named resources.
    ipCidrNode: 10.42.0.0/16 # CIDR range of the cluster's nodes.
# DO NOT EDIT. State of the Constellation Kubernetes cluster.
clusterValues:
    clusterID: "" # Unique identifier of the cluster.
    ownerID: "" # Unique identifier of the owner of the cluster.
    measurementSalt: "" # Salt used to generate the ClusterID on the bootstrapping node.
EOM

sysctl net.ipv4.ip_forward=1
sysctl net.ipv6.conf.default.forwarding=1
sysctl net.ipv6.conf.all.forwarding=1

iptables -t nat -C POSTROUTING -o "${DEFAULT_INTERFACE}" -j MASQUERADE || iptables -t nat -I POSTROUTING -o "${DEFAULT_INTERFACE}" -j MASQUERADE
iptables -C FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT || iptables -I FORWARD -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT
iptables -P FORWARD ACCEPT

ip tuntap add dev tap0 mode tap user "${USER}" || true
ip link set tap0 up promisc on
ip link set tap0 master br0

iptables -C FORWARD -i tap0 -o "${DEFAULT_INTERFACE}" -j ACCEPT || iptables -I FORWARD -i tap0 -o "${DEFAULT_INTERFACE}" -j ACCEPT

ip tuntap add dev tap1 mode tap user "${USER}" || true
ip link set tap1 up promisc on
ip link set tap1 master br0

iptables -C FORWARD -i tap1 -o "${DEFAULT_INTERFACE}" -j ACCEPT || iptables -I FORWARD -i tap1 -o "${DEFAULT_INTERFACE}" -j ACCEPT

#
# ovmf
#

cp AMDSEV/usr/local/share/qemu/OVMF_VARS.fd OVMF_VARS_0.fd
cp AMDSEV/usr/local/share/qemu/OVMF_VARS.fd OVMF_VARS_1.fd

#
# swtpm
#

mkdir "${PWD}"/tpm0 || true
swtpm_setup --tpm2 --tpmstate "${PWD}/tpm0" --create-ek-cert --create-platform-cert --allow-signing --overwrite --pcr-banks - --logfile "${PWD}/tpm0/setup.log"
swtpm socket --tpm2 --tpmstate dir="${PWD}/tpm0",mode=0600 --ctrl type=unixio,path="${PWD}/tpm0/swtpm-sock" --log file="${PWD}/tpm0/tpm.log",level=20,truncate --pid file="${PWD}/swtpm0.pid" &

mkdir "${PWD}"/tpm1 || true
swtpm_setup --tpm2 --tpmstate "${PWD}/tpm1" --create-ek-cert --create-platform-cert --allow-signing --overwrite --pcr-banks - --logfile "${PWD}/tpm1/setup.log"
swtpm socket --tpm2 --tpmstate dir="${PWD}/tpm1",mode=0600 --ctrl type=unixio,path="${PWD}/tpm1/swtpm-sock" --log file="${PWD}/tpm1/tpm.log",level=20,truncate --pid file="${PWD}/swtpm1.pid" &

tmux new-session -d -s const-sess

tmux split-window
tmux split-window

launch_cmd_base_sev="AMDSEV/usr/local/bin/qemu-system-x86_64 \
	-enable-kvm \
	-cpu EPYC-v4 \
	-machine q35,smm=off \
	-smp 4,maxcpus=255 \
	-m 2048M,slots=5,maxmem=$((2048 + 8192))M \
	-no-reboot \
	-bios AMDSEV/usr/local/share/qemu/OVMF_CODE.fd \
	-drive file=./image.raw,if=none,id=disk1,format=raw,readonly=on \
	-device virtio-blk-pci,drive=disk1,id=virtio-disk1,disable-legacy=on,iommu_platform=true,bootindex=1 \
	-machine memory-encryption=sev0,vmport=off \
	-object memory-backend-memfd,id=ram1,size=2048M,share=true,prealloc=false \
	-machine memory-backend=ram1 \
	-object sev-snp-guest,id=sev0,cbitpos=51,reduced-phys-bits=1 \
	-nographic  \
	-device virtio-blk-pci,drive=disk2,id=virtio-disk2 \
	-tpmdev emulator,id=tpm0,chardev=chrtpm \
	-device tpm-crb,tpmdev=tpm0"

# shellcheck disable=2034
launch_cmd_base_no_sev="AMDSEV/usr/local/bin/qemu-system-x86_64 \
	-enable-kvm \
	-cpu EPYC-v4 \
	-machine q35 \
	-smp 1,maxcpus=255 \
	-m 2048M,slots=5,maxmem=10240M \
	-no-reboot \
	-drive if=pflash,format=raw,unit=0,file=${PWD}/OVMF_CODE.fd,readonly=true \
	-drive file=./image.raw,if=none,id=disk1,format=raw,readonly=on \
	-device virtio-blk-pci,drive=disk1,id=virtio-disk1,disable-legacy=on,iommu_platform=true,bootindex=1 \
	-nographic  \
	-device virtio-blk-pci,drive=disk2,id=virtio-disk2 \
	-tpmdev emulator,id=tpm0,chardev=chrtpm \
	-device tpm-crb,tpmdev=tpm0"

launch_cmd_0="${launch_cmd_base_sev} \
	-drive if=pflash,format=raw,unit=0,file=${PWD}/OVMF_VARS_0.fd \
	-device virtio-net,netdev=network0,mac=${mac_0} \
	-netdev tap,id=network0,ifname=tap0,script=no,downscript=no \
	-drive file=./disk0.img,id=disk2,if=none,format=raw \
	-chardev socket,id=chrtpm,path=${PWD}/tpm0/swtpm-sock"
launch_cmd_1="${launch_cmd_base_sev} \
	-drive if=pflash,format=raw,unit=0,file=${PWD}/OVMF_VARS_1.fd \
	-device virtio-net,netdev=network0,mac=${mac_1} \
	-netdev tap,id=network0,ifname=tap1,script=no,downscript=no \
	-drive file=./disk1.img,id=disk2,if=none,format=raw \
	-chardev socket,id=chrtpm,path=${PWD}/tpm1/swtpm-sock"

init_cmd="./constellation apply --skip-phases infrastructure"

tmux send -t const-sess:0.0 "${launch_cmd_0}" ENTER
sleep 3
tmux send -t const-sess:0.1 "${launch_cmd_1}" ENTER
tmux send -t const-sess:0.2 "${init_cmd}" ENTER

tmux a -t const-sess