Git-Mirrors/constellation
1
0
Fork 0
mirror of https://github.com/edgelesssys/constellation.git synced 2024-12-24 06:59:40 -05:00
Code Issues Packages Projects Releases Wiki Activity

libvirt: build containerized libvirt as nix container image

Browse Source
This commit is contained in:
Malte Poll 2023-11-28 10:52:37 +01:00
parent fb735419ac
commit cd6e03049a
18 changed files with 204 additions and 193 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

44
.github/workflows/build-libvirt-container.yml vendored Normal file
View File

@ -0,0 +1,44 @@
name: Build libvirtd base container
on:
push:
branches:
- "main"
paths:
- "flake.nix"
- "flake.lock"
- "nix/containers/libvirtd_base.nix"
- ".github/workflows/build-libvirt-container.yml"
workflow_dispatch:
jobs:
build-container:
runs-on: ubuntu-22.04
permissions:
contents: read
packages: write
steps:
- name: Checkout
uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
- name: Setup bazel
uses: ./.github/actions/setup_bazel_nix
with:
useCache: "false"
nixTools: |
crane
gzip
- name: Log in to the Container registry
uses: ./.github/actions/container_registry_login
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build container
run: |
nix build .#libvirtd_base
gunzip < result > libvirtd_base.tar
crane push libvirtd_base.tar ghcr.io/edgelesssys/constellation/libvirtd-base
rm -f libvirtd_base.tar

2
bazel/oci/containers.bzl
View File

@ -51,7 +51,7 @@ def containers():
"identifier": "libvirt",
"image_name": "libvirt",
"name": "libvirt",
"oci": "//cli/internal/libvirt:constellation_libvirt",
"oci": "@libvirtd_base//:libvirtd_base",
"repotag_file": "//bazel/release:libvirt_tag.txt",
"used_by": ["config"],
},

7
bazel/toolchains/container_images.bzl
View File

@ -1,5 +1,5 @@
"""
This file contains external container images used by the project.
This file contains container images that are pulled from container registries.
"""
load("@rules_oci//oci:pull.bzl", "oci_pull")
@ -14,3 +14,8 @@ def containter_image_deps():
"linux/arm64",
],
)
oci_pull(
name = "libvirtd_base",
digest = "sha256:f5aca956c8d67059725feb4bf8a7d96da71a51efe84288c74a52fcf6855a13bd",
image = "ghcr.io/edgelesssys/constellation/libvirtd-base",
)

60
cli/internal/libvirt/BUILD.bazel
View File

@ -1,7 +1,4 @@
load("@io_bazel_rules_go//go:def.bzl", "go_library")
load("@rules_oci//oci:defs.bzl", "oci_image")
load("@rules_pkg//:pkg.bzl", "pkg_tar")
load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", "strip_prefix")
go_library(
name = "libvirt",
@ -17,60 +14,3 @@ go_library(
"@com_github_spf13_afero//:afero",
],
)
pkg_files(
name = "etc",
srcs = [
"//cli/internal/libvirt/etc:passwd_db",
],
attributes = pkg_attributes(
group = "root",
mode = "0644",
owner = "root",
),
prefix = "etc",
strip_prefix = strip_prefix.from_pkg(),
)
pkg_files(
name = "nvram",
srcs = [
"//cli/internal/libvirt/nvram:nvram_vars",
],
prefix = "usr/share/OVMF",
strip_prefix = strip_prefix.from_pkg(),
)
pkg_files(
name = "libvirt_conf",
srcs = [
"libvirtd.conf",
"qemu.conf",
],
prefix = "/etc/libvirt",
)
pkg_tar(
name = "start",
srcs = [
"start.sh",
":etc",
":libvirt_conf",
":nvram",
],
mode = "0755",
)
oci_image(
name = "constellation_libvirt",
architecture = "amd64",
entrypoint = ["/start.sh"],
os = "linux",
tars = [
# TODO(malt3): test if libvirt works before merging this change!!!
"@libvirt_x86_64-linux//:closure.tar",
"@libvirt_x86_64-linux//:bin-linktree.tar",
":start",
],
visibility = ["//visibility:public"],
)

8
cli/internal/libvirt/etc/BUILD.bazel
View File

@ -1,8 +0,0 @@
filegroup(
name = "passwd_db",
srcs = glob(
["**/*"],
exclude = ["BUILD"],
),
visibility = ["//visibility:public"],
)

51
cli/internal/libvirt/etc/group
View File

@ -1,51 +0,0 @@
root:x:0:
bin:x:1:
daemon:x:2:
sys:x:3:
adm:x:4:
tty:x:5:
disk:x:6:
lp:x:7:
mem:x:8:
kmem:x:9:
wheel:x:10:
cdrom:x:11:
mail:x:12:
man:x:15:
dialout:x:18:
floppy:x:19:
games:x:20:
tape:x:33:
video:x:39:
ftp:x:50:
lock:x:54:
audio:x:63:
users:x:100:
nobody:x:65534:
tss:x:59:
dbus:x:81:
unbound:x:999:
utmp:x:22:
utempter:x:35:
saslauth:x:76:saslauth
input:x:104:
kvm:x:36:qemu
render:x:105:
sgx:x:106:
systemd-journal:x:190:
systemd-network:x:192:
systemd-oom:x:997:
systemd-resolve:x:193:
polkitd:x:996:
rtkit:x:172:
gluster:x:995:
dnsmasq:x:994:
rpc:x:32:
brlapi:x:993:
rpcuser:x:29:
qemu:x:107:
pipewire:x:992:
geoclue:x:991:
libvirt:x:990:
systemd-coredump:x:989:
systemd-timesync:x:988:

31
cli/internal/libvirt/etc/passwd
View File

@ -1,31 +0,0 @@
root:x:0:0:root:/root:/bin/bash
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
sync:x:5:0:sync:/sbin:/bin/sync
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
halt:x:7:0:halt:/sbin:/sbin/halt
mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
operator:x:11:0:operator:/root:/sbin/nologin
games:x:12:100:games:/usr/games:/sbin/nologin
ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
nobody:x:65534:65534:Kernel Overflow User:/:/sbin/nologin
tss:x:59:59:Account used for TPM access:/:/usr/sbin/nologin
dbus:x:81:81:System message bus:/:/sbin/nologin
unbound:x:999:999:Unbound DNS resolver:/var/lib/unbound:/sbin/nologin
saslauth:x:998:76:Saslauthd user:/run/saslauthd:/sbin/nologin
systemd-network:x:192:192:systemd Network Management:/:/usr/sbin/nologin
systemd-oom:x:997:997:systemd Userspace OOM Killer:/:/usr/sbin/nologin
systemd-resolve:x:193:193:systemd Resolver:/:/usr/sbin/nologin
polkitd:x:996:996:User for polkitd:/:/sbin/nologin
rtkit:x:172:172:RealtimeKit:/proc:/sbin/nologin
gluster:x:995:995:GlusterFS daemons:/run/gluster:/sbin/nologin
dnsmasq:x:994:994:Dnsmasq DHCP and DNS server:/var/lib/dnsmasq:/usr/sbin/nologin
rpc:x:32:32:Rpcbind Daemon:/var/lib/rpcbind:/sbin/nologin
rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin
qemu:x:107:107:qemu user:/:/sbin/nologin
pipewire:x:993:992:PipeWire System Daemon:/var/run/pipewire:/sbin/nologin
geoclue:x:992:991:User for geoclue:/var/lib/geoclue:/sbin/nologin
systemd-coredump:x:989:989:systemd Core Dumper:/:/usr/sbin/nologin
systemd-timesync:x:988:988:systemd Time Synchronization:/:/usr/sbin/nologin

5
cli/internal/libvirt/libvirtd.conf
View File

@ -1,5 +0,0 @@
listen_tls = 0
listen_tcp = 1
tcp_port = "16599"
listen_addr = "localhost"
auth_tcp = "none"

8
cli/internal/libvirt/nvram/BUILD.bazel
View File

@ -1,8 +0,0 @@
filegroup(
name = "nvram_vars",
srcs = glob(
["**/*.fd"],
exclude = ["BUILD"],
),
visibility = ["//visibility:public"],
)

BIN
cli/internal/libvirt/nvram/constellation_vars.production.fd
View File

Binary file not shown.

BIN
cli/internal/libvirt/nvram/constellation_vars.testing.fd
View File

Binary file not shown.

1
cli/internal/libvirt/qemu.conf
View File

@ -1 +0,0 @@
cgroup_controllers = []

20
cli/internal/libvirt/start.sh
View File

@ -1,20 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
shopt -s inherit_errexit
# ensure library cache is up to date
ldconfig
chown -R tss:root /var/lib/swtpm-localca
# Assign qemu the GID of the host system's 'kvm' group to avoid permission issues for environments defaulting to 660 for /dev/kvm (e.g. Debian-based distros)
KVM_HOST_GID="$(stat -c '%g' /dev/kvm)"
groupadd -o -g "${KVM_HOST_GID}" host-kvm
usermod -a -G host-kvm qemu
# Start libvirt daemon
libvirtd --daemon --listen
virtlogd --daemon
sleep infinity

2
dev-docs/workflows/qemu.md
View File

@ -11,7 +11,7 @@ You may either use [your local libvirt setup](#local-libvirt-setup) if it meets
## Containerized libvirt
Constellation will automatically deploy a containerized libvirt instance, if no connection URI is defined in the Constellation config file.
Follow the steps in our [libvirt readme](../../cli/internal/libvirt/README.md) if you wish to build your own image.
Follow the steps in our [libvirt readme](../../nix/container/README.md) if you wish to build your own image.
## Local libvirt setup

2
flake.nix
View File

@ -49,6 +49,8 @@
packages.libvirt = callPackage ./nix/cc/libvirt.nix { pkgs = pkgsUnstable; pkgsLinux = import nixpkgsUnstable { system = "x86_64-linux"; }; };
packages.libvirtd_base = callPackage ./nix/container/libvirtd_base.nix { pkgs = pkgsUnstable; pkgsLinux = import nixpkgsUnstable { system = "x86_64-linux"; }; };
packages.awscli2 = pkgsUnstable.awscli2;
packages.bazel_6 = pkgsUnstable.bazel_6;

15
cli/internal/libvirt/README.md → nix/container/README.md
View File

@ -11,14 +11,19 @@ Connecting to the libvirt daemon running in the container and manage the deploym
virsh -c "qemu+tcp://localhost:16599/system"
```
## Docker image
## Container image
Build the image:
Update the base image (`ghcr.io/edgelesssys/constellation/libvirtd-base`):
```shell
nix build .#libvirtd_base
cat result | gunzip > libvirtd_base.tar
crane push libvirtd_base.tar ghcr.io/edgelesssys/constellation/libvirtd-base
```
Push the final image to your own registry (`ghcr.io/<USERNAME>/constellation/libvirtd`):
```shell
bazel build //cli/internal/libvirt:constellation_libvirt
bazel build //bazel/release:libvirt_sum
bazel build //bazel/release:libvirt_tar
bazel run //bazel/release:libvirt_push
```

139
nix/container/libvirtd_base.nix Normal file
View File

@ -0,0 +1,139 @@
{ pkgs
, pkgsLinux
, stdenv
}:
let
passwd = pkgs.writeTextDir "etc/passwd" ''
root:x:0:0:root:/root:/bin/sh
bin:x:1:1:bin:/bin:/sbin/nologin
daemon:x:2:2:daemon:/sbin:/sbin/nologin
adm:x:3:4:adm:/var/adm:/sbin/nologin
lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
sync:x:5:0:sync:/sbin:/bin/sync
shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
halt:x:7:0:halt:/sbin:/sbin/halt
nobody:x:65534:65534:Kernel Overflow User:/:/sbin/nologin
tss:x:59:59:Account used for TPM access:/:/usr/sbin/nologin
saslauth:x:998:76:Saslauthd user:/run/saslauthd:/sbin/nologin
polkitd:x:996:996:User for polkitd:/:/sbin/nologin
dnsmasq:x:994:994:Dnsmasq DHCP and DNS server:/var/lib/dnsmasq:/usr/sbin/nologin
rpc:x:32:32:Rpcbind Daemon:/var/lib/rpcbind:/sbin/nologin
rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin
qemu:x:107:107:qemu user:/:/sbin/nologin
'';
group = pkgs.writeTextDir "etc/group" ''
root:x:0:
bin:x:1:
daemon:x:2:
sys:x:3:
adm:x:4:
tty:x:5:
disk:x:6:
lp:x:7:
mem:x:8:
kmem:x:9:
wheel:x:10:
lock:x:54:
users:x:100:
nobody:x:65534:
tss:x:59:
utmp:x:22:
utempter:x:35:
saslauth:x:76:saslauth
input:x:104:
kvm:x:36:qemu
sgx:x:106:
polkitd:x:996:
dnsmasq:x:994:
rpc:x:32:
rpcuser:x:29:
qemu:x:107:
libvirt:x:990:
'';
libvirtdConf = pkgs.writeTextDir "etc/libvirt/libvirtd.conf" ''
listen_tls = 0
listen_tcp = 1
tcp_port = "16599"
listen_addr = "localhost"
auth_tcp = "none"
'';
qemuConf = pkgs.writeTextDir "var/lib/libvirt/qemu.conf" ''
cgroup_controllers = []
'';
startScript = pkgsLinux.writeShellApplication {
name = "start.sh";
runtimeInputs = with pkgsLinux; [
shadow
coreutils
libvirt
qemu
swtpm
];
text = ''
set -euo pipefail
shopt -s inherit_errexit
# Assign qemu the GID of the host system's 'kvm' group to avoid permission issues for environments defaulting to 660 for /dev/kvm (e.g. Debian-based distros)
KVM_HOST_GID="$(stat -c '%g' /dev/kvm)"
groupadd -o -g "''${KVM_HOST_GID}" host-kvm || true
usermod -a -G host-kvm qemu || true
# Start libvirt daemon
libvirtd -f /etc/libvirt/libvirtd.conf --daemon --listen
virtlogd --daemon
sleep infinity
'';
};
ovmf = stdenv.mkDerivation {
name = "OVMF";
postInstall = ''
mkdir -p $out/usr/share/
ln -s ${pkgsLinux.OVMFFull.fd}/FV $out/usr/share/OVMF
'';
propagatedBuildInputs = with pkgsLinux; [
OVMF
];
dontUnpack = true;
};
in
pkgs.dockerTools.buildImage {
name = "ghcr.io/edgelesssys/constellation/libvirtd-base";
copyToRoot = with pkgsLinux.dockerTools; [
passwd
group
libvirtdConf
qemuConf
ovmf
startScript
usrBinEnv
caCertificates
pkgsLinux.busybox
];
config = {
Cmd = [ "/bin/start.sh" ];
};
runAsRoot = ''
#!${pkgs.runtimeShell}
mkdir -p /tmp
mkdir -p /run
mkdir -p /var/lock
mkdir -p /var/log/libvirt
mkdir -p /var/lib/swtpm-localca
mkdir -p /var/lib/libvirt/boot
mkdir -p /var/lib/libvirt/dnsmasq
mkdir -p /var/lib/libvirt/filesystems
mkdir -p /var/lib/libvirt/images
mkdir -p /var/lib/libvirt/libxl
mkdir -p /var/lib/libvirt/lxc
mkdir -p /var/lib/libvirt/network
mkdir -p /var/lib/libvirt/qemu
mkdir -p /var/lib/libvirt/swtpm
chmod 1777 /tmp
chown -R tss:root /var/lib/swtpm-localca
chown -R qemu:qemu /var/lib/libvirt/qemu
chown -R root:libvirt /var/log/libvirt/
'';
}

2
terraform/infrastructure/qemu/variables.tf
View File

@ -67,7 +67,7 @@ variable "image_format" {
}
variable "firmware" {
type = string
default = "/usr/share/OVMF/OVMF_CODE.secboot.fd"
default = "/usr/share/OVMF/OVMF_CODE.fd"
description = "path to UEFI firmware file. Use \"OVMF_CODE_4M.ms.fd\" on Ubuntu and \"OVMF_CODE.fd\" or \"OVMF_CODE.secboot.fd\" on Fedora."
}