libvirt: build containerized libvirt as nix container image

.github/workflows/build-libvirt-container.yml

@@ -0,0 +1,44 @@
+name: Build libvirtd base container
+
+on:
+  push:
+    branches:
+       - "main"
+    paths:
+      - "flake.nix"
+      - "flake.lock"
+      - "nix/containers/libvirtd_base.nix"
+      - ".github/workflows/build-libvirt-container.yml"
+  workflow_dispatch:
+
+jobs:
+  build-container:
+    runs-on: ubuntu-22.04
+    permissions:
+      contents: read
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0
+
+      - name: Setup bazel
+        uses: ./.github/actions/setup_bazel_nix
+        with:
+          useCache: "false"
+          nixTools: |
+            crane
+            gzip
+
+      - name: Log in to the Container registry
+        uses: ./.github/actions/container_registry_login
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build container
+        run: |
+          nix build .#libvirtd_base
+          gunzip < result > libvirtd_base.tar
+          crane push libvirtd_base.tar ghcr.io/edgelesssys/constellation/libvirtd-base
+          rm -f libvirtd_base.tar

bazel/oci/containers.bzl

@@ -51,7 +51,7 @@ def containers():
             "identifier": "libvirt",
             "image_name": "libvirt",
             "name": "libvirt",
-            "oci": "//cli/internal/libvirt:constellation_libvirt",
+            "oci": "@libvirtd_base//:libvirtd_base",
             "repotag_file": "//bazel/release:libvirt_tag.txt",
             "used_by": ["config"],
         },

bazel/toolchains/container_images.bzl

@@ -1,5 +1,5 @@
 """
-This file contains external container images used by the project.
+This file contains container images that are pulled from container registries.
 """
 
 load("@rules_oci//oci:pull.bzl", "oci_pull")
@@ -14,3 +14,8 @@ def containter_image_deps():
             "linux/arm64",
         ],
     )
+    oci_pull(
+        name = "libvirtd_base",
+        digest = "sha256:f5aca956c8d67059725feb4bf8a7d96da71a51efe84288c74a52fcf6855a13bd",
+        image = "ghcr.io/edgelesssys/constellation/libvirtd-base",
+    )

cli/internal/libvirt/BUILD.bazel

@@ -1,7 +1,4 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_library")
-load("@rules_oci//oci:defs.bzl", "oci_image")
-load("@rules_pkg//:pkg.bzl", "pkg_tar")
-load("@rules_pkg//pkg:mappings.bzl", "pkg_attributes", "pkg_files", "strip_prefix")
 
 go_library(
     name = "libvirt",
@@ -17,60 +14,3 @@ go_library(
         "@com_github_spf13_afero//:afero",
     ],
 )
-
-pkg_files(
-    name = "etc",
-    srcs = [
-        "//cli/internal/libvirt/etc:passwd_db",
-    ],
-    attributes = pkg_attributes(
-        group = "root",
-        mode = "0644",
-        owner = "root",
-    ),
-    prefix = "etc",
-    strip_prefix = strip_prefix.from_pkg(),
-)
-
-pkg_files(
-    name = "nvram",
-    srcs = [
-        "//cli/internal/libvirt/nvram:nvram_vars",
-    ],
-    prefix = "usr/share/OVMF",
-    strip_prefix = strip_prefix.from_pkg(),
-)
-
-pkg_files(
-    name = "libvirt_conf",
-    srcs = [
-        "libvirtd.conf",
-        "qemu.conf",
-    ],
-    prefix = "/etc/libvirt",
-)
-
-pkg_tar(
-    name = "start",
-    srcs = [
-        "start.sh",
-        ":etc",
-        ":libvirt_conf",
-        ":nvram",
-    ],
-    mode = "0755",
-)
-
-oci_image(
-    name = "constellation_libvirt",
-    architecture = "amd64",
-    entrypoint = ["/start.sh"],
-    os = "linux",
-    tars = [
-        # TODO(malt3): test if libvirt works before merging this change!!!
-        "@libvirt_x86_64-linux//:closure.tar",
-        "@libvirt_x86_64-linux//:bin-linktree.tar",
-        ":start",
-    ],
-    visibility = ["//visibility:public"],
-)

cli/internal/libvirt/etc/BUILD.bazel

@@ -1,8 +0,0 @@
-filegroup(
-    name = "passwd_db",
-    srcs = glob(
-        ["**/*"],
-        exclude = ["BUILD"],
-    ),
-    visibility = ["//visibility:public"],
-)

cli/internal/libvirt/etc/group

@@ -1,51 +0,0 @@
-root:x:0:
-bin:x:1:
-daemon:x:2:
-sys:x:3:
-adm:x:4:
-tty:x:5:
-disk:x:6:
-lp:x:7:
-mem:x:8:
-kmem:x:9:
-wheel:x:10:
-cdrom:x:11:
-mail:x:12:
-man:x:15:
-dialout:x:18:
-floppy:x:19:
-games:x:20:
-tape:x:33:
-video:x:39:
-ftp:x:50:
-lock:x:54:
-audio:x:63:
-users:x:100:
-nobody:x:65534:
-tss:x:59:
-dbus:x:81:
-unbound:x:999:
-utmp:x:22:
-utempter:x:35:
-saslauth:x:76:saslauth
-input:x:104:
-kvm:x:36:qemu
-render:x:105:
-sgx:x:106:
-systemd-journal:x:190:
-systemd-network:x:192:
-systemd-oom:x:997:
-systemd-resolve:x:193:
-polkitd:x:996:
-rtkit:x:172:
-gluster:x:995:
-dnsmasq:x:994:
-rpc:x:32:
-brlapi:x:993:
-rpcuser:x:29:
-qemu:x:107:
-pipewire:x:992:
-geoclue:x:991:
-libvirt:x:990:
-systemd-coredump:x:989:
-systemd-timesync:x:988:

cli/internal/libvirt/etc/passwd

@@ -1,31 +0,0 @@
-root:x:0:0:root:/root:/bin/bash
-bin:x:1:1:bin:/bin:/sbin/nologin
-daemon:x:2:2:daemon:/sbin:/sbin/nologin
-adm:x:3:4:adm:/var/adm:/sbin/nologin
-lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
-sync:x:5:0:sync:/sbin:/bin/sync
-shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
-halt:x:7:0:halt:/sbin:/sbin/halt
-mail:x:8:12:mail:/var/spool/mail:/sbin/nologin
-operator:x:11:0:operator:/root:/sbin/nologin
-games:x:12:100:games:/usr/games:/sbin/nologin
-ftp:x:14:50:FTP User:/var/ftp:/sbin/nologin
-nobody:x:65534:65534:Kernel Overflow User:/:/sbin/nologin
-tss:x:59:59:Account used for TPM access:/:/usr/sbin/nologin
-dbus:x:81:81:System message bus:/:/sbin/nologin
-unbound:x:999:999:Unbound DNS resolver:/var/lib/unbound:/sbin/nologin
-saslauth:x:998:76:Saslauthd user:/run/saslauthd:/sbin/nologin
-systemd-network:x:192:192:systemd Network Management:/:/usr/sbin/nologin
-systemd-oom:x:997:997:systemd Userspace OOM Killer:/:/usr/sbin/nologin
-systemd-resolve:x:193:193:systemd Resolver:/:/usr/sbin/nologin
-polkitd:x:996:996:User for polkitd:/:/sbin/nologin
-rtkit:x:172:172:RealtimeKit:/proc:/sbin/nologin
-gluster:x:995:995:GlusterFS daemons:/run/gluster:/sbin/nologin
-dnsmasq:x:994:994:Dnsmasq DHCP and DNS server:/var/lib/dnsmasq:/usr/sbin/nologin
-rpc:x:32:32:Rpcbind Daemon:/var/lib/rpcbind:/sbin/nologin
-rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin
-qemu:x:107:107:qemu user:/:/sbin/nologin
-pipewire:x:993:992:PipeWire System Daemon:/var/run/pipewire:/sbin/nologin
-geoclue:x:992:991:User for geoclue:/var/lib/geoclue:/sbin/nologin
-systemd-coredump:x:989:989:systemd Core Dumper:/:/usr/sbin/nologin
-systemd-timesync:x:988:988:systemd Time Synchronization:/:/usr/sbin/nologin

cli/internal/libvirt/libvirtd.conf

@@ -1,5 +0,0 @@
-listen_tls = 0
-listen_tcp = 1
-tcp_port = "16599"
-listen_addr = "localhost"
-auth_tcp = "none"

cli/internal/libvirt/nvram/BUILD.bazel

@@ -1,8 +0,0 @@
-filegroup(
-    name = "nvram_vars",
-    srcs = glob(
-        ["**/*.fd"],
-        exclude = ["BUILD"],
-    ),
-    visibility = ["//visibility:public"],
-)

cli/internal/libvirt/qemu.conf

@@ -1 +0,0 @@
-cgroup_controllers = []

cli/internal/libvirt/start.sh

@@ -1,20 +0,0 @@
-#!/usr/bin/env bash
-
-set -euo pipefail
-shopt -s inherit_errexit
-
-# ensure library cache is up to date
-ldconfig
-
-chown -R tss:root /var/lib/swtpm-localca
-
-# Assign qemu the GID of the host system's 'kvm' group to avoid permission issues for environments defaulting to 660 for /dev/kvm (e.g. Debian-based distros)
-KVM_HOST_GID="$(stat -c '%g' /dev/kvm)"
-groupadd -o -g "${KVM_HOST_GID}" host-kvm
-usermod -a -G host-kvm qemu
-
-# Start libvirt daemon
-libvirtd --daemon --listen
-virtlogd --daemon
-
-sleep infinity

dev-docs/workflows/qemu.md

@@ -11,7 +11,7 @@ You may either use [your local libvirt setup](#local-libvirt-setup) if it meets
 ## Containerized libvirt
 
 Constellation will automatically deploy a containerized libvirt instance, if no connection URI is defined in the Constellation config file.
-Follow the steps in our [libvirt readme](../../cli/internal/libvirt/README.md) if you wish to build your own image.
+Follow the steps in our [libvirt readme](../../nix/container/README.md) if you wish to build your own image.
 
 ## Local libvirt setup
 

flake.nix

@@ -49,6 +49,8 @@
 
       packages.libvirt = callPackage ./nix/cc/libvirt.nix { pkgs = pkgsUnstable; pkgsLinux = import nixpkgsUnstable { system = "x86_64-linux"; }; };
 
+      packages.libvirtd_base = callPackage ./nix/container/libvirtd_base.nix { pkgs = pkgsUnstable; pkgsLinux = import nixpkgsUnstable { system = "x86_64-linux"; }; };
+
       packages.awscli2 = pkgsUnstable.awscli2;
 
       packages.bazel_6 = pkgsUnstable.bazel_6;

nix/container/README.md

@@ -11,14 +11,19 @@ Connecting to the libvirt daemon running in the container and manage the deploym
 virsh -c "qemu+tcp://localhost:16599/system"
 ```
 
-## Docker image
+## Container image
 
-Build the image:
+Update the base image (`ghcr.io/edgelesssys/constellation/libvirtd-base`):
+
+```shell
+nix build .#libvirtd_base
+cat result | gunzip > libvirtd_base.tar
+crane push libvirtd_base.tar ghcr.io/edgelesssys/constellation/libvirtd-base
+```
+
+Push the final image to your own registry (`ghcr.io/<USERNAME>/constellation/libvirtd`):
 
 ```shell
-bazel build //cli/internal/libvirt:constellation_libvirt
-bazel build //bazel/release:libvirt_sum
-bazel build //bazel/release:libvirt_tar
 bazel run //bazel/release:libvirt_push
 ```
 

nix/container/libvirtd_base.nix

@@ -0,0 +1,139 @@
+{ pkgs
+, pkgsLinux
+, stdenv
+}:
+let
+  passwd = pkgs.writeTextDir "etc/passwd" ''
+    root:x:0:0:root:/root:/bin/sh
+    bin:x:1:1:bin:/bin:/sbin/nologin
+    daemon:x:2:2:daemon:/sbin:/sbin/nologin
+    adm:x:3:4:adm:/var/adm:/sbin/nologin
+    lp:x:4:7:lp:/var/spool/lpd:/sbin/nologin
+    sync:x:5:0:sync:/sbin:/bin/sync
+    shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown
+    halt:x:7:0:halt:/sbin:/sbin/halt
+    nobody:x:65534:65534:Kernel Overflow User:/:/sbin/nologin
+    tss:x:59:59:Account used for TPM access:/:/usr/sbin/nologin
+    saslauth:x:998:76:Saslauthd user:/run/saslauthd:/sbin/nologin
+    polkitd:x:996:996:User for polkitd:/:/sbin/nologin
+    dnsmasq:x:994:994:Dnsmasq DHCP and DNS server:/var/lib/dnsmasq:/usr/sbin/nologin
+    rpc:x:32:32:Rpcbind Daemon:/var/lib/rpcbind:/sbin/nologin
+    rpcuser:x:29:29:RPC Service User:/var/lib/nfs:/sbin/nologin
+    qemu:x:107:107:qemu user:/:/sbin/nologin
+  '';
+  group = pkgs.writeTextDir "etc/group" ''
+    root:x:0:
+    bin:x:1:
+    daemon:x:2:
+    sys:x:3:
+    adm:x:4:
+    tty:x:5:
+    disk:x:6:
+    lp:x:7:
+    mem:x:8:
+    kmem:x:9:
+    wheel:x:10:
+    lock:x:54:
+    users:x:100:
+    nobody:x:65534:
+    tss:x:59:
+    utmp:x:22:
+    utempter:x:35:
+    saslauth:x:76:saslauth
+    input:x:104:
+    kvm:x:36:qemu
+    sgx:x:106:
+    polkitd:x:996:
+    dnsmasq:x:994:
+    rpc:x:32:
+    rpcuser:x:29:
+    qemu:x:107:
+    libvirt:x:990:
+  '';
+  libvirtdConf = pkgs.writeTextDir "etc/libvirt/libvirtd.conf" ''
+    listen_tls = 0
+    listen_tcp = 1
+    tcp_port = "16599"
+    listen_addr = "localhost"
+    auth_tcp = "none"
+  '';
+  qemuConf = pkgs.writeTextDir "var/lib/libvirt/qemu.conf" ''
+    cgroup_controllers = []
+  '';
+  startScript = pkgsLinux.writeShellApplication {
+    name = "start.sh";
+    runtimeInputs = with pkgsLinux; [
+      shadow
+      coreutils
+      libvirt
+      qemu
+      swtpm
+    ];
+    text = ''
+      set -euo pipefail
+      shopt -s inherit_errexit
+
+      # Assign qemu the GID of the host system's 'kvm' group to avoid permission issues for environments defaulting to 660 for /dev/kvm (e.g. Debian-based distros)
+      KVM_HOST_GID="$(stat -c '%g' /dev/kvm)"
+
+      groupadd -o -g "''${KVM_HOST_GID}" host-kvm || true
+      usermod -a -G host-kvm qemu || true
+
+      # Start libvirt daemon
+      libvirtd -f /etc/libvirt/libvirtd.conf --daemon --listen
+      virtlogd --daemon
+
+      sleep infinity
+    '';
+  };
+  ovmf = stdenv.mkDerivation {
+    name = "OVMF";
+    postInstall = ''
+      mkdir -p $out/usr/share/
+      ln -s ${pkgsLinux.OVMFFull.fd}/FV  $out/usr/share/OVMF
+    '';
+    propagatedBuildInputs = with pkgsLinux; [
+      OVMF
+    ];
+    dontUnpack = true;
+  };
+in
+pkgs.dockerTools.buildImage {
+  name = "ghcr.io/edgelesssys/constellation/libvirtd-base";
+  copyToRoot = with pkgsLinux.dockerTools; [
+    passwd
+    group
+    libvirtdConf
+    qemuConf
+    ovmf
+    startScript
+    usrBinEnv
+    caCertificates
+    pkgsLinux.busybox
+  ];
+  config = {
+    Cmd = [ "/bin/start.sh" ];
+  };
+  runAsRoot = ''
+    #!${pkgs.runtimeShell}
+    mkdir -p /tmp
+    mkdir -p /run
+    mkdir -p /var/lock
+    mkdir -p /var/log/libvirt
+    mkdir -p /var/lib/swtpm-localca
+    mkdir -p /var/lib/libvirt/boot
+    mkdir -p /var/lib/libvirt/dnsmasq
+    mkdir -p /var/lib/libvirt/filesystems
+    mkdir -p /var/lib/libvirt/images
+    mkdir -p /var/lib/libvirt/libxl
+    mkdir -p /var/lib/libvirt/lxc
+    mkdir -p /var/lib/libvirt/network
+    mkdir -p /var/lib/libvirt/qemu
+    mkdir -p /var/lib/libvirt/swtpm
+
+    chmod 1777 /tmp
+    chown -R tss:root /var/lib/swtpm-localca
+    chown -R qemu:qemu /var/lib/libvirt/qemu
+    chown -R root:libvirt /var/log/libvirt/
+  '';
+}

terraform/infrastructure/qemu/variables.tf

@@ -67,7 +67,7 @@ variable "image_format" {
 }
 variable "firmware" {
   type        = string
-  default     = "/usr/share/OVMF/OVMF_CODE.secboot.fd"
+  default     = "/usr/share/OVMF/OVMF_CODE.fd"
   description = "path to UEFI firmware file. Use \"OVMF_CODE_4M.ms.fd\" on Ubuntu and \"OVMF_CODE.fd\" or \"OVMF_CODE.secboot.fd\" on Fedora."
 }