docs: how to reproduce released artifacts (#3451)

* ci: test reproducability with different dependency installation methods

* nix: mitigate nix store optimisiation

* docs: reproducible builds

Co-authored-by: Daniel Weiße <66256922+daniel-weisse@users.noreply.github.com>

Co-authored-by: Thomas Tendyck <51411342+thomasten@users.noreply.github.com>

* ci: upgrade ubuntu runners for reproducible builds

---------

Co-authored-by: Thomas Tendyck <51411342+thomasten@users.noreply.github.com>

.github/workflows/reproducible-builds.yml

@@ -1,8 +1,9 @@
 # Verify that Constellation builds are reproducible.
 #
-# The build-* jobs' matrix has two dimensions: a list of targets to build and
+# The build-* jobs' matrix has three dimensions: a list of targets to build, a
-# a list of runners to build on. The produced binaries and OS images are
+# list of runners to build on and a method of installing dependencies. The
-# expected to be bit-for-bit identical, regardless of the chosen build runner.
+# produced binaries and OS images are expected to be bit-for-bit identical,
+# without any dependencies on the runtime setup details.
 #
 # The compare-* jobs only have the target dimension. They obtain the built
 # targets from all runners and check that there are no diffs between them.
@@ -24,10 +25,15 @@ jobs:
             - "cli_enterprise_linux_amd64"
             - "cli_enterprise_linux_arm64"
             - "cli_enterprise_windows_amd64"
-          runner: ["ubuntu-22.04", "ubuntu-20.04"]
+          runner:
+            - "ubuntu-24.04"
+            - "ubuntu-22.04"
+          deps:
+            - conventional
+            - eccentric
     env:
         bazel_target: "//cli:${{ matrix.target }}"
-        binary: "${{ matrix.target }}-${{ matrix.runner }}"
+        binary: "${{ matrix.target }}-${{ matrix.runner }}-${{ matrix.deps }}"
     runs-on: ${{ matrix.runner }}
     steps:
       - name: Checkout
@@ -35,8 +41,22 @@ jobs:
         with:
           ref: ${{ !github.event.pull_request.head.repo.fork && github.head_ref || '' }}
 
-      - name: Setup bazel
+      - name: Setup dependencies
         uses: ./.github/actions/setup_bazel_nix
+        if: matrix.deps == 'conventional'
+
+      - name: Setup dependencies (eccentric)
+        if: matrix.deps == 'eccentric'
+        run: |
+          version=$(cat .bazelversion)
+          mkdir -p "$HOME/.local/bin"
+          curl -fsSL -o "$HOME/.local/bin/bazel" "https://github.com/bazelbuild/bazel/releases/download/$version/bazel-$version-linux-x86_64"
+          chmod a+x "$HOME/.local/bin/bazel"
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+
+          curl -fsSL -o "$HOME/.local/bin/nix-installer" https://github.com/DeterminateSystems/nix-installer/releases/download/v0.26.3/nix-installer-x86_64-linux # renovate:github-release
+          chmod a+x "$HOME/.local/bin/nix-installer"
+          "$HOME/.local/bin/nix-installer" install --no-confirm
 
       - name: Build
         shell: bash
@@ -59,13 +79,13 @@ jobs:
       - name: Upload binary artifact
         uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
-          name: "binaries-${{ matrix.target }}-${{ matrix.runner }}"
+          name: "binaries-${{ matrix.target }}-${{ matrix.runner }}-${{ matrix.deps }}"
           path: "${{ env.binary }}"
 
       - name: Upload hash artifact
         uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4.3.4
         with:
-          name: "sha256sums-${{ matrix.target }}-${{ matrix.runner }}"
+          name: "sha256sums-${{ matrix.target }}-${{ matrix.runner }}-${{ matrix.deps }}"
           path: "${{ env.binary }}.sha256"
 
   build-osimages:
@@ -77,12 +97,24 @@ jobs:
             - "aws_aws-nitro-tpm_console"
             - "qemu_qemu-vtpm_debug"
             - "gcp_gcp-sev-snp_nightly"
-          runner: ["ubuntu-22.04", "ubuntu-20.04"]
+          runner: ["ubuntu-24.04", "ubuntu-22.04"]
     env:
         bazel_target: "//image/system:${{ matrix.target }}"
         binary: "osimage-${{ matrix.target }}-${{ matrix.runner }}"
     runs-on: ${{ matrix.runner }}
     steps:
+      - name: Remove security hardening features
+        if: matrix.runner == 'ubuntu-24.04'
+        shell: bash
+        run: |
+          # Taken from https://github.com/systemd/mkosi/blob/fcacc94b9f72d9b6b1f03779b0c6e07209ceb54b/action.yaml#L42-L57.
+          sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_unconfined=0
+          sudo sysctl --ignore --write kernel.apparmor_restrict_unprivileged_userns=0
+          # This command fails with a non-zero error code even though it unloads the apparmor profiles.
+          # https://gitlab.com/apparmor/apparmor/-/issues/403
+          sudo aa-teardown || true
+          sudo apt-get remove -y apparmor
+
       - name: Checkout
         uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
         with:

docs/docs/workflows/reproducible-builds.md

@@ -0,0 +1,63 @@
+# Reproduce released artifacts
+
+Constellation has first-class support for [reproducible builds](https://reproducible-builds.org).
+Reproducing the released artifacts is an alternative to [signature verification](verify-cli.md) that doesn't require trusting Edgeless Systems' release process.
+The following sections describe how to rebuild an artifact and how Constellation ensures that the rebuild reproduces the artifacts bit-by-bit.
+
+## Build environment prerequisites
+
+The build systems used by Constellation - [Bazel](https://bazel.build/) and [Nix](https://nixos.org) - are designed for deterministic, reproducible builds.
+These two dependencies should be the only prerequisites for a successful build.
+However, it can't be ruled out completely that peculiarities of the host affect the build result.
+Thus, we recommend the following host setup for best results:
+
+1. A Linux operating system not older than v5.4.
+2. The GNU C library not older than v2.31 (avoid `musl`).
+3. GNU `coreutils` not older than v8.30 (avoid `busybox`).
+4. An `ext4` filesystem for building.
+5. AppArmor turned off.
+
+This is given, for example, on an Ubuntu 22.04 system, which is also used for reproducibility tests.
+
+:::note
+
+To avoid any backwards-compatibility issues, the host software versions should also not be much newer than the Constellation release.
+
+:::
+
+## Run the build
+
+The following instructions outline qualitatively how to reproduce a build.
+Constellation implements these instructions in the [Reproducible Builds workflow](https://github.com/edgelesssys/constellation/actions/workflows/reproducible-builds.yml), which continuously tests for reproducibility.
+The workflow is a good place to look up specific version numbers and build steps.
+
+1. Check out the Constellation repository at the tag corresponding to the release.
+
+   ```bash
+   git clone https://github.com/edgelesssys/constellation.git
+   cd constellation
+   git checkout v2.20.0
+   ```
+
+2. [Install the Bazel release](https://bazel.build/install) specified in `.bazelversion`.
+3. [Install Nix](https://nixos.org/download/) (any recent version should do).
+4. Run the build with `bazel build $target` for one of the following targets of interest:
+
+   ```data
+   //cli:cli_enterprise_darwin_amd64
+   //cli:cli_enterprise_darwin_arm64
+   //cli:cli_enterprise_linux_amd64
+   //cli:cli_enterprise_linux_arm64
+   //cli:cli_enterprise_windows_amd64
+   ```
+
+5. Compare the build result with the downloaded release artifact.
+
+<!-- TODO(burgerdev): document reproducing images -->
+
+## Feedback
+
+Reproduction failures often indicate a bug in the build system or in the build definitions.
+Therefore, we're interested in any reproducibility issues you might encounter.
+[Start a bug report](https://github.com/edgelesssys/constellation/issues/new/choose) and describe the details of your build environment.
+Make sure to include your result binary or a [`diffoscope`](https://diffoscope.org/) report, if possible.

docs/sidebars.js

@@ -220,6 +220,11 @@ const sidebars = {
           label: 'Consume SBOMs',
           id: 'workflows/sbom',
         },
+        {
+          type: 'doc',
+          label: 'Reproduce release artifacts',
+          id: 'workflows/reproducible-builds',
+        },
         {
           type: 'doc',
           label: 'Troubleshooting',

nix/cc/cryptsetup.nix

@@ -11,7 +11,7 @@ pkgs.symlinkJoin {
   paths = packages;
   buildInputs = packages;
   postBuild = ''
-    tar -cf $out/closure.tar --mtime="@$SOURCE_DATE_EPOCH" --sort=name ${closure}
+    tar -cf $out/closure.tar --mtime="@$SOURCE_DATE_EPOCH" --sort=name --hard-dereference ${closure}
     echo "${rpath}" > $out/rpath
     cp ${cc}/nix-support/dynamic-linker $out/dynamic-linker
   '';

nix/cc/libvirt.nix

@@ -11,7 +11,7 @@ pkgs.symlinkJoin {
   paths = packages;
   buildInputs = packages;
   postBuild = ''
-    tar -cf $out/closure.tar --mtime="@$SOURCE_DATE_EPOCH" --sort=name ${closure}
+    tar -cf $out/closure.tar --mtime="@$SOURCE_DATE_EPOCH" --sort=name --hard-dereference ${closure}
     tar --transform 's+^./+bin/+' -cf $out/bin-linktree.tar --mtime="@$SOURCE_DATE_EPOCH" --sort=name -C $out/bin .
     echo "${rpath}" > $out/rpath
     cp ${cc}/nix-support/dynamic-linker $out/dynamic-linker

renovate.json5

@@ -246,12 +246,12 @@
       "datasourceTemplate": "github-releases",
     },
     {
-      "fileMatch": ["(^|\\/)versions.go$"],
+      "fileMatch": ["(^|\\/)versions.go$", "[.]github\\/(actions|workflows)\\/.*[.]ya?ml"],
       "matchStrings": [
         // Match github releases.
         // example match:' "https://github.com/foo/bar/releases/download/v1.2.3/foo.bin" // renovate:github-release'
         // (foo/bar -> depName, v1.2.3 -> currentValue)
-        " \"https:\\/\\/github\\.com\\/(?<depName>[^\\/\\s\"]+\\/[^\\/\\s\"]+)\\/releases\\/download\\/(?<currentValue>[^\\/\\s\"]+)\\/[^\"]+\"[^\\n]+\\/\\/ renovate:github-release",
+        "https:\\/\\/github\\.com\\/(?<depName>[^\\/\\s\"]+\\/[^\\/\\s\"]+)\\/releases\\/download\\/(?<currentValue>[^\\/\\s\"]+).*renovate:github-release",
       ],
       "datasourceTemplate": "github-releases",
     },