Merge pull request #9777

87a8e0b2c ci: development build backports [0.18] (tobtoht)
84e44dd01 tests: Fix tools::is_hdd unit tests (iamamyth)

.github/actions/set-make-job-count/action.yml

@@ -0,0 +1,22 @@
+name: 'set-make-job-count'
+description: 'Set the MAKE_JOB_COUNT environment variable to a value suitable for the host runner'
+runs:
+  using: "composite"
+  steps:
+    # Each job runner requires 2.25 GiB (i.e. 1024 * 9/4 MiB) memory and
+    # a dedicated logical CPU core
+    - name: set-jobs-macOS
+      if: runner.os == 'macOS'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(sysctl -n hw.memsize) * 4 / (1073741824 * 9) )) $(sysctl -n hw.logicalcpu) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: bash
+    - name: set-jobs-windows
+      if: runner.os == 'Windows'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: msys2 {0}
+    - name: set-jobs-linux
+      if: runner.os == 'Linux'
+      run: |
+        echo MAKE_JOB_COUNT=$(expr $(printf '%s\n%s' $(( $(grep MemTotal: /proc/meminfo | cut -d: -f2 | cut -dk -f1) * 4 / (1048576 * 9) )) $(nproc) | sort -n | head -n1) '|' 1) >> $GITHUB_ENV
+      shell: bash

.github/workflows/build.yml

@@ -2,6 +2,9 @@ name: ci/gh-actions/cli
 
 on:
   push:
+    paths-ignore:
+      - 'docs/**'
+      - '**/README.md'
   pull_request:
     paths-ignore:
       - 'docs/**'
@@ -9,20 +12,22 @@ on:
 
 # The below variables reduce repetitions across similar targets
 env:
-  REMOVE_BUNDLED_BOOST : rm -rf /usr/local/share/boost
+  # ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere.
-  BUILD_DEFAULT_LINUX: |
+  BUILD_DEFAULT_LINUX: 'cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build --target all && cmake --build build --target wallet_api'
-        cmake -S . -B build -D ARCH="default" -D BUILD_TESTS=ON -D CMAKE_BUILD_TYPE=Release && cmake --build build --parallel 4
+  APT_INSTALL_LINUX: 'apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libusb-1.0-0-dev libprotobuf-dev protobuf-compiler ccache git'
-  APT_INSTALL_LINUX: 'sudo apt -y install build-essential cmake libboost-all-dev miniupnpc libunbound-dev graphviz doxygen libunwind8-dev pkg-config libssl-dev libzmq3-dev libsodium-dev libhidapi-dev libnorm-dev libusb-1.0-0-dev libpgm-dev libprotobuf-dev protobuf-compiler ccache'
   APT_SET_CONF: |
-        echo "Acquire::Retries \"3\";"         | sudo tee -a /etc/apt/apt.conf.d/80-custom
+    tee -a /etc/apt/apt.conf.d/80-custom << EOF
-        echo "Acquire::http::Timeout \"120\";" | sudo tee -a /etc/apt/apt.conf.d/80-custom
+    Acquire::Retries "3";
-        echo "Acquire::ftp::Timeout \"120\";"  | sudo tee -a /etc/apt/apt.conf.d/80-custom
+    Acquire::http::Timeout "120";
+    Acquire::ftp::Timeout "120";
+    EOF
   CCACHE_SETTINGS: |
     ccache --max-size=150M
     ccache --set-config=compression=true
 
 jobs:
   build-macos:
+    name: 'macOS (brew)'
     runs-on: macOS-latest
     env:
       CCACHE_TEMPDIR: /tmp/.ccache-temp
@@ -35,6 +40,7 @@ jobs:
           path: /Users/runner/Library/Caches/ccache
           key: ccache-${{ runner.os }}-build-${{ github.sha }}
           restore-keys: ccache-${{ runner.os }}-build-
+      - uses: ./.github/actions/set-make-job-count
       - name: install dependencies
         run: |
           HOMEBREW_NO_AUTO_UPDATE=1 brew install boost@1.85 hidapi openssl zmq libpgm miniupnpc expat libunwind-headers protobuf ccache
@@ -42,9 +48,10 @@ jobs:
       - name: build
         run: |
           ${{env.CCACHE_SETTINGS}}
-        make -j3
+          make -j${{env.MAKE_JOB_COUNT}}
 
   build-windows:
+    name: 'Windows (MSYS2)'
     runs-on: windows-latest
     env:
       CCACHE_TEMPDIR: C:\Users\runneradmin\.ccache-temp
@@ -74,110 +81,151 @@ jobs:
           curl -O https://repo.msys2.org/mingw/mingw64/mingw-w64-x86_64-icu-75.1-2-any.pkg.tar.zst
           echo "bf57882d43efcdfd746463613ea982c69b64aa4ba9bed4cb24c02a81ad06c3a9 mingw-w64-x86_64-icu-75.1-2-any.pkg.tar.zst" | sha256sum -c
           pacman --noconfirm -U mingw-w64-x86_64-boost-1.86.0-7-any.pkg.tar.zst mingw-w64-x86_64-boost-libs-1.86.0-7-any.pkg.tar.zst mingw-w64-x86_64-icu-75.1-2-any.pkg.tar.zst
+      - uses: ./.github/actions/set-make-job-count
       - name: build
         run: |
           ${{env.CCACHE_SETTINGS}}
-        make release-static-win64 -j4
+          make release-static-win64 -j${{env.MAKE_JOB_COUNT}}
 
-# See the OS labels and monitor deprecations here:
+  build-debian:
-# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
+    # Oldest supported Debian version
+    name: 'Debian 10'
+    runs-on: ubuntu-latest
+    container:
+      image: debian:10
+      env:
+        DEBIAN_FRONTEND: noninteractive
+    steps:
+      - name: set apt conf
+        run: ${{env.APT_SET_CONF}}
+      - name: update apt
+        run: apt update
+      - name: install monero dependencies
+        run: ${{env.APT_INSTALL_LINUX}}
+      - name: configure git
+        run: git config --global --add safe.directory '*'
+      - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - uses: ./.github/actions/set-make-job-count
+      - name: build
+        env:
+          CMAKE_BUILD_PARALLEL_LEVEL: ${{env.MAKE_JOB_COUNT}}
+        run: ${{env.BUILD_DEFAULT_LINUX}}
 
   build-ubuntu:
-    runs-on: ${{ matrix.os }}
+    name: ${{ matrix.name }}
-    env:
+    runs-on: ubuntu-latest
-      CCACHE_TEMPDIR: /tmp/.ccache-temp
     strategy:
+      fail-fast: false
       matrix:
-        os: [ubuntu-22.04, ubuntu-20.04]
+        include:
+          # Oldest supported Ubuntu LTS version
+          - name: Ubuntu 20.04
+            container: ubuntu:20.04
+
+          # Most popular Ubuntu LTS version
+          - name: Ubuntu 22.04
+            container: ubuntu:22.04
+    container:
+      image: ${{ matrix.container }}
+      env:
+        DEBIAN_FRONTEND: noninteractive
+        CCACHE_TEMPDIR: /tmp/.ccache-temp
+        CCACHE_DIR: ~/.ccache
     steps:
+      - name: set apt conf
+        run: ${{env.APT_SET_CONF}}
+      - name: update apt
+        run: apt update
+      - name: install monero dependencies
+        run: ${{env.APT_INSTALL_LINUX}}
+      - name: configure git
+        run: git config --global --add safe.directory '*'
       - uses: actions/checkout@v4
         with:
           submodules: recursive
       - uses: actions/cache@v4
         with:
           path: ~/.ccache
-        key: ccache-${{ runner.os }}-build-${{ matrix.os }}-${{ github.sha }}
+          key: ccache-${{ matrix.container }}-build-${{ github.sha }}
-        restore-keys: ccache-${{ runner.os }}-build-${{ matrix.os }}
+          restore-keys: ccache-${{ matrix.container }}-build-
-    - name: remove bundled boost
+      - uses: ./.github/actions/set-make-job-count
-      run: ${{env.REMOVE_BUNDLED_BOOST}}
-    - name: set apt conf
-      run: ${{env.APT_SET_CONF}}
-    - name: update apt
-      run: sudo apt update
-    - name: install monero dependencies
-      run: ${{env.APT_INSTALL_LINUX}}
       - name: build
+        env:
+          CMAKE_BUILD_PARALLEL_LEVEL: ${{env.MAKE_JOB_COUNT}}
         run: |
           ${{env.CCACHE_SETTINGS}}
           ${{env.BUILD_DEFAULT_LINUX}}
 
-  libwallet-ubuntu:
+  test-ubuntu:
-    runs-on: ubuntu-20.04
+    name: "${{ matrix.name }} (tests)"
+    needs: build-ubuntu
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - name: Ubuntu 20.04
+            container: ubuntu:20.04
+    container:
+      image: ${{ matrix.container }}
       env:
+        DEBIAN_FRONTEND: noninteractive
         CCACHE_TEMPDIR: /tmp/.ccache-temp
+        CCACHE_DIR: ~/.ccache
+      # Setting up a loop device (losetup) requires additional capabilities.
+      # tests/create_test_disks.sh
+      options: --privileged
     steps:
+      - name: set apt conf
+        run: ${{env.APT_SET_CONF}}
+      - name: update apt
+        run: apt update
+      - name: install monero dependencies
+        run: ${{env.APT_INSTALL_LINUX}}
+      - name: install pip
+        run: apt install -y python3-pip
+      - name: install Python dependencies
+        run: pip install requests psutil monotonic zmq deepdiff
+      - name: configure git
+        run: git config --global --add safe.directory '*'
       - uses: actions/checkout@v4
         with:
           submodules: recursive
       - uses: actions/cache@v4
         with:
           path: ~/.ccache
-        key: ccache-${{ runner.os }}-libwallet-${{ github.sha }}
+          key: ccache-${{ matrix.container }}-build-${{ github.sha }}
-        restore-keys: ccache-${{ runner.os }}-libwallet-
+          restore-keys: ccache-${{ matrix.container }}-build-
-    - name: remove bundled boost
+      - name: create dummy disk drives for testing
-      run: ${{env.REMOVE_BUNDLED_BOOST}}
+        run: tests/create_test_disks.sh >> $GITHUB_ENV
-    - name: set apt conf
+      - uses: ./.github/actions/set-make-job-count
-      run: ${{env.APT_SET_CONF}}
-    - name: update apt
-      run: sudo apt update
-    - name: install monero dependencies
-      run: ${{env.APT_INSTALL_LINUX}}
-    - name: build
-      run: |
-        ${{env.CCACHE_SETTINGS}}
-        cmake .
-        make wallet_api -j4
-
-  test-ubuntu:
-    needs: build-ubuntu
-    runs-on: ubuntu-20.04
-    env:
-      CCACHE_TEMPDIR: /tmp/.ccache-temp
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        submodules: recursive
-    - name: ccache
-      uses: actions/cache@v4
-      with:
-        path: ~/.ccache
-        key: ccache-${{ runner.os }}-build-ubuntu-latest-${{ github.sha }}
-        restore-keys: ccache-${{ runner.os }}-build-ubuntu-latest
-    - name: remove bundled boost
-      run: ${{env.REMOVE_BUNDLED_BOOST}}
-    - name: set apt conf
-      run: ${{env.APT_SET_CONF}}
-    - name: update apt
-      run: sudo apt update
-    - name: install monero dependencies
-      run: ${{env.APT_INSTALL_LINUX}}
-    - name: install Python dependencies
-      run: pip install requests psutil monotonic zmq deepdiff
       - name: tests
         env:
           CTEST_OUTPUT_ON_FAILURE: ON
           DNS_PUBLIC: tcp://9.9.9.9
+          CMAKE_BUILD_PARALLEL_LEVEL: ${{env.MAKE_JOB_COUNT}}
         run: |
           ${{env.CCACHE_SETTINGS}}
           ${{env.BUILD_DEFAULT_LINUX}}
           cmake --build build --target test
 
-# ARCH="default" (not "native") ensures, that a different execution host can execute binaries compiled elsewhere.
-# BUILD_SHARED_LIBS=ON speeds up the linkage part a bit, reduces size, and is the only place where the dynamic linkage is tested.
-
   source-archive:
-    runs-on: ubuntu-20.04
+    name: "source archive"
+    runs-on: ubuntu-latest
+    container:
+      image: ubuntu:20.04
+      env:
+        DEBIAN_FRONTEND: noninteractive
     steps:
+      - name: set apt conf
+        run: ${{env.APT_SET_CONF}}
+      - name: update apt
+        run: apt update
+      - name: install dependencies
+        run: apt install -y git python3-pip
+      - name: configure git
+        run: git config --global --add safe.directory '*'
       - uses: actions/checkout@v4
         with:
           fetch-depth: 0
@@ -188,8 +236,8 @@ jobs:
           export VERSION="monero-$(git describe)"
           export OUTPUT="$VERSION.tar"
           echo "OUTPUT=$OUTPUT" >> $GITHUB_ENV
-        /home/runner/.local/bin/git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT"
+          git-archive-all --prefix "$VERSION/" --force-submodules "$OUTPUT"
       - uses: actions/upload-artifact@v4
         with:
           name: ${{ env.OUTPUT }}
-        path: /home/runner/work/monero/monero/${{ env.OUTPUT }}
+          path: ${{ env.OUTPUT }}

tests/create_test_disks.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+set -e
+
+LOOP_DEVICE_MAJOR=7
+LOOP_DEVICE_MIN_ID=100
+
+echo_err() {
+    echo "$@" >&2
+}
+
+root_exec() {
+    if [[ $EUID -ne 0 ]]; then
+        ${ROOT_EXEC_CMD:-sudo} "$@"
+    else
+        "$@"
+    fi
+}
+
+create_device_node() {
+    local -r last_id=$(find /dev/ -name 'loop[0-9]*' -printf '%f\n' | sed 's/^loop//' | sort -r -n | head -1)
+    local id=$((last_id + 1))
+    if [[ "$id" -lt "$LOOP_DEVICE_MIN_ID" ]]; then
+        id="$LOOP_DEVICE_MIN_ID"
+    fi
+    local path
+    for (( i=0; i<10; i++ )); do
+        path="/dev/loop$id"
+        if [[ ! -e "$path" ]] && root_exec mknod "$path" b "$LOOP_DEVICE_MAJOR" "$id"; then
+            echo "$path"
+            return 0
+        fi
+        $((id++))
+    done
+    return 1
+}
+
+device_mountpoint() {
+    local -r datadir="$1"
+    local -r dev="$2"
+    if [[ -z "$datadir" || -z "$dev" ]]; then
+        echo_err "Usage: device_mountpoint <data dir> <device>"
+        return 1
+    fi
+    echo "$datadir/mnt-$(basename "$dev")"
+}
+
+create_device() {
+    local -r datadir="$1"
+    if [[ -z "$datadir" ]]; then
+        echo_err "Usage: create_device <data dir>"
+        return 1
+    fi
+    local -r dev=$(create_device_node)
+    local -r fs="$datadir/$(basename "$dev").vhd"
+    local -r mountpoint=$(device_mountpoint "$datadir" "$dev")
+    echo_err
+    echo_err "# Device $dev"
+    dd if=/dev/zero of="$fs" bs=64K count=128 >/dev/null 2>&1
+    root_exec losetup "$dev" "$fs"
+    root_exec mkfs.ext4 "$dev" >/dev/null 2>&1
+    mkdir "$mountpoint"
+    root_exec mount "$dev" "$mountpoint"
+    echo "$dev"
+}
+
+# Unused by default, but helpful for local development
+destroy_device() {
+    local -r datadir="$1"
+    local -r dev="$2"
+    if [[ -z "$datadir" || -z "$dev" ]]; then
+        echo_err "Usage: destroy_device <data dir> <device>"
+        return 1
+    fi
+    echo_err "Destroying device $dev"
+    root_exec umount $(device_mountpoint "$datadir" "$dev")
+    root_exec losetup -d "$dev"
+    root_exec rm "$dev"
+}
+
+block_device_path() {
+    device_name=$(basename "$1")
+    device_minor=${device_name/#loop}
+    echo "/sys/dev/block/$LOOP_DEVICE_MAJOR:$device_minor"
+}
+
+tmpdir=$(mktemp --tmpdir -d monerotest.XXXXXXXX)
+echo_err "Creating devices using temporary directory: $tmpdir"
+
+dev_rot=$(create_device "$tmpdir")
+bdev_rot=$(block_device_path "$dev_rot")
+echo 1 | root_exec tee "$bdev_rot/queue/rotational" >/dev/null
+echo MONERO_TEST_DEVICE_HDD=$(device_mountpoint "$tmpdir" "$dev_rot")
+
+dev_ssd=$(create_device "$tmpdir")
+bdev_ssd=$(block_device_path "$dev_ssd")
+echo 0 | root_exec tee "$bdev_ssd/queue/rotational" >/dev/null
+echo MONERO_TEST_DEVICE_SSD=$(device_mountpoint "$tmpdir" "$dev_ssd")

tests/unit_tests/is_hdd.cpp

@@ -1,17 +1,36 @@
 #include "common/util.h"
+#include <cstdlib>
 #include <string>
 #include <gtest/gtest.h>
+#include <boost/optional/optional_io.hpp> /* required to output boost::optional in assertions */
+
+#ifndef GTEST_SKIP
+#include <iostream>
+#define SKIP_TEST(reason) do {std::cerr << "Skipping test: " << reason << std::endl; return;} while(0)
+#else
+#define SKIP_TEST(reason) GTEST_SKIP() << reason
+#endif
 
 #if defined(__GLIBC__)
-TEST(is_hdd, linux_os_root)
+TEST(is_hdd, rotational_drive) {
-{
+  const char *hdd = std::getenv("MONERO_TEST_DEVICE_HDD");
-  std::string path = "/";
+  if (hdd == nullptr)
-  EXPECT_TRUE(tools::is_hdd(path.c_str()) != boost::none);
+    SKIP_TEST("No rotational disk device configured");
+  EXPECT_EQ(tools::is_hdd(hdd), boost::optional<bool>(true));
 }
-#else
+
-TEST(is_hdd, unknown_os)
+TEST(is_hdd, ssd) {
-{
+  const char *ssd = std::getenv("MONERO_TEST_DEVICE_SSD");
-  std::string path = "";
+  if (ssd == nullptr)
-  EXPECT_FALSE(tools::is_hdd(path.c_str()) != boost::none);
+    SKIP_TEST("No SSD device configured");
+  EXPECT_EQ(tools::is_hdd(ssd), boost::optional<bool>(false));
+}
+
+TEST(is_hdd, unknown_attrs) {
+  EXPECT_EQ(tools::is_hdd("/dev/null"), boost::none);
 }
 #endif
+TEST(is_hdd, stability)
+{
+  EXPECT_NO_THROW(tools::is_hdd(""));
+}