Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2024-10-01 01:26:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'capnp-speculative' into 'main'

Browse Source
This commit is contained in:
Christien Rioux 2023-11-08 13:22:09 +00:00
commit 7d1b789d38
12 changed files with 236 additions and 253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
Cargo.lock generated
View File

@ -1830,6 +1830,18 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a481586acf778f1b1455424c343f71124b048ffa5f4fc3f8f6ae9dc432dcb3c7"
[[package]]
name = "filetime"
version = "0.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4029edd3e734da6fe05b6cd7bd2960760a616bd2ddd0d59a0124746d6272af0"
dependencies = [
"cfg-if 1.0.0",
"libc",
"redox_syscall 0.3.5",
"windows-sys 0.48.0",
]
[[package]]
name = "flate2"
version = "1.0.28"
@ -3930,6 +3942,15 @@ dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29"
dependencies = [
"bitflags 1.3.2",
]
[[package]]
name = "redox_syscall"
version = "0.4.1"
@ -5521,6 +5542,7 @@ dependencies = [
"enum-as-inner",
"enumset",
"eyre",
"filetime",
"flume",
"futures-util",
"getrandom",

16
DEVELOPMENT.md
View File

@ -22,9 +22,12 @@ welcome!
Running the setup script requires:
* Android SDK and NDK
* Rust
#### Optionally, to build for Android:
* Android SDK and NDK
You may decide to use Android Studio [here](https://developer.android.com/studio)
to maintain your Android dependencies. If so, use the dependency manager
within your IDE. If you plan on using Flutter for Veilid development, the Android Studio
@ -95,12 +98,15 @@ Development of Veilid on MacOS is possible on both Intel and ARM hardware.
Development requires:
* Android Studio
* Xcode, preferably latest version
* Homebrew [here](https://brew.sh)
* Android SDK and NDK
* Rust
#### Optionally, to build for Android:
* Android Studio
* Android SDK and NDK
You will need to use Android Studio [here](https://developer.android.com/studio)
to maintain your Android dependencies. Use the SDK Manager in the IDE to install the following packages (use package details view to select version):
@ -150,9 +156,7 @@ Install Git from <https://git-scm.com/download/win>
Install Rust from <https://static.rust-lang.org/rustup/dist/x86_64-pc-windows-msvc/rustup-init.exe> (this may prompt you to run the Visual Studio Installer, and reboot, before proceeding).
Ensure that protoc.exe is in a directory in your path. For example, it can be obtained from <https://github.com/protocolbuffers/protobuf/releases/download/v24.3/protoc-24.3-win64.zip>
Ensure that capnp.exe (for Cap’n Proto 1.0.1) is in a directory in your path. For example, it can be obtained from the <https://capnproto.org/capnproto-c++-win32-1.0.1.zip> distribution. Please note that the Windows Package Manager Community Repository (i.e., winget) as of 2023-09-15 has version 0.10.4, which is not sufficient.
Optional (only if modifying capnp schema): Ensure that capnp.exe (for Cap’n Proto 1.0.1) is in a directory in your path. For example, it can be obtained from the <https://capnproto.org/capnproto-c++-win32-1.0.1.zip> distribution. Please note that the Windows Package Manager Community Repository (i.e., winget) as of 2023-09-15 has version 0.10.4, which is not sufficient.
Start a Command Prompt window.

26
Earthfile
View File

@ -26,22 +26,6 @@ deps-base:
RUN apt-get -y update
RUN apt-get install -y iproute2 curl build-essential cmake libssl-dev openssl file git pkg-config libdbus-1-dev libdbus-glib-1-dev libgirepository1.0-dev libcairo2-dev checkinstall unzip libncursesw5-dev libncurses5-dev
# Install Cap'n Proto
deps-capnp:
FROM +deps-base
COPY scripts/earthly/install_capnproto.sh /
RUN /bin/bash /install_capnproto.sh 1; rm /install_capnproto.sh
SAVE ARTIFACT /usr/local/bin/capnp* bin/
SAVE ARTIFACT /usr/local/lib/lib* lib/
SAVE ARTIFACT /usr/local/lib/pkgconfig lib/
# Install protoc
deps-protoc:
FROM +deps-base
COPY scripts/earthly/install_protoc.sh /
RUN /bin/bash /install_protoc.sh; rm /install_protoc.sh
SAVE ARTIFACT /usr/local/bin/protoc bin/protoc
# Install Rust
deps-rust:
FROM +deps-base
@ -75,11 +59,6 @@ deps-rust:
# Install android tooling
deps-android:
FROM +deps-base
BUILD +deps-protoc
COPY +deps-protoc/bin/* /usr/local/bin/
BUILD +deps-capnp
COPY +deps-capnp/bin/* /usr/local/bin/
COPY +deps-capnp/lib/* /usr/local/lib/
BUILD +deps-rust
COPY +deps-rust/cargo /usr/local/cargo
COPY +deps-rust/rustup /usr/local/rustup
@ -96,11 +75,6 @@ deps-android:
# Just linux build not android
deps-linux:
FROM +deps-base
BUILD +deps-protoc
COPY +deps-protoc/bin/* /usr/local/bin/
BUILD +deps-capnp
COPY +deps-capnp/bin/* /usr/local/bin/
COPY +deps-capnp/lib/* /usr/local/lib/
BUILD +deps-rust
COPY +deps-rust/cargo /usr/local/cargo
COPY +deps-rust/rustup /usr/local/rustup

140
dev-setup/setup_linux.sh
View File

@ -17,75 +17,75 @@ if ! lsb_release -d | grep -qEi 'debian|buntu|mint' && [ -z "$(command -v dnf)"
echo Not a supported Linux
exit 1
fi
while true; do
read -p "Did you install Android SDK? Y/N " response
read -p "Did you install Android SDK? Y/N " response
case $response in
[yY] ) echo Checking android setup...;
case $response in
[yY] ) echo Checking android setup...;
# ensure ANDROID_SDK_ROOT is defined and exists
if [ -d "$ANDROID_SDK_ROOT" ]; then
echo '[X] $ANDROID_SDK_ROOT is defined and exists'
else
echo '$ANDROID_SDK_ROOT is not defined or does not exist'
exit 1
fi
# ensure ANDROID_SDK_ROOT is defined and exists
if [ -d "$ANDROID_SDK_ROOT" ]; then
echo '[X] $ANDROID_SDK_ROOT is defined and exists'
else
echo '$ANDROID_SDK_ROOT is not defined or does not exist'
exit 1
fi
# ensure Android Command Line Tools exist
if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then
echo '[X] Android command line tools are installed'
else
echo 'Android command line tools are not installed'
exit 1
fi
# ensure Android Command Line Tools exist
if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then
echo '[X] Android command line tools are installed'
else
echo 'Android command line tools are not installed'
exit 1
fi
# ensure ANDROID_NDK_HOME is defined and exists
if [ -d "$ANDROID_NDK_HOME" ]; then
echo '[X] $ANDROID_NDK_HOME is defined and exists'
else
echo '$ANDROID_NDK_HOME is not defined or does not exist'
exit 1
fi
# ensure ANDROID_NDK_HOME is defined and exists
if [ -d "$ANDROID_NDK_HOME" ]; then
echo '[X] $ANDROID_NDK_HOME is defined and exists'
else
echo '$ANDROID_NDK_HOME is not defined or does not exist'
exit 1
fi
# ensure ndk is installed
if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then
echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME'
else
echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME'
exit 1
fi
# ensure ndk is installed
if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then
echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME'
else
echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME'
exit 1
fi
# ensure cmake is installed
if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then
echo '[X] Android SDK CMake is installed'
else
echo 'Android SDK CMake is not installed'
exit 1
fi
# ensure cmake is installed
if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then
echo '[X] Android SDK CMake is installed'
else
echo 'Android SDK CMake is not installed'
exit 1
fi
# ensure emulator is installed
if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then
echo '[X] Android SDK emulator is installed'
else
echo 'Android SDK emulator is not installed'
exit 1
fi
# ensure emulator is installed
if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then
echo '[X] Android SDK emulator is installed'
else
echo 'Android SDK emulator is not installed'
exit 1
fi
# ensure adb is installed
if command -v adb &> /dev/null; then
echo '[X] adb is available in the path'
else
echo 'adb is not available in the path'
exit 1
fi
break;;
[nN] ) echo Skipping android SDK config check...;
break;;
* ) echo invalid response;;
esac
# ensure adb is installed
if command -v adb &> /dev/null; then
echo '[X] adb is available in the path'
else
echo 'adb is not available in the path'
exit 1
fi
break;;
[nN] ) echo Skipping Android SDK config check...;
break;;
* ) echo invalid response;;
esac
done
# ensure rustup is installed
if command -v rustup &> /dev/null; then
echo '[X] rustup is available in the path'
@ -119,7 +119,21 @@ cargo install wasm-bindgen-cli wasm-pack cargo-edit
# install pip packages
pip3 install --upgrade bumpversion
# Install capnproto using the same mechanism as our earthly build
$SCRIPTDIR/../scripts/earthly/install_capnproto.sh
# Install protoc using the same mechanism as our earthly build
$SCRIPTDIR/../scripts/earthly/install_protoc.sh
# install capnp
while true; do
read -p "Will you be modifying the capnproto schema? Y/N (say N if unsure)" response
case $response in
[yY] ) echo Installing capnproto...;
# Install capnproto using the same mechanism as our earthly build
$SCRIPTDIR/../scripts/earthly/install_capnproto.sh
break;;
[nN] ) echo Skipping capnproto installation...;
break;;
* ) echo invalid response;;
esac
done

120
dev-setup/setup_macos.sh
View File

@ -7,69 +7,77 @@ if [ ! "$(uname)" == "Darwin" ]; then
echo Not running on MacOS
exit 1
fi
read -p "Did you install Android SDK? Y/N " response
case $response in
[yY] ) echo Checking android setup...;
# ensure ANDROID_SDK_ROOT is defined and exists
if [ -d "$ANDROID_SDK_ROOT" ]; then
echo '[X] $ANDROID_SDK_ROOT is defined and exists'
else
echo '$ANDROID_SDK_ROOT is not defined or does not exist'
exit 1
fi
while true; do
# ensure Android Command Line Tools exist
if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then
echo '[X] Android command line tools are installed'
else
echo 'Android command line tools are not installed'
exit 1
fi
read -p "Did you install Android SDK? Y/N " response
case $response in
[yY] ) echo Checking android setup...;
# ensure ANDROID_SDK_ROOT is defined and exists
if [ -d "$ANDROID_SDK_ROOT" ]; then
echo '[X] $ANDROID_SDK_ROOT is defined and exists'
else
echo '$ANDROID_SDK_ROOT is not defined or does not exist'
exit 1
fi
# ensure Android SDK packages are installed
$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33
# ensure Android Command Line Tools exist
if [ -d "$ANDROID_SDK_ROOT/cmdline-tools/latest/bin" ]; then
echo '[X] Android command line tools are installed'
else
echo 'Android command line tools are not installed'
exit 1
fi
# ensure ANDROID_NDK_HOME is defined and exists
if [ -d "$ANDROID_NDK_HOME" ]; then
echo '[X] $ANDROID_NDK_HOME is defined and exists'
else
echo '$ANDROID_NDK_HOME is not defined or does not exist'
exit 1
fi
# ensure Android SDK packages are installed
$ANDROID_SDK_ROOT/cmdline-tools/latest/bin/sdkmanager build-tools\;33.0.1 ndk\;25.1.8937393 cmake\;3.22.1 platform-tools platforms\;android-33
# ensure ndk is installed
if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then
echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME'
else
echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME'
exit 1
fi
# ensure ANDROID_NDK_HOME is defined and exists
if [ -d "$ANDROID_NDK_HOME" ]; then
echo '[X] $ANDROID_NDK_HOME is defined and exists'
else
echo '$ANDROID_NDK_HOME is not defined or does not exist'
exit 1
fi
# ensure cmake is installed
if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then
echo '[X] Android SDK CMake is installed'
else
echo 'Android SDK CMake is not installed'
exit 1
fi
# ensure ndk is installed
if [ -f "$ANDROID_NDK_HOME/ndk-build" ]; then
echo '[X] Android NDK is installed at the location $ANDROID_NDK_HOME'
else
echo 'Android NDK is not installed at the location $ANDROID_NDK_HOME'
exit 1
fi
# ensure emulator is installed
if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then
echo '[X] Android SDK emulator is installed'
else
echo 'Android SDK emulator is not installed'
exit 1
fi
# ensure cmake is installed
if [ -d "$ANDROID_SDK_ROOT/cmake" ]; then
echo '[X] Android SDK CMake is installed'
else
echo 'Android SDK CMake is not installed'
exit 1
fi
# ensure adb is installed
if command -v adb &> /dev/null; then
echo '[X] adb is available in the path'
else
echo 'adb is not available in the path'
exit 1
fi
esac
# ensure emulator is installed
if [ -d "$ANDROID_SDK_ROOT/emulator" ]; then
echo '[X] Android SDK emulator is installed'
else
echo 'Android SDK emulator is not installed'
exit 1
fi
# ensure adb is installed
if command -v adb &> /dev/null; then
echo '[X] adb is available in the path'
else
echo 'adb is not available in the path'
exit 1
fi
break;;
[nN] ) echo Skipping Android SDK config check...;
break;;
* ) echo invalid response;;
esac
done
# ensure brew is installed
if command -v brew &> /dev/null; then
@ -130,7 +138,7 @@ if [ "$BREW_USER" == "" ]; then
BREW_USER=`whoami`
fi
fi
sudo -H -u $BREW_USER brew install capnp cmake wabt llvm protobuf openjdk@17 jq
sudo -H -u $BREW_USER brew install capnp cmake wabt llvm openjdk@17 jq
# install targets
rustup target add aarch64-apple-darwin aarch64-apple-ios aarch64-apple-ios-sim x86_64-apple-darwin x86_64-apple-ios wasm32-unknown-unknown aarch64-linux-android armv7-linux-androideabi i686-linux-android x86_64-linux-android

37
scripts/earthly/install_protoc.sh
View File

@ -1,37 +0,0 @@
#!/bin/bash
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
PROTOC_VERSION="24.3" # Keep in sync with veilid-core/build.rs
UNAME_M=$(uname -m)
if [[ "$UNAME_M" == "x86_64" ]]; then
PROTOC_ARCH=x86_64
elif [[ "$UNAME_M" == "aarch64" ]]; then
PROTOC_ARCH=aarch_64
else
echo Unsupported build architecture
exit 1
fi
mkdir /tmp/protoc-install
pushd /tmp/protoc-install
curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-linux-$PROTOC_ARCH.zip
unzip protoc-$PROTOC_VERSION-linux-$PROTOC_ARCH.zip
if [ "$EUID" -ne 0 ]; then
if command -v checkinstall &> /dev/null; then
sudo checkinstall --pkgversion=$PROTOC_VERSION -y cp -r bin include /usr/local/
cp *.deb ~
else
sudo cp -r bin include /usr/local/
fi
popd
sudo rm -rf /tmp/protoc-install
else
if command -v checkinstall &> /dev/null; then
checkinstall --pkgversion=$PROTOC_VERSION -y cp -r bin include /usr/local/
cp *.deb ~
else
cp -r bin include /usr/local/
fi
popd
rm -rf /tmp/protoc-install
fi

1
veilid-core/Cargo.toml
View File

@ -274,6 +274,7 @@ wasm-logger = "0.2.0"
[build-dependencies]
capnpc = "0.18.0"
glob = "0.3.1"
filetime = "0.2.22"
[package.metadata.wasm-pack.profile.release]
wasm-opt = ["-O", "--enable-mutable-globals"]

107
veilid-core/build.rs
View File

@ -1,26 +1,23 @@
use filetime::{set_file_mtime, FileTime};
use glob::glob;
use std::{
env,
env, fs, io,
path::Path,
process::{Command, Stdio},
};
const CAPNP_VERSION: &str = "1.0.1"; // Keep in sync with scripts/install_capnp.sh
const PROTOC_VERSION: &str = "24.3"; // Keep in sync with scripts/install_protoc.sh
const CAPNP_VERSION: &str = "1.0.1";
fn get_desired_capnp_version_string() -> String {
CAPNP_VERSION.to_string()
}
fn get_desired_protoc_version_string() -> String {
PROTOC_VERSION.to_string()
}
fn get_capnp_version_string() -> String {
let output = Command::new("capnp")
.arg("--version")
.stdout(Stdio::piped())
.output()
.expect("capnp was not in the PATH");
.expect("capnp was not in the PATH, and is required for the build when you have changed any .capnp files");
let s = String::from_utf8(output.stdout)
.expect("'capnp --version' output was not a valid string")
.trim()
@ -32,35 +29,29 @@ fn get_capnp_version_string() -> String {
s[20..].to_owned()
}
fn get_protoc_version_string() -> String {
let output = Command::new("protoc")
.arg("--version")
.stdout(Stdio::piped())
.output()
.expect("protoc was not in the PATH");
let s = String::from_utf8(output.stdout)
.expect("'protoc --version' output was not a valid string")
.trim()
.to_owned();
fn is_input_file_outdated<P1, P2>(input: P1, output: P2) -> io::Result<bool>
where
P1: AsRef<Path>,
P2: AsRef<Path>,
{
let out_meta = fs::metadata(output);
if let Ok(meta) = out_meta {
let output_mtime = meta.modified()?;
if !s.starts_with("libprotoc ") {
panic!("invalid protoc version string: {}", s);
// if input file is more recent than our output, we are outdated
let input_meta = fs::metadata(input)?;
let input_mtime = input_meta.modified()?;
Ok(input_mtime > output_mtime)
} else {
// output file not found, we are outdated
Ok(true)
}
s[10..].to_owned()
}
fn main() {
if std::env::var("DOCS_RS").is_ok()
|| std::env::var("CARGO_CFG_DOC").is_ok()
|| std::env::var("BUILD_DOCS").is_ok()
{
return;
}
fn do_capnp_build() {
let desired_capnp_version_string = get_desired_capnp_version_string();
let capnp_version_string = get_capnp_version_string();
let desired_protoc_version_string = get_desired_protoc_version_string();
let protoc_version_string = get_protoc_version_string();
// Check capnp version
let desired_capnp_major_version = desired_capnp_version_string
@ -89,45 +80,19 @@ fn main() {
);
}
// Check protoc version
let desired_protoc_major_version = desired_protoc_version_string
.split_once('.')
.unwrap()
.0
.parse::<usize>()
.expect("should be valid int");
if protoc_version_string
.split_once('.')
.unwrap()
.0
.parse::<usize>()
.expect("should be valid int")
< desired_protoc_major_version
{
panic!(
"protoc version should be at least major version {} but is {}",
desired_protoc_major_version, protoc_version_string
);
} else if protoc_version_string != desired_protoc_version_string {
println!(
"cargo:warning=protoc version may be untested: {}",
protoc_version_string
);
}
::capnpc::CompilerCommand::new()
.file("proto/veilid.capnp")
.output_path(".")
.run()
.expect("compiling schema");
// Fix for missing __extenddftf2 on Android x86_64 Emulator
// If successful, update modification time
set_file_mtime("proto/veilid_capnp.rs", FileTime::now()).unwrap();
}
// Fix for missing __extenddftf2 on Android x86_64 Emulator
fn fix_android_emulator() {
let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap();
// if target_os == "android" || target_os == "linux" {
// println!("cargo:rustc-link-lib=stdc++");
// } else {
// println!("cargo:rustc-link-lib=c++");
// }
let target_arch = env::var("CARGO_CFG_TARGET_ARCH").unwrap();
if target_arch == "x86_64" && target_os == "android" {
let missing_library = "clang_rt.builtins-x86_64-android";
@ -142,3 +107,19 @@ fn main() {
println!("cargo:rustc-link-lib=static={missing_library}");
}
}
fn main() {
if std::env::var("DOCS_RS").is_ok()
|| std::env::var("CARGO_CFG_DOC").is_ok()
|| std::env::var("BUILD_DOCS").is_ok()
{
return;
}
if is_input_file_outdated("./proto/veilid.capnp", "./proto/veilid_capnp.rs").unwrap() {
println!("cargo:warning=rebuilding proto/veilid_capnp.rs because it is older than proto/veilid.capnp");
do_capnp_build();
}
fix_android_emulator();
}

7
veilid-core/src/intf/native/system.rs
View File

@ -191,3 +191,10 @@ pub async fn ptr_lookup(ip_addr: IpAddr) -> EyreResult<String> {
}
}
}
pub fn env_variable_is_defined<S: AsRef<str>>(s: S) -> bool {
match std::env::var(s.as_ref()) {
Ok(v) => !v.is_empty(),
Err(_) => false,
}
}

4
veilid-core/src/intf/wasm/system.rs
View File

@ -39,3 +39,7 @@ pub async fn txt_lookup<S: AsRef<str>>(_host: S) -> EyreResult<Vec<String>> {
pub async fn ptr_lookup(_ip_addr: IpAddr) -> EyreResult<String> {
bail!("wasm does not support ptr lookup")
}
pub fn env_variable_is_defined<S: AsRef<str>>(s: S) -> bool {
false
}

5
veilid-core/src/tests/common/test_dht.rs
View File

@ -334,6 +334,11 @@ async fn wait_for_public_internet_ready(api: &VeilidAPI) {
}
pub async fn test_all() {
if intf::env_variable_is_defined("CI") {
info!("skipping DHT test in CI");
return;
}
let (update_callback, config_callback) = setup_veilid_core();
let api = api_startup(update_callback, config_callback)
.await

4
veilid-core/src/tests/native/mod.rs
View File

@ -12,8 +12,6 @@ pub async fn run_all_tests() {
test_types::test_all().await;
info!("TEST: test_veilid_core");
test_veilid_core::test_all().await;
info!("TEST: test_dht");
test_dht::test_all().await;
info!("TEST: test_veilid_config");
test_veilid_config::test_all().await;
info!("TEST: test_connection_table");
@ -32,6 +30,8 @@ pub async fn run_all_tests() {
veilid_api::tests::test_serialize_json::test_all().await;
info!("TEST: routing_table::test_serialize_routing_table");
routing_table::tests::test_serialize_routing_table::test_all().await;
info!("TEST: test_dht");
test_dht::test_all().await;
info!("Finished unit tests");
}