From c01be8f62d8510f622f31e89b7d260a69c97f4dc Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Fri, 15 Sep 2023 11:45:12 -0400 Subject: [PATCH] capnp and protoc versioning --- .capnp_version | 1 + .earthlyignore | 4 - .protoc_version | 1 + Cargo.lock | 17 +++- Earthfile | 4 +- dev-setup/setup_windows.bat | 4 +- scripts/earthly/install_capnproto.sh | 9 +- scripts/earthly/install_protoc.sh | 21 +++- veilid-core/Cargo.toml | 4 +- veilid-core/build.rs | 98 +++++++++++++++++++ .../src/rpc_processor/coders/dial_info.rs | 18 +++- 11 files changed, 155 insertions(+), 26 deletions(-) create mode 100644 .capnp_version create mode 100644 .protoc_version diff --git a/.capnp_version b/.capnp_version new file mode 100644 index 00000000..7f207341 --- /dev/null +++ b/.capnp_version @@ -0,0 +1 @@ +1.0.1 \ No newline at end of file diff --git a/.earthlyignore b/.earthlyignore index 3523db04..c5e19354 100644 --- a/.earthlyignore +++ b/.earthlyignore @@ -1,8 +1,4 @@ .vscode .git -external/keyring-manager/android_test/.gradle -external/keyring-manager/android_test/app/build -external/keyring-manager/android_test/build -external/keyring-manager/android_test/local.properties target veilid-core/pkg diff --git a/.protoc_version b/.protoc_version new file mode 100644 index 00000000..ed18113a --- /dev/null +++ b/.protoc_version @@ -0,0 +1 @@ +24.3 \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 1883fa09..076c129f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -829,15 +829,18 @@ checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" [[package]] name = "capnp" -version = "0.17.2" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95e65021d89250bbfe7c2791789ced2c4bdc21b0e8bb59c64f3fd6145a5fd678" +checksum = "9eddbd729bd9742aa22d29e871a42ffea7f216a4ddbfdaf09ea88150ef2e7f76" +dependencies = [ + "embedded-io", +] [[package]] name = "capnpc" -version = "0.17.2" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbbc3763fb3e6635188e9cc51ee11a26f8777c553ca377430818dbebaaf6042b" +checksum = "5067f3c8ee94d993d03150153e9a57a6ff330127b1c1ad76475051e1cef79c2d" dependencies = [ "capnp", ] @@ -1635,6 +1638,12 @@ version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +[[package]] +name = "embedded-io" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bbadc628dc286b9ae02f0cb0f5411c056eb7487b72f0083203f115de94060" + [[package]] name = "enum-as-inner" version = "0.5.1" diff --git a/Earthfile b/Earthfile index c5d30cc6..5c1353cd 100644 --- a/Earthfile +++ b/Earthfile @@ -74,14 +74,14 @@ deps-linux: # Code + Linux deps code-linux: FROM +deps-linux - COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid + COPY --dir .cargo .capnp_version .protoc_version files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml WORKDIR /veilid # Code + Linux + Android deps code-android: FROM +deps-android - COPY --dir .cargo files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid + COPY --dir .cargo .capnp_version .protoc_version files scripts veilid-cli veilid-core veilid-server veilid-tools veilid-flutter veilid-wasm Cargo.lock Cargo.toml /veilid RUN cat /veilid/scripts/earthly/cargo-linux/config.toml >> /veilid/.cargo/config.toml RUN cat /veilid/scripts/earthly/cargo-android/config.toml >> /veilid/.cargo/config.toml WORKDIR /veilid diff --git a/dev-setup/setup_windows.bat b/dev-setup/setup_windows.bat index 6f0f3916..456ab3c7 100644 --- a/dev-setup/setup_windows.bat +++ b/dev-setup/setup_windows.bat @@ -21,8 +21,8 @@ IF NOT DEFINED PROTOC_FOUND ( FOR %%X IN (capnp.exe) DO (SET CAPNP_FOUND=%%~$PATH:X) IF NOT DEFINED CAPNP_FOUND ( - echo capnproto compiler ^(capnp^) is required but it's not installed. Install capnp 0.10.4 or higher. Ensure it is in your path. Aborting. - echo capnp is available here: https://capnproto.org/capnproto-c++-win32-0.10.4.zip + echo capnproto compiler ^(capnp^) is required but it's not installed. Install capnp 1.0.1 or higher. Ensure it is in your path. Aborting. + echo capnp is available here: https://capnproto.org/capnproto-c++-win32-1.0.1.zip goto end ) diff --git a/scripts/earthly/install_capnproto.sh b/scripts/earthly/install_capnproto.sh index 033dced7..8bb78283 100755 --- a/scripts/earthly/install_capnproto.sh +++ b/scripts/earthly/install_capnproto.sh @@ -1,9 +1,12 @@ #!/bin/bash +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +CAPNPROTO_VERSION=$(cat "$SCRIPTDIR/../../.capnp_version") + mkdir /tmp/capnproto-install pushd /tmp/capnproto-install -curl -O https://capnproto.org/capnproto-c++-0.10.4.tar.gz -tar zxf capnproto-c++-0.10.4.tar.gz -cd capnproto-c++-0.10.4 +curl -O https://capnproto.org/capnproto-c++-${CAPNPROTO_VERSION}.tar.gz +tar zxf capnproto-c++-${CAPNPROTO_VERSION}.tar.gz +cd capnproto-c++-${CAPNPROTO_VERSION} ./configure --without-openssl make -j$1 check if [ "$EUID" -ne 0 ]; then diff --git a/scripts/earthly/install_protoc.sh b/scripts/earthly/install_protoc.sh index 272008f5..d2ee75ad 100755 --- a/scripts/earthly/install_protoc.sh +++ b/scripts/earthly/install_protoc.sh @@ -1,13 +1,24 @@ #!/bin/bash -VERSION=23.3 +SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +PROTOC_VERSION=$(cat "$SCRIPTDIR/../../.protoc_version") + +UNAME_M=$(uname -m) +if [[ "$UNAME_M" == "x86_64" ]]; then + PROTOC_ARCH=x86_64 +elif [[ "$UNAME_M" == "aarch64" ]]; then + PROTOC_ARCH=aarch_64 +else + echo Unsupported build architecture + exit 1 +fi mkdir /tmp/protoc-install pushd /tmp/protoc-install -curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$VERSION/protoc-$VERSION-linux-x86_64.zip -unzip protoc-$VERSION-linux-x86_64.zip +curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v$PROTOC_VERSION/protoc-$PROTOC_VERSION-linux-$PROTOC_ARCH.zip +unzip protoc-$PROTOC_VERSION-linux-$PROTOC_ARCH.zip if [ "$EUID" -ne 0 ]; then if command -v checkinstall &> /dev/null; then - sudo checkinstall --pkgversion=$VERSION -y cp -r bin include /usr/local/ + sudo checkinstall --pkgversion=$PROTOC_VERSION -y cp -r bin include /usr/local/ cp *.deb ~ else sudo cp -r bin include /usr/local/ @@ -16,7 +27,7 @@ if [ "$EUID" -ne 0 ]; then sudo rm -rf /tmp/protoc-install else if command -v checkinstall &> /dev/null; then - checkinstall --pkgversion=$VERSION -y cp -r bin include /usr/local/ + checkinstall --pkgversion=$PROTOC_VERSION -y cp -r bin include /usr/local/ cp *.deb ~ else cp -r bin include /usr/local/ diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 96f2d9aa..d1ff28cd 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -129,7 +129,7 @@ trust-dns-resolver = { version = "0.22.0", optional = true } enum-as-inner = "=0.5.1" # temporary fix for trust-dns-resolver v0.22.0 # Serialization -capnp = { version = "0.17.2", default_features = false } +capnp = { version = "0.18.1", default_features = false } serde = { version = "1.0.183", features = ["derive"] } serde_json = { version = "1.0.105" } serde-big-array = "0.5.1" @@ -282,7 +282,7 @@ wasm-logger = "0.2.0" ### BUILD OPTIONS [build-dependencies] -capnpc = "0.17.2" +capnpc = "0.18.0" [package.metadata.wasm-pack.profile.release] wasm-opt = ["-O", "--enable-mutable-globals"] diff --git a/veilid-core/build.rs b/veilid-core/build.rs index 8c2b8364..6f4fe446 100644 --- a/veilid-core/build.rs +++ b/veilid-core/build.rs @@ -1,4 +1,102 @@ +use std::path::PathBuf; +use std::process::{Command, Stdio}; + +fn get_workspace_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("..") + .canonicalize() + .expect("want workspace dir") +} +fn get_desired_capnp_version_string() -> String { + std::fs::read_to_string(get_workspace_dir().join(".capnp_version")) + .expect("should find .capnp_version file") + .trim() + .to_owned() +} + +fn get_capnp_version_string() -> String { + let output = Command::new("capnpc") + .arg("--version") + .stdout(Stdio::piped()) + .output() + .expect("capnpc was not in the PATH"); + let s = String::from_utf8(output.stdout) + .expect("'capnpc --version' output was not a valid string") + .trim() + .to_owned(); + + if !s.starts_with("Cap'n Proto version ") { + panic!("invalid capnpc version string: {}", s); + } + s[20..].to_owned() +} + +fn get_desired_protoc_version_string() -> String { + std::fs::read_to_string(get_workspace_dir().join(".protoc_version")) + .expect("should find .protoc_version file") + .trim() + .to_owned() +} + +fn get_protoc_version_string() -> String { + let output = Command::new("protoc") + .arg("--version") + .stdout(Stdio::piped()) + .output() + .expect("protoc was not in the PATH"); + let s = String::from_utf8(output.stdout) + .expect("'protoc --version' output was not a valid string") + .trim() + .to_owned(); + + if !s.starts_with("libprotoc ") { + panic!("invalid protoc version string: {}", s); + } + s[10..].to_owned() +} + fn main() { + let desired_capnp_version_string = get_desired_capnp_version_string(); + let capnp_version_string = get_capnp_version_string(); + let desired_protoc_version_string = get_desired_protoc_version_string(); + let protoc_version_string = get_protoc_version_string(); + + // Check capnp version + let desired_capnp_major_version = + usize::from_str_radix(desired_capnp_version_string.split_once(".").unwrap().0, 10) + .expect("should be valid int"); + + if usize::from_str_radix(capnp_version_string.split_once(".").unwrap().0, 10) + .expect("should be valid int") + != desired_capnp_major_version + { + panic!( + "capnproto version should be major version 1, preferably {} but is {}", + desired_capnp_version_string, capnp_version_string + ); + } else if capnp_version_string != desired_capnp_version_string { + println!( + "capnproto version may be untested: {}", + capnp_version_string + ); + } + + // Check protoc version + let desired_protoc_major_version = + usize::from_str_radix(desired_protoc_version_string.split_once(".").unwrap().0, 10) + .expect("should be valid int"); + if usize::from_str_radix(protoc_version_string.split_once(".").unwrap().0, 10) + .expect("should be valid int") + < desired_protoc_major_version + { + panic!( + "capnproto version should be at least major version {} but is {}", + desired_protoc_major_version, protoc_version_string + ); + } else if protoc_version_string != desired_protoc_version_string { + println!("protoc version may be untested: {}", protoc_version_string); + } + ::capnpc::CompilerCommand::new() .file("proto/veilid.capnp") .run() diff --git a/veilid-core/src/rpc_processor/coders/dial_info.rs b/veilid-core/src/rpc_processor/coders/dial_info.rs index 400075b8..94d3c9ae 100644 --- a/veilid-core/src/rpc_processor/coders/dial_info.rs +++ b/veilid-core/src/rpc_processor/coders/dial_info.rs @@ -32,8 +32,13 @@ pub fn decode_dial_info(reader: &veilid_capnp::dial_info::Reader) -> Result { let wss = wss.map_err(RPCError::protocol)?; @@ -44,8 +49,13 @@ pub fn decode_dial_info(reader: &veilid_capnp::dial_info::Reader) -> Result