From 1654d03ad22f193269e9a83361247d75240c24cd Mon Sep 17 00:00:00 2001 From: Teknique Date: Tue, 23 May 2023 11:03:46 -0700 Subject: [PATCH 1/8] Added 1st simple rkyv serializer test --- veilid-core/src/tests/mod.rs | 1 + veilid-core/src/tests/native/mod.rs | 10 ++++++++++ veilid-core/src/veilid_api/mod.rs | 2 ++ veilid-core/src/veilid_api/tests/mod.rs | 3 +++ .../src/veilid_api/tests/test_serialize_rkyv.rs | 16 ++++++++++++++++ 5 files changed, 32 insertions(+) create mode 100644 veilid-core/src/veilid_api/tests/mod.rs create mode 100644 veilid-core/src/veilid_api/tests/test_serialize_rkyv.rs diff --git a/veilid-core/src/tests/mod.rs b/veilid-core/src/tests/mod.rs index 2a050ac5..609182dd 100644 --- a/veilid-core/src/tests/mod.rs +++ b/veilid-core/src/tests/mod.rs @@ -12,3 +12,4 @@ use super::*; pub use common::*; pub use crypto::tests::*; pub use network_manager::tests::*; +pub use veilid_api::tests::*; diff --git a/veilid-core/src/tests/native/mod.rs b/veilid-core/src/tests/native/mod.rs index d6fa1cfa..f7fcef81 100644 --- a/veilid-core/src/tests/native/mod.rs +++ b/veilid-core/src/tests/native/mod.rs @@ -3,6 +3,7 @@ use crate::crypto::tests::*; use crate::network_manager::tests::*; use crate::tests::common::*; +use crate::veilid_api::tests::*; use crate::*; /////////////////////////////////////////////////////////////////////////// @@ -163,5 +164,14 @@ cfg_if! { }) } + #[test] + #[serial] + fn run_test_serialize_rkyv() { + setup(); + block_on(async { + test_serialize_rkyv::test_all().await; + }) + } + } } diff --git a/veilid-core/src/veilid_api/mod.rs b/veilid-core/src/veilid_api/mod.rs index eb4cb312..d046b71f 100644 --- a/veilid-core/src/veilid_api/mod.rs +++ b/veilid-core/src/veilid_api/mod.rs @@ -7,6 +7,8 @@ mod routing_context; mod serialize_helpers; mod types; +pub mod tests; + pub use api::*; pub use debug::*; pub use error::*; diff --git a/veilid-core/src/veilid_api/tests/mod.rs b/veilid-core/src/veilid_api/tests/mod.rs new file mode 100644 index 00000000..97891419 --- /dev/null +++ b/veilid-core/src/veilid_api/tests/mod.rs @@ -0,0 +1,3 @@ +pub mod test_serialize_rkyv; + +use super::*; diff --git a/veilid-core/src/veilid_api/tests/test_serialize_rkyv.rs b/veilid-core/src/veilid_api/tests/test_serialize_rkyv.rs new file mode 100644 index 00000000..0566f7b7 --- /dev/null +++ b/veilid-core/src/veilid_api/tests/test_serialize_rkyv.rs @@ -0,0 +1,16 @@ +use crate::*; + +pub async fn test_simple_string() { + let plain = "basic string".to_string(); + let serialized = b"basic string\x0c\x00\x00\x00\xf4\xff\xff\xff".to_vec(); + + let a = to_rkyv(&plain); + assert_eq!(a.unwrap(), serialized); + + let b = from_rkyv::(serialized); + assert_eq!(b.unwrap(), plain); +} + +pub async fn test_all() { + test_simple_string().await; +} From f46fb878646bff944c94dd68d67e4bee38814c48 Mon Sep 17 00:00:00 2001 From: Teknique Date: Tue, 23 May 2023 11:04:36 -0700 Subject: [PATCH 2/8] Remove extra parens that annoy `cargo check` --- veilid-core/src/network_manager/connection_table.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/veilid-core/src/network_manager/connection_table.rs b/veilid-core/src/network_manager/connection_table.rs index 0a09eb21..b0879150 100644 --- a/veilid-core/src/network_manager/connection_table.rs +++ b/veilid-core/src/network_manager/connection_table.rs @@ -187,7 +187,7 @@ impl ConnectionTable { pub fn get_last_connection_by_remote(&self, remote: PeerAddress) -> Option { let mut inner = self.inner.lock(); - let id = inner.ids_by_remote.get(&remote).map(|v| v[(v.len() - 1)])?; + let id = inner.ids_by_remote.get(&remote).map(|v| v[v.len() - 1])?; let protocol_index = Self::protocol_to_index(remote.protocol_type()); let out = inner.conn_by_id[protocol_index].get(&id).unwrap(); Some(out.get_handle()) From f7c5154fab25d0d398852209d88ad2cb824d13ec Mon Sep 17 00:00:00 2001 From: Teknique Date: Wed, 24 May 2023 16:54:07 -0700 Subject: [PATCH 3/8] Basic framework for testing RoutingTable load/save --- veilid-core/src/routing_table/mod.rs | 25 +++++- veilid-core/src/routing_table/tests/mod.rs | 3 + .../src/routing_table/tests/test_serialize.rs | 80 +++++++++++++++++++ veilid-core/src/tests/mod.rs | 1 + veilid-core/src/tests/native/mod.rs | 9 +++ 5 files changed, 115 insertions(+), 3 deletions(-) create mode 100644 veilid-core/src/routing_table/tests/mod.rs create mode 100644 veilid-core/src/routing_table/tests/test_serialize.rs diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 04e21ee5..3665c3c9 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -13,6 +13,8 @@ mod stats_accounting; mod tasks; mod types; +pub mod tests; + use super::*; use crate::crypto::*; @@ -290,8 +292,8 @@ impl RoutingTable { debug!("finished routing table terminate"); } - /// Serialize routing table to table store - async fn save_buckets(&self) -> EyreResult<()> { + /// Serialize the routing table. + fn serialized_buckets(&self) -> EyreResult<(BTreeMap>>, Vec>)> { // Since entries are shared by multiple buckets per cryptokind // we need to get the list of all unique entries when serializing let mut all_entries: Vec> = Vec::new(); @@ -319,6 +321,13 @@ impl RoutingTable { all_entry_bytes.push(entry_bytes); } + Ok((serialized_bucket_map, all_entry_bytes)) + } + + /// Write the serialized routing table to the table store. + async fn save_buckets(&self) -> EyreResult<()> { + let (serialized_bucket_map, all_entry_bytes) = self.serialized_buckets()?; + let table_store = self.unlocked_inner.network_manager().table_store(); let tdb = table_store.open("routing_table", 1).await?; let dbx = tdb.transact(); @@ -333,7 +342,6 @@ impl RoutingTable { dbx.commit().await?; Ok(()) } - /// Deserialize routing table from table store async fn load_buckets(&self) -> EyreResult<()> { // Deserialize bucket map and all entries from the table store @@ -350,7 +358,18 @@ impl RoutingTable { // Reconstruct all entries let inner = &mut *self.inner.write(); + self.populate_routing_table(inner, serialized_bucket_map, all_entry_bytes)?; + Ok(()) + } + + /// Write the deserialized table store data to the routing table. + pub fn populate_routing_table( + &self, + inner: &mut RoutingTableInner, + serialized_bucket_map: BTreeMap>>, + all_entry_bytes: Vec>, + ) -> EyreResult<()> { let mut all_entries: Vec> = Vec::with_capacity(all_entry_bytes.len()); for entry_bytes in all_entry_bytes { let entryinner = diff --git a/veilid-core/src/routing_table/tests/mod.rs b/veilid-core/src/routing_table/tests/mod.rs new file mode 100644 index 00000000..8205daa2 --- /dev/null +++ b/veilid-core/src/routing_table/tests/mod.rs @@ -0,0 +1,3 @@ +pub mod test_serialize; + +use super::*; diff --git a/veilid-core/src/routing_table/tests/test_serialize.rs b/veilid-core/src/routing_table/tests/test_serialize.rs new file mode 100644 index 00000000..14df616c --- /dev/null +++ b/veilid-core/src/routing_table/tests/test_serialize.rs @@ -0,0 +1,80 @@ +use crate::*; + +fn fake_routing_table() -> routing_table::RoutingTable { + let veilid_config = VeilidConfig::new(); + let block_store = BlockStore::new(veilid_config.clone()); + let protected_store = ProtectedStore::new(veilid_config.clone()); + let table_store = TableStore::new(veilid_config.clone()); + let crypto = Crypto::new( + veilid_config.clone(), + table_store.clone(), + protected_store.clone(), + ); + let storage_manager = storage_manager::StorageManager::new( + veilid_config.clone(), + crypto.clone(), + protected_store.clone(), + table_store.clone(), + block_store.clone(), + ); + let network_manager = network_manager::NetworkManager::new( + veilid_config.clone(), + storage_manager, + protected_store.clone(), + table_store.clone(), + block_store.clone(), + crypto.clone(), + ); + routing_table::RoutingTable::new(network_manager) +} + +pub async fn test_routingtable_buckets_round_trip() { + let original = fake_routing_table(); + let copy = fake_routing_table(); + original.init().await.unwrap(); + copy.init().await.unwrap(); + + // Add lots of routes to `original` here to exercise all various types. + + let (serialized_bucket_map, all_entry_bytes) = original.serialized_buckets().unwrap(); + + copy.populate_routing_table( + &mut copy.inner.write(), + serialized_bucket_map, + all_entry_bytes, + ) + .unwrap(); + + let original_inner = &*original.inner.read(); + let copy_inner = &*copy.inner.read(); + + let routing_table_keys: Vec<_> = original_inner.buckets.keys().clone().collect(); + let copy_keys: Vec<_> = copy_inner.buckets.keys().clone().collect(); + + assert_eq!(routing_table_keys.len(), copy_keys.len()); + + for crypto in routing_table_keys { + // The same keys are present in the original and copy RoutingTables. + let original_buckets = original_inner.buckets.get(&crypto).unwrap(); + let copy_buckets = copy_inner.buckets.get(&crypto).unwrap(); + + // Recurse into RoutingTable.inner.buckets + for (left_buckets, right_buckets) in original_buckets.iter().zip(copy_buckets.iter()) { + // Recurse into RoutingTable.inner.buckets.entries + for ((left_crypto, left_entries), (right_crypto, right_entries)) in + left_buckets.entries().zip(right_buckets.entries()) + { + assert_eq!(left_crypto, right_crypto); + + assert_eq!( + format!("{:?}", left_entries), + format!("{:?}", right_entries) + ); + } + } + } +} + +pub async fn test_all() { + test_routingtable_buckets_round_trip().await; +} diff --git a/veilid-core/src/tests/mod.rs b/veilid-core/src/tests/mod.rs index 609182dd..8e9d815f 100644 --- a/veilid-core/src/tests/mod.rs +++ b/veilid-core/src/tests/mod.rs @@ -12,4 +12,5 @@ use super::*; pub use common::*; pub use crypto::tests::*; pub use network_manager::tests::*; +pub use routing_table::tests::*; pub use veilid_api::tests::*; diff --git a/veilid-core/src/tests/native/mod.rs b/veilid-core/src/tests/native/mod.rs index f7fcef81..57855174 100644 --- a/veilid-core/src/tests/native/mod.rs +++ b/veilid-core/src/tests/native/mod.rs @@ -2,6 +2,7 @@ #![cfg(not(target_arch = "wasm32"))] use crate::crypto::tests::*; use crate::network_manager::tests::*; +use crate::routing_table::tests::*; use crate::tests::common::*; use crate::veilid_api::tests::*; use crate::*; @@ -173,5 +174,13 @@ cfg_if! { }) } + #[test] + #[serial] + fn run_test_routing_table_serialize() { + setup(); + block_on(async { + routing_table::tests::test_serialize::test_all().await; + }) + } } } From 4d3d549a4a1c9da177a31f33452faeeedde9f739 Mon Sep 17 00:00:00 2001 From: Teknique Date: Wed, 24 May 2023 17:52:11 -0700 Subject: [PATCH 4/8] TableStore requires a ProtectedStore arg now --- veilid-core/src/routing_table/tests/test_serialize.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/veilid-core/src/routing_table/tests/test_serialize.rs b/veilid-core/src/routing_table/tests/test_serialize.rs index 14df616c..d959ded0 100644 --- a/veilid-core/src/routing_table/tests/test_serialize.rs +++ b/veilid-core/src/routing_table/tests/test_serialize.rs @@ -4,7 +4,7 @@ fn fake_routing_table() -> routing_table::RoutingTable { let veilid_config = VeilidConfig::new(); let block_store = BlockStore::new(veilid_config.clone()); let protected_store = ProtectedStore::new(veilid_config.clone()); - let table_store = TableStore::new(veilid_config.clone()); + let table_store = TableStore::new(veilid_config.clone(), protected_store.clone()); let crypto = Crypto::new( veilid_config.clone(), table_store.clone(), From ff711080ab0f17548b890275c889648cb03ae8f3 Mon Sep 17 00:00:00 2001 From: Teknique Date: Thu, 25 May 2023 09:59:55 -0700 Subject: [PATCH 5/8] Extract RoutingTable inline types --- veilid-core/src/routing_table/mod.rs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/veilid-core/src/routing_table/mod.rs b/veilid-core/src/routing_table/mod.rs index 3665c3c9..1d63fcb5 100644 --- a/veilid-core/src/routing_table/mod.rs +++ b/veilid-core/src/routing_table/mod.rs @@ -57,6 +57,8 @@ pub struct LowLevelPortInfo { } pub type RoutingTableEntryFilter<'t> = Box>) -> bool + Send + 't>; +pub type SerializedBuckets = Vec>; +pub type SerializedBucketMap = BTreeMap; #[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct RoutingTableHealth { @@ -293,13 +295,13 @@ impl RoutingTable { } /// Serialize the routing table. - fn serialized_buckets(&self) -> EyreResult<(BTreeMap>>, Vec>)> { + fn serialized_buckets(&self) -> EyreResult<(SerializedBucketMap, SerializedBuckets)> { // Since entries are shared by multiple buckets per cryptokind // we need to get the list of all unique entries when serializing let mut all_entries: Vec> = Vec::new(); // Serialize all buckets and get map of entries - let mut serialized_bucket_map: BTreeMap>> = BTreeMap::new(); + let mut serialized_bucket_map: SerializedBucketMap = BTreeMap::new(); { let mut entry_map: HashMap<*const BucketEntry, u32> = HashMap::new(); let inner = &*self.inner.read(); @@ -347,11 +349,11 @@ impl RoutingTable { // Deserialize bucket map and all entries from the table store let tstore = self.unlocked_inner.network_manager().table_store(); let tdb = tstore.open("routing_table", 1).await?; - let Some(serialized_bucket_map): Option>>> = tdb.load_rkyv(0, b"serialized_bucket_map").await? else { + let Some(serialized_bucket_map): Option = tdb.load_rkyv(0, b"serialized_bucket_map").await? else { log_rtab!(debug "no bucket map in saved routing table"); return Ok(()); }; - let Some(all_entry_bytes): Option>> = tdb.load_rkyv(0, b"all_entry_bytes").await? else { + let Some(all_entry_bytes): Option = tdb.load_rkyv(0, b"all_entry_bytes").await? else { log_rtab!(debug "no all_entry_bytes in saved routing table"); return Ok(()); }; @@ -367,8 +369,8 @@ impl RoutingTable { pub fn populate_routing_table( &self, inner: &mut RoutingTableInner, - serialized_bucket_map: BTreeMap>>, - all_entry_bytes: Vec>, + serialized_bucket_map: SerializedBucketMap, + all_entry_bytes: SerializedBuckets, ) -> EyreResult<()> { let mut all_entries: Vec> = Vec::with_capacity(all_entry_bytes.len()); for entry_bytes in all_entry_bytes { From f9a9949466f3a325a15abe89725105e3e6982f25 Mon Sep 17 00:00:00 2001 From: Teknique Date: Thu, 25 May 2023 10:42:25 -0700 Subject: [PATCH 6/8] Added serialization tests to iOS/Android and wasm --- veilid-core/src/tests/native/mod.rs | 7 ++++++- veilid-core/tests/web.rs | 12 ++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/veilid-core/src/tests/native/mod.rs b/veilid-core/src/tests/native/mod.rs index 57855174..f6675a6f 100644 --- a/veilid-core/src/tests/native/mod.rs +++ b/veilid-core/src/tests/native/mod.rs @@ -11,6 +11,7 @@ use crate::*; #[allow(dead_code)] pub async fn run_all_tests() { + // iOS and Android tests also run these. info!("TEST: test_host_interface"); test_host_interface::test_all().await; info!("TEST: test_types"); @@ -31,6 +32,10 @@ pub async fn run_all_tests() { test_crypto::test_all().await; info!("TEST: test_envelope_receipt"); test_envelope_receipt::test_all().await; + info!("TEST: veilid_api::test_serialize"); + veilid_api::tests::test_serialize_rkyv::test_all().await; + info!("TEST: routing_table::test_serialize"); + routing_table::tests::test_serialize::test_all().await; info!("Finished unit tests"); } @@ -170,7 +175,7 @@ cfg_if! { fn run_test_serialize_rkyv() { setup(); block_on(async { - test_serialize_rkyv::test_all().await; + veilid_api::tests::test_serialize_rkyv::test_all().await; }) } diff --git a/veilid-core/tests/web.rs b/veilid-core/tests/web.rs index 8d03d596..4e77ee6b 100644 --- a/veilid-core/tests/web.rs +++ b/veilid-core/tests/web.rs @@ -89,3 +89,15 @@ async fn exec_test_envelope_receipt() { setup(); test_envelope_receipt::test_all().await; } + +#[wasm_bindgen_test] +async fn veilid_api__test_serialize_rkyv() { + setup(); + veilid_api::test_serialize_rkyv::test_all().await; +} + +#[wasm_bindgen_test] +async fn routing_table__test_serialize() { + setup(); + routing_table::test_serialize::test_all().await; +} From 983176a9e7496132404b3d5e3e83acbd8c16e070 Mon Sep 17 00:00:00 2001 From: Teknique Date: Thu, 25 May 2023 10:56:24 -0700 Subject: [PATCH 7/8] =?UTF-8?q?terminate()=20what=20we=E2=80=99ve=20init()?= =?UTF-8?q?ed.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- veilid-core/src/routing_table/tests/test_serialize.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/veilid-core/src/routing_table/tests/test_serialize.rs b/veilid-core/src/routing_table/tests/test_serialize.rs index d959ded0..169796d9 100644 --- a/veilid-core/src/routing_table/tests/test_serialize.rs +++ b/veilid-core/src/routing_table/tests/test_serialize.rs @@ -73,6 +73,10 @@ pub async fn test_routingtable_buckets_round_trip() { } } } + + // Even if these are mocks, we should still practice good hygiene. + original.terminate().await; + copy.terminate().await; } pub async fn test_all() { From db4646a7160131260d43115de9875ef7fb6f71e3 Mon Sep 17 00:00:00 2001 From: Teknique Date: Thu, 25 May 2023 11:02:49 -0700 Subject: [PATCH 8/8] Warnings cleanups --- veilid-core/src/routing_table/tests/mod.rs | 2 -- veilid-core/src/tests/native/mod.rs | 4 ++-- veilid-core/src/veilid_api/tests/mod.rs | 2 -- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/veilid-core/src/routing_table/tests/mod.rs b/veilid-core/src/routing_table/tests/mod.rs index 8205daa2..209cc9d3 100644 --- a/veilid-core/src/routing_table/tests/mod.rs +++ b/veilid-core/src/routing_table/tests/mod.rs @@ -1,3 +1 @@ pub mod test_serialize; - -use super::*; diff --git a/veilid-core/src/tests/native/mod.rs b/veilid-core/src/tests/native/mod.rs index f6675a6f..911af7c3 100644 --- a/veilid-core/src/tests/native/mod.rs +++ b/veilid-core/src/tests/native/mod.rs @@ -2,9 +2,9 @@ #![cfg(not(target_arch = "wasm32"))] use crate::crypto::tests::*; use crate::network_manager::tests::*; -use crate::routing_table::tests::*; +use crate::routing_table; use crate::tests::common::*; -use crate::veilid_api::tests::*; +use crate::veilid_api; use crate::*; /////////////////////////////////////////////////////////////////////////// diff --git a/veilid-core/src/veilid_api/tests/mod.rs b/veilid-core/src/veilid_api/tests/mod.rs index 97891419..3c3d2f33 100644 --- a/veilid-core/src/veilid_api/tests/mod.rs +++ b/veilid-core/src/veilid_api/tests/mod.rs @@ -1,3 +1 @@ pub mod test_serialize_rkyv; - -use super::*;