diff --git a/veilid-core/src/crypto/byte_array_types.rs b/veilid-core/src/crypto/byte_array_types.rs index ff75900c..7922d32b 100644 --- a/veilid-core/src/crypto/byte_array_types.rs +++ b/veilid-core/src/crypto/byte_array_types.rs @@ -7,8 +7,6 @@ use core::hash::Hash; use data_encoding::BASE64URL_NOPAD; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; - ////////////////////////////////////////////////////////////////////// /// Length of a public key in bytes diff --git a/veilid-core/src/crypto/types/mod.rs b/veilid-core/src/crypto/types/mod.rs index 0c9b76d2..8b8de7de 100644 --- a/veilid-core/src/crypto/types/mod.rs +++ b/veilid-core/src/crypto/types/mod.rs @@ -5,8 +5,6 @@ use core::convert::TryInto; use core::fmt; use core::hash::Hash; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; - /// Cryptography version fourcc code pub type CryptoKind = FourCC; diff --git a/veilid-core/src/intf/native/protected_store.rs b/veilid-core/src/intf/native/protected_store.rs index 9bc08620..47176ecd 100644 --- a/veilid-core/src/intf/native/protected_store.rs +++ b/veilid-core/src/intf/native/protected_store.rs @@ -1,7 +1,6 @@ use crate::*; use data_encoding::BASE64URL_NOPAD; use keyring_manager::*; -use rkyv::{bytecheck::CheckBytes, Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; use std::path::Path; pub struct ProtectedStoreInner { diff --git a/veilid-core/src/intf/table_db.rs b/veilid-core/src/intf/table_db.rs index 194a19fe..485760f1 100644 --- a/veilid-core/src/intf/table_db.rs +++ b/veilid-core/src/intf/table_db.rs @@ -1,8 +1,4 @@ use crate::*; -use rkyv::{ - bytecheck::CheckBytes, Archive as RkyvArchive, Deserialize as RkyvDeserialize, - Serialize as RkyvSerialize, -}; cfg_if! { if #[cfg(target_arch = "wasm32")] { diff --git a/veilid-core/src/lib.rs b/veilid-core/src/lib.rs index c9accd40..1a446dbb 100644 --- a/veilid-core/src/lib.rs +++ b/veilid-core/src/lib.rs @@ -42,9 +42,18 @@ pub use veilid_tools as tools; use enumset::*; use rkyv::{ - bytecheck, bytecheck::CheckBytes, Archive as RkyvArchive, Deserialize as RkyvDeserialize, - Serialize as RkyvSerialize, + bytecheck, bytecheck::CheckBytes, de::deserializers::SharedDeserializeMap, with::Skip, + Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, }; +type RkyvSerializer = rkyv::ser::serializers::CompositeSerializer< + rkyv::ser::serializers::AlignedSerializer, + rkyv::ser::serializers::FallbackScratch< + rkyv::ser::serializers::HeapScratch<1024>, + rkyv::ser::serializers::AllocScratch, + >, + rkyv::ser::serializers::SharedSerializeMap, +>; +type RkyvDefaultValidator<'t> = rkyv::validation::validators::DefaultValidator<'t>; use serde::*; pub mod veilid_capnp { diff --git a/veilid-core/src/network_manager/types/mod.rs b/veilid-core/src/network_manager/types/mod.rs index e5b494f8..3c1c1e9d 100644 --- a/veilid-core/src/network_manager/types/mod.rs +++ b/veilid-core/src/network_manager/types/mod.rs @@ -25,7 +25,3 @@ pub use peer_address::*; pub use protocol_type::*; pub use signal_info::*; pub use socket_address::*; - -use enumset::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; diff --git a/veilid-core/src/routing_table/bucket_entry.rs b/veilid-core/src/routing_table/bucket_entry.rs index 0f1919d2..e6a992ff 100644 --- a/veilid-core/src/routing_table/bucket_entry.rs +++ b/veilid-core/src/routing_table/bucket_entry.rs @@ -1,8 +1,6 @@ use super::*; use core::sync::atomic::{AtomicU32, Ordering}; -use rkyv::{ - with::Skip, Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, -}; + /// Reliable pings are done with increased spacing between pings diff --git a/veilid-core/src/routing_table/route_spec_store/mod.rs b/veilid-core/src/routing_table/route_spec_store/mod.rs index 302b2b6f..618da7f8 100644 --- a/veilid-core/src/routing_table/route_spec_store/mod.rs +++ b/veilid-core/src/routing_table/route_spec_store/mod.rs @@ -16,9 +16,6 @@ pub use route_spec_store_content::*; pub use route_stats::*; use crate::veilid_api::*; -use rkyv::{ - with::Skip, Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize, -}; /// The size of the remote private route cache const REMOTE_PRIVATE_ROUTE_CACHE_SIZE: usize = 1024; diff --git a/veilid-core/src/routing_table/types/mod.rs b/veilid-core/src/routing_table/types/mod.rs index f86299bd..217c5d48 100644 --- a/veilid-core/src/routing_table/types/mod.rs +++ b/veilid-core/src/routing_table/types/mod.rs @@ -19,7 +19,3 @@ pub use routing_domain::*; pub use signed_direct_node_info::*; pub use signed_node_info::*; pub use signed_relayed_node_info::*; - -use enumset::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; diff --git a/veilid-core/src/rpc_processor/mod.rs b/veilid-core/src/rpc_processor/mod.rs index 37efa402..0e43f208 100644 --- a/veilid-core/src/rpc_processor/mod.rs +++ b/veilid-core/src/rpc_processor/mod.rs @@ -425,6 +425,9 @@ impl RPCProcessor { Err(RPCError::unimplemented("search_dht_multi_key")).map_err(logthru_rpc!(error)) } +get rid of multi key, finish resolve node with find_node_rpc, then do putvalue/getvalue, probably in storagemanager. + + /// Search the DHT for a specific node corresponding to a key unless we have that node in our routing table already, and return the node reference /// Note: This routine can possible be recursive, hence the SendPinBoxFuture async form pub fn resolve_node( diff --git a/veilid-core/src/storage_manager/mod.rs b/veilid-core/src/storage_manager/mod.rs index e20ba720..4dec1ce2 100644 --- a/veilid-core/src/storage_manager/mod.rs +++ b/veilid-core/src/storage_manager/mod.rs @@ -200,7 +200,12 @@ impl StorageManager { } /// # DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ] - fn get_key(vcrypto: CryptoSystemVersion, record: &Record) -> TypedKey { + fn get_key(vcrypto: CryptoSystemVersion, record: &Record) -> TypedKey + where + D: RkyvArchive + RkyvSerialize, + for<'t> ::Archived: CheckBytes>, + ::Archived: RkyvDeserialize, + { let compiled = record.descriptor().schema_data(); let mut hash_data = Vec::::with_capacity(PUBLIC_KEY_LENGTH + 4 + compiled.len()); hash_data.extend_from_slice(&vcrypto.kind().0); @@ -213,7 +218,7 @@ impl StorageManager { async fn lock(&self) -> Result, VeilidAPIError> { let inner = asyncmutex_lock_arc!(&self.inner); if !inner.initialized { - apibail_generic!("not initialized"); + apibail_not_initialized!(); } Ok(inner) } @@ -260,6 +265,18 @@ impl StorageManager { .await } + async fn do_get_value( + &self, + mut inner: AsyncMutexGuardArc, + key: TypedKey, + subkey: ValueSubkey, + ) -> Result, VeilidAPIError> { + let Some(rpc_processor) = inner.rpc_processor.clone() else { + apibail_not_initialized!(); + }; + + // + } async fn open_record_inner( &self, mut inner: AsyncMutexGuardArc, @@ -267,17 +284,34 @@ impl StorageManager { writer: Option, safety_selection: SafetySelection, ) -> Result { + // Ensure the record is closed + if inner.opened_records.contains_key(&key) { + return Err(VeilidAPIError::generic( + "record is already open and should be closed first", + )); + } + // Get cryptosystem let Some(vcrypto) = self.unlocked_inner.crypto.get(key.kind) else { apibail_generic!("unsupported cryptosystem"); }; // See if we have a local record already or not - let cb = |r: &Record| { + let cb = |r: &mut Record| { // Process local record + + // Keep the safety selection we opened the record with + r.detail_mut().safety_selection = safety_selection; + + // Return record details (r.owner().clone(), r.schema()) }; - if let Some((owner, schema)) = inner.local_record_store.unwrap().with_record(key, cb) { + if let Some((owner, schema)) = inner + .local_record_store + .as_mut() + .unwrap() + .with_record_mut(key, cb) + { // Had local record // If the writer we chose is also the owner, we have the owner secret @@ -293,27 +327,23 @@ impl StorageManager { }; // Write open record - inner.opened_records.insert(key, OpenedRecord { writer }); + inner.opened_records.insert(key, OpenedRecord::new(writer)); // Make DHT Record Descriptor to return - let descriptor = DHTRecordDescriptor { - key, - owner, - owner_secret, - schema, - }; + let descriptor = DHTRecordDescriptor::new(key, owner, owner_secret, schema); Ok(descriptor) } else { - // No record yet + // No record yet, try to get it from the network + self.do_get_value(inner, key, 0).await // Make DHT Record Descriptor to return - let descriptor = DHTRecordDescriptor { - key, - owner, - owner_secret, - schema, - }; - Ok(descriptor) + // let descriptor = DHTRecordDescriptor { + // key, + // owner, + // owner_secret, + // schema, + // }; + // Ok(descriptor) } } @@ -352,7 +382,8 @@ impl StorageManager { self.close_record_inner(inner, key).await?; } - // Remove + // Remove the record from the local store + //inner.local_record_store.unwrap().de unimplemented!(); } diff --git a/veilid-core/src/storage_manager/record_store.rs b/veilid-core/src/storage_manager/record_store.rs index 06b659c4..966de820 100644 --- a/veilid-core/src/storage_manager/record_store.rs +++ b/veilid-core/src/storage_manager/record_store.rs @@ -7,7 +7,12 @@ use super::*; use hashlink::LruCache; -pub struct RecordStore { +pub struct RecordStore +where + D: RkyvArchive + RkyvSerialize, + for<'t> ::Archived: CheckBytes>, + ::Archived: RkyvDeserialize, +{ table_store: TableStore, name: String, limits: RecordStoreLimits, @@ -25,7 +30,12 @@ pub struct RecordStore { purge_dead_records_mutex: Arc>, } -impl RecordStore { +impl RecordStore +where + D: RkyvArchive + RkyvSerialize, + for<'t> ::Archived: CheckBytes>, + ::Archived: RkyvDeserialize, +{ pub fn new(table_store: TableStore, name: &str, limits: RecordStoreLimits) -> Self { let subkey_cache_size = limits.subkey_cache_size as usize; Self { @@ -56,10 +66,10 @@ impl RecordStore { // Pull record index from table into a vector to ensure we sort them let record_table_keys = record_table.get_keys(0)?; - let mut record_index_saved: Vec<(RecordTableKey, Record)> = + let mut record_index_saved: Vec<(RecordTableKey, Record)> = Vec::with_capacity(record_table_keys.len()); for rtk in record_table_keys { - if let Some(vr) = record_table.load_rkyv::(0, &rtk)? { + if let Some(vr) = record_table.load_rkyv::>(0, &rtk)? { let rik = RecordTableKey::try_from(rtk.as_ref())?; record_index_saved.push((rik, vr)); } @@ -288,6 +298,27 @@ impl RecordStore { out } + pub fn with_record_mut(&mut self, key: TypedKey, f: F) -> Option + where + F: FnOnce(&mut Record) -> R, + { + // Get record from index + let mut out = None; + let rtk = RecordTableKey { key }; + if let Some(record) = self.record_index.get_mut(&rtk) { + // Callback + out = Some(f(record)); + + // Touch + record.touch(get_aligned_timestamp()); + } + if out.is_some() { + self.mark_record_changed(rtk); + } + + out + } + pub async fn get_subkey( &mut self, key: TypedKey, diff --git a/veilid-core/src/storage_manager/types/local_record_detail.rs b/veilid-core/src/storage_manager/types/local_record_detail.rs index 8c02f85e..e632e749 100644 --- a/veilid-core/src/storage_manager/types/local_record_detail.rs +++ b/veilid-core/src/storage_manager/types/local_record_detail.rs @@ -1,8 +1,5 @@ use super::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; - /// Information required to handle locally opened records #[derive( Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, @@ -11,5 +8,5 @@ use serde::*; pub struct LocalRecordDetail { /// The last 'safety selection' used when creating/opening this record. /// Even when closed, this safety selection applies to republication attempts by the system. - safety_selection: SafetySelection, + pub safety_selection: SafetySelection, } diff --git a/veilid-core/src/storage_manager/types/record.rs b/veilid-core/src/storage_manager/types/record.rs index 6b07eafe..51a2edd0 100644 --- a/veilid-core/src/storage_manager/types/record.rs +++ b/veilid-core/src/storage_manager/types/record.rs @@ -1,12 +1,15 @@ use super::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; #[derive( Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, )] #[archive_attr(repr(C), derive(CheckBytes))] -pub struct Record { +pub struct Record +where + D: RkyvArchive + RkyvSerialize, + for<'t> ::Archived: CheckBytes>, + ::Archived: RkyvDeserialize, +{ descriptor: SignedValueDescriptor, subkey_count: usize, last_touched_ts: Timestamp, @@ -14,7 +17,12 @@ pub struct Record { detail: D, } -impl Record { +impl Record +where + D: RkyvArchive + RkyvSerialize, + for<'t> ::Archived: CheckBytes>, + ::Archived: RkyvDeserialize, +{ pub fn new( cur_ts: Timestamp, descriptor: SignedValueDescriptor, diff --git a/veilid-core/src/storage_manager/types/record_data.rs b/veilid-core/src/storage_manager/types/record_data.rs index e439ed67..a9f8ed51 100644 --- a/veilid-core/src/storage_manager/types/record_data.rs +++ b/veilid-core/src/storage_manager/types/record_data.rs @@ -1,6 +1,4 @@ use super::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; #[derive( Clone, diff --git a/veilid-core/src/storage_manager/types/remote_record_detail.rs b/veilid-core/src/storage_manager/types/remote_record_detail.rs index 40d41895..e835faa6 100644 --- a/veilid-core/src/storage_manager/types/remote_record_detail.rs +++ b/veilid-core/src/storage_manager/types/remote_record_detail.rs @@ -1,8 +1,5 @@ use super::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; - #[derive( Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize, )] diff --git a/veilid-core/src/storage_manager/types/signed_value_data.rs b/veilid-core/src/storage_manager/types/signed_value_data.rs index 29c69dce..5f2759f7 100644 --- a/veilid-core/src/storage_manager/types/signed_value_data.rs +++ b/veilid-core/src/storage_manager/types/signed_value_data.rs @@ -1,6 +1,4 @@ use super::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; ///////////////////////////////////////////////////////////////////////////////////////////////////// /// diff --git a/veilid-core/src/storage_manager/types/signed_value_descriptor.rs b/veilid-core/src/storage_manager/types/signed_value_descriptor.rs index 917d3dfc..5cef1f2a 100644 --- a/veilid-core/src/storage_manager/types/signed_value_descriptor.rs +++ b/veilid-core/src/storage_manager/types/signed_value_descriptor.rs @@ -1,6 +1,4 @@ use super::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; ///////////////////////////////////////////////////////////////////////////////////////////////////// /// diff --git a/veilid-core/src/veilid_api/error.rs b/veilid-core/src/veilid_api/error.rs index 7e04b947..f2ae12e4 100644 --- a/veilid-core/src/veilid_api/error.rs +++ b/veilid-core/src/veilid_api/error.rs @@ -1,5 +1,13 @@ use super::*; +#[allow(unused_macros)] +#[macro_export] +macro_rules! apibail_not_initialized { + () => { + return Err(VeilidAPIError::not_initialized()) + }; +} + #[allow(unused_macros)] #[macro_export] macro_rules! apibail_timeout { diff --git a/veilid-core/src/veilid_api/routing_context.rs b/veilid-core/src/veilid_api/routing_context.rs index ef0607d6..ed950234 100644 --- a/veilid-core/src/veilid_api/routing_context.rs +++ b/veilid-core/src/veilid_api/routing_context.rs @@ -220,7 +220,7 @@ impl RoutingContext { ) -> Result { let storage_manager = self.api.storage_manager()?; storage_manager - .open_record(key, secret, self.unlocked_inner.safety_selection) + .open_record(key, writer, self.unlocked_inner.safety_selection) .await } diff --git a/veilid-core/src/veilid_config.rs b/veilid-core/src/veilid_config.rs index a45d7f62..c6dc1ddf 100644 --- a/veilid-core/src/veilid_config.rs +++ b/veilid-core/src/veilid_config.rs @@ -1,6 +1,4 @@ use crate::*; -use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize}; -use serde::*; //////////////////////////////////////////////////////////////////////////////////////////////// pub type ConfigCallbackReturn = Result, VeilidAPIError>;