refactor api to clean up internals

veilid-core/src/storage_manager/mod.rs

@@ -1,12 +1,21 @@
 mod keys;
+mod record;
+mod record_data;
 mod record_store;
 mod record_store_limits;
-mod value_record;
+mod signed_value_data;
+mod signed_value_descriptor;
+mod value_detail;
 
 use keys::*;
+use record::*;
+use record_data::*;
 use record_store::*;
 use record_store_limits::*;
-use value_record::*;
+
+pub use signed_value_data::*;
+pub use signed_value_descriptor::*;
+pub use value_detail::*;
 
 use super::*;
 use crate::rpc_processor::*;
@@ -150,12 +159,28 @@ impl StorageManager {
         debug!("finished storage manager shutdown");
     }
 
-    async fn new_local_record(&self, key: TypedKey, record: Record) -> Result<(), VeilidAPIError> {
+    /// # DHT Key = Hash(ownerKeyKind) of: [ ownerKeyValue, schema ]
+    fn get_key(&self, vcrypto: CryptoSystemVersion, record: &Record) -> TypedKey {
+        let compiled = record.descriptor().schema_data();
+        let mut hash_data = Vec::<u8>::with_capacity(PUBLIC_KEY_LENGTH + 4 + compiled.len());
+        hash_data.extend_from_slice(&vcrypto.kind().0);
+        hash_data.extend_from_slice(&record.owner().bytes);
+        hash_data.extend_from_slice(compiled);
+        let hash = vcrypto.generate_hash(&hash_data);
+        TypedKey::new(vcrypto.kind(), hash)
+    }
+
+    async fn new_local_record(
+        &self,
+        vcrypto: CryptoSystemVersion,
+        record: Record,
+    ) -> Result<(), VeilidAPIError> {
         // add value record to record store
         let mut inner = self.inner.lock();
         let Some(local_record_store) = inner.local_record_store.as_mut() else {
             apibail_generic!("not initialized");
         };
+        let key = self.get_key(vcrypto.clone(), &record);
         local_record_store.new_record(key, record).await
     }
 
@@ -170,15 +195,25 @@ impl StorageManager {
             apibail_generic!("unsupported cryptosystem");
         };
 
+        // Compile the dht schema
+        let schema_data = schema.compile();
+
         // New values require a new owner key
-        let keypair = vcrypto.generate_keypair();
-        let key = TypedKey::new(kind, keypair.key);
-        let secret = keypair.secret;
+        let owner = vcrypto.generate_keypair();
+
+        // Make a signed value descriptor for this dht value
+        let signed_value_descriptor = SignedValueDescriptor::new(owner.key, )
 
         // Add new local value record
         let cur_ts = get_aligned_timestamp();
-        let record = Record::new(cur_ts, Some(secret), schema, safety_selection);
-        self.new_local_record(key, record)
+        let record = Record::new(
+            cur_ts,
+            owner.key,
+            Some(owner.secret),
+            schema,
+            safety_selection,
+        );
+        self.new_local_record(vcrypto.clone(), record)
             .await
             .map_err(VeilidAPIError::internal)?;
 

veilid-core/src/storage_manager/record.rs

@@ -3,46 +3,15 @@ use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as
 use serde::*;
 
 #[derive(
-    Clone,
-    Debug,
-    Default,
-    PartialEq,
-    Eq,
-    Serialize,
-    Deserialize,
-    RkyvArchive,
-    RkyvSerialize,
-    RkyvDeserialize,
-)]
-#[archive_attr(repr(C), derive(CheckBytes))]
-pub struct RecordData {
-    pub value_data: ValueData,
-    pub signature: Signature,
-}
-
-impl RecordData {
-    pub fn total_size(&self) -> usize {
-        mem::size_of::<ValueData>() + self.value_data.data().len()
-    }
-}
-
-#[derive(
-    Clone,
-    Debug,
-    Default,
-    PartialEq,
-    Eq,
-    Serialize,
-    Deserialize,
-    RkyvArchive,
-    RkyvSerialize,
-    RkyvDeserialize,
+    Clone, Debug, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
 )]
 #[archive_attr(repr(C), derive(CheckBytes))]
 pub struct Record {
     last_touched_ts: Timestamp,
-    secret: Option<SecretKey>,
-    schema: DHTSchema,
+    descriptor: SignedValueDescriptor,
+    subkey_count: usize,
+
+    owner_secret: Option<SecretKey>,
     safety_selection: SafetySelection,
     record_data_size: usize,
 }
@@ -50,21 +19,31 @@ pub struct Record {
 impl Record {
     pub fn new(
         cur_ts: Timestamp,
-        secret: Option<SecretKey>,
-        schema: DHTSchema,
+        descriptor: SignedValueDescriptor,
+        owner_secret: Option<SecretKey>,
         safety_selection: SafetySelection,
-    ) -> Self {
-        Self {
+    ) -> Result<Self, VeilidAPIError> {
+        let schema = descriptor.schema()?;
+        let subkey_count = schema.subkey_count();
+        Ok(Self {
             last_touched_ts: cur_ts,
-            secret,
-            schema,
+            descriptor,
+            subkey_count,
+            owner_secret,
             safety_selection,
             record_data_size: 0,
-        }
+        })
+    }
+
+    pub fn descriptor(&self) -> &SignedValueDescriptor {
+        &self.descriptor
+    }
+    pub fn owner(&self) -> &PublicKey {
+        self.descriptor.owner()
     }
 
     pub fn subkey_count(&self) -> usize {
-        self.schema.subkey_count()
+        self.subkey_count
     }
 
     pub fn touch(&mut self, cur_ts: Timestamp) {
@@ -83,7 +62,12 @@ impl Record {
         self.record_data_size
     }
 
+    pub fn schema(&self) -> DHTSchema {
+        // unwrap is safe here because descriptor is immutable and set in new()
+        self.descriptor.schema().unwrap()
+    }
+
     pub fn total_size(&self) -> usize {
-        mem::size_of::<Record>() + self.schema.data_size() + self.record_data_size
+        mem::size_of::<Record>() + self.descriptor.total_size() + self.record_data_size
     }
 }

veilid-core/src/storage_manager/record_data.rs

@@ -0,0 +1,33 @@
+use super::*;
+use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
+use serde::*;
+
+#[derive(
+    Clone,
+    Debug,
+    PartialEq,
+    Eq,
+    PartialOrd,
+    Ord,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct RecordData {
+    signed_value_data: SignedValueData,
+}
+
+impl RecordData {
+    pub fn new(signed_value_data: SignedValueData) -> Self {
+        Self { signed_value_data }
+    }
+    pub fn signed_value_data(&self) -> &SignedValueData {
+        &self.signed_value_data
+    }
+    pub fn total_size(&self) -> usize {
+        mem::size_of::<RecordData>() + self.signed_value_data.value_data().data().len()
+    }
+}

veilid-core/src/storage_manager/record_store.rs

@@ -286,7 +286,7 @@ impl RecordStore {
         &mut self,
         key: TypedKey,
         subkey: ValueSubkey,
-    ) -> Result<Option<RecordData>, VeilidAPIError> {
+    ) -> Result<Option<SignedValueData>, VeilidAPIError> {
         // record from index
         let rtk = RecordTableKey { key };
         let subkey_count = {
@@ -314,7 +314,7 @@ impl RecordStore {
         // If subkey exists in subkey cache, use that
         let stk = SubkeyTableKey { key, subkey };
         if let Some(record_data) = self.subkey_cache.get_mut(&stk) {
-            let out = record_data.clone();
+            let out = record_data.signed_value_data().clone();
 
             return Ok(Some(out));
         }
@@ -323,7 +323,7 @@ impl RecordStore {
             .load_rkyv::<RecordData>(0, &stk.bytes())
             .map_err(VeilidAPIError::internal)?
         {
-            let out = record_data.clone();
+            let out = record_data.signed_value_data().clone();
 
             // Add to cache, do nothing with lru out
             self.add_to_subkey_cache(stk, record_data);
@@ -338,10 +338,10 @@ impl RecordStore {
         &mut self,
         key: TypedKey,
         subkey: ValueSubkey,
-        record_data: RecordData,
+        signed_value_data: SignedValueData,
     ) -> Result<(), VeilidAPIError> {
         // Check size limit for data
-        if record_data.value_data.data().len() > self.limits.max_subkey_size {
+        if signed_value_data.value_data().data().len() > self.limits.max_subkey_size {
             return Err(VeilidAPIError::generic("record subkey too large"));
         }
 
@@ -388,6 +388,9 @@ impl RecordStore {
             }
         }
 
+        // Make new record data
+        let record_data = RecordData::new(signed_value_data);
+
         // Check new total record size
         let new_record_data_size = record_data.total_size();
         let new_total_size = total_size + new_record_data_size - prior_record_data_size;

veilid-core/src/storage_manager/signed_value_data.rs

@@ -0,0 +1,92 @@
+use super::*;
+use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
+use serde::*;
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+///
+
+#[derive(
+    Clone,
+    Debug,
+    PartialOrd,
+    PartialEq,
+    Eq,
+    Ord,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct SignedValueData {
+    value_data: ValueData,
+    signature: Signature,
+}
+impl SignedValueData {
+    pub fn new(
+        value_data: ValueData,
+        owner: PublicKey,
+        subkey: ValueSubkey,
+        signature: Signature,
+        vcrypto: CryptoSystemVersion,
+    ) -> Result<Self, VeilidAPIError> {
+        let node_info_bytes = Self::make_signature_bytes(&value_data, &owner, subkey)?;
+
+        // validate signature
+        vcrypto.verify(&value_data.writer(), &node_info_bytes, &signature)?;
+        Ok(Self {
+            value_data,
+            signature,
+        })
+    }
+
+    pub fn make_signature(
+        value_data: ValueData,
+        owner: PublicKey,
+        subkey: ValueSubkey,
+        vcrypto: CryptoSystemVersion,
+        writer_secret: SecretKey,
+    ) -> Result<Self, VeilidAPIError> {
+        let node_info_bytes = Self::make_signature_bytes(&value_data, &owner, subkey)?;
+
+        // create signature
+        let signature = vcrypto.sign(&value_data.writer(), &writer_secret, &node_info_bytes)?;
+        Ok(Self {
+            value_data,
+            signature,
+        })
+    }
+
+    pub fn value_data(&self) -> &ValueData {
+        &self.value_data
+    }
+
+    pub fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    pub fn total_size(&self) -> usize {
+        (mem::size_of::<Self>() - mem::size_of::<ValueData>()) + self.value_data.total_size()
+    }
+
+    fn make_signature_bytes(
+        value_data: &ValueData,
+        owner: &PublicKey,
+        subkey: ValueSubkey,
+    ) -> Result<Vec<u8>, VeilidAPIError> {
+        let mut node_info_bytes =
+            Vec::with_capacity(PUBLIC_KEY_LENGTH + 4 + 4 + value_data.data().len());
+
+        // Add owner to signature
+        node_info_bytes.extend_from_slice(&owner.bytes);
+        // Add subkey to signature
+        node_info_bytes.extend_from_slice(&subkey.to_le_bytes());
+        // Add sequence number to signature
+        node_info_bytes.extend_from_slice(&value_data.seq().to_le_bytes());
+        // Add data to signature
+        node_info_bytes.extend_from_slice(value_data.data());
+
+        Ok(node_info_bytes)
+    }
+}

veilid-core/src/storage_manager/signed_value_descriptor.rs

@@ -0,0 +1,78 @@
+use super::*;
+use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
+use serde::*;
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+///
+
+#[derive(
+    Clone,
+    Debug,
+    PartialOrd,
+    PartialEq,
+    Eq,
+    Ord,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct SignedValueDescriptor {
+    owner: PublicKey,
+    schema_data: Vec<u8>,
+    signature: Signature,
+}
+impl SignedValueDescriptor {
+    pub fn new(
+        owner: PublicKey,
+        schema_data: Vec<u8>,
+        signature: Signature,
+        vcrypto: CryptoSystemVersion,
+    ) -> Result<Self, VeilidAPIError> {
+        // validate signature
+        vcrypto.verify(&owner, &schema_data, &signature)?;
+
+        Ok(Self {
+            owner,
+            schema_data,
+            signature,
+        })
+    }
+
+    pub fn owner(&self) -> &PublicKey {
+        &self.owner
+    }
+
+    pub fn schema_data(&self) -> &[u8] {
+        &self.schema_data
+    }
+
+    pub fn schema(&self) -> Result<DHTSchema, VeilidAPIError> {
+        DHTSchema::try_from(self.schema_data.as_slice())
+    }
+
+    pub fn signature(&self) -> &Signature {
+        &self.signature
+    }
+
+    pub fn make_signature(
+        owner: PublicKey,
+        schema_data: Vec<u8>,
+        vcrypto: CryptoSystemVersion,
+        owner_secret: SecretKey,
+    ) -> Result<Self, VeilidAPIError> {
+        // create signature
+        let signature = vcrypto.sign(&owner, &owner_secret, &schema_data)?;
+        Ok(Self {
+            owner,
+            schema_data,
+            signature,
+        })
+    }
+
+    pub fn total_size(&self) -> usize {
+        mem::size_of::<Self>() + self.schema_data.len()
+    }
+}

veilid-core/src/storage_manager/value_detail.rs

@@ -0,0 +1,43 @@
+use super::*;
+use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
+use serde::*;
+
+/////////////////////////////////////////////////////////////////////////////////////////////////////
+///
+
+#[derive(
+    Clone,
+    Debug,
+    PartialOrd,
+    PartialEq,
+    Eq,
+    Ord,
+    Serialize,
+    Deserialize,
+    RkyvArchive,
+    RkyvSerialize,
+    RkyvDeserialize,
+)]
+#[archive_attr(repr(C), derive(CheckBytes))]
+pub struct ValueDetail {
+    signed_value_data: SignedValueData,
+    descriptor: Option<SignedValueDescriptor>,
+}
+
+impl ValueDetail {
+    pub fn new(
+        signed_value_data: SignedValueData,
+        descriptor: Option<SignedValueDescriptor>,
+    ) -> Self {
+        Self {
+            signed_value_data,
+            descriptor,
+        }
+    }
+    pub fn signed_value_data(&self) -> &SignedValueData {
+        &self.signed_value_data
+    }
+    pub fn descriptor(&self) -> Option<&SignedValueDescriptor> {
+        self.descriptor.as_ref()
+    }
+}