api work

veilid-cli/src/client_api_connection.rs

@@ -92,7 +92,7 @@ impl veilid_client::Server for VeilidClientImpl {
             VeilidUpdate::Config(config) => {
                 self.comproc.update_config(config);
             }
-            VeilidUpdate::Route(route) => {
+            VeilidUpdate::RouteChange(route) => {
                 self.comproc.update_route(route);
             }
             VeilidUpdate::Shutdown => self.comproc.update_shutdown(),

veilid-cli/src/command_processor.rs

@@ -406,7 +406,7 @@ reply               - reply to an AppCall not handled directly by the server
     pub fn update_config(&mut self, config: veilid_core::VeilidStateConfig) {
         self.inner_mut().ui.set_config(config.config)
     }
-    pub fn update_route(&mut self, route: veilid_core::VeilidStateRoute) {
+    pub fn update_route(&mut self, route: veilid_core::VeilidRouteChange) {
         let mut out = String::new();
         if !route.dead_routes.is_empty() {
             out.push_str(&format!("Dead routes: {:?}", route.dead_routes));

veilid-core/src/routing_table/debug.rs

@@ -2,28 +2,6 @@ use super::*;
 use routing_table::tasks::bootstrap::BOOTSTRAP_TXT_VERSION_0;
 
 impl RoutingTable {
-    pub(crate) fn debug_info_nodeinfo(&self) -> String {
-        let mut out = String::new();
-        let inner = self.inner.read();
-        out += "Routing Table Info:\n";
-
-        out += &format!("   Node Ids: {}\n", self.unlocked_inner.node_ids());
-        out += &format!(
-            "   Self Latency Stats Accounting: {:#?}\n\n",
-            inner.self_latency_stats_accounting
-        );
-        out += &format!(
-            "   Self Transfer Stats Accounting: {:#?}\n\n",
-            inner.self_transfer_stats_accounting
-        );
-        out += &format!(
-            "   Self Transfer Stats: {:#?}\n\n",
-            inner.self_transfer_stats
-        );
-
-        out
-    }
-
     pub(crate) async fn debug_info_txtrecord(&self) -> String {
         let mut out = String::new();
 
@@ -71,14 +49,34 @@ impl RoutingTable {
                 node_ids,
                 some_hostname.unwrap()
             );
-            for short_url in short_urls {
+            out += &short_urls.join(",");
-                out += &format!(",{}", short_url);
-            }
             out += "\n";
         }
         out
     }
 
+    pub(crate) fn debug_info_nodeinfo(&self) -> String {
+        let mut out = String::new();
+        let inner = self.inner.read();
+        out += "Routing Table Info:\n";
+
+        out += &format!("   Node Ids: {}\n", self.unlocked_inner.node_ids());
+        out += &format!(
+            "   Self Latency Stats Accounting: {:#?}\n\n",
+            inner.self_latency_stats_accounting
+        );
+        out += &format!(
+            "   Self Transfer Stats Accounting: {:#?}\n\n",
+            inner.self_transfer_stats_accounting
+        );
+        out += &format!(
+            "   Self Transfer Stats: {:#?}\n\n",
+            inner.self_transfer_stats
+        );
+
+        out
+    }
+
     pub(crate) fn debug_info_dialinfo(&self) -> String {
         let ldis = self.dial_info_details(RoutingDomain::LocalNetwork);
         let gdis = self.dial_info_details(RoutingDomain::PublicInternet);

veilid-core/src/routing_table/route_spec_store/route_spec_store.rs

@@ -115,7 +115,7 @@ impl RouteSpecStore {
             dr
         };
 
-        let update = VeilidUpdate::Route(VeilidStateRoute {
+        let update = VeilidUpdate::RouteChange(VeilidRouteChange {
             dead_routes,
             dead_remote_routes,
         });

veilid-core/src/storage_manager/debug.rs

@@ -0,0 +1,18 @@
+use super::*;
+
+impl StorageManager {
+    pub(crate) async fn debug_local_records(&self) -> String {
+        let inner = self.inner.lock().await;
+        let Some(local_record_store) = &inner.local_record_store else {
+            return "not initialized".to_owned();
+        };
+        local_record_store.debug_records()
+    }
+    pub(crate) async fn debug_remote_records(&self) -> String {
+        let inner = self.inner.lock().await;
+        let Some(remote_record_store) = &inner.remote_record_store else {
+            return "not initialized".to_owned();
+        };
+        remote_record_store.debug_records()
+    }
+}

veilid-core/src/storage_manager/mod.rs

@@ -1,3 +1,4 @@
+mod debug;
 mod get_value;
 mod keys;
 mod record_store;

veilid-core/src/storage_manager/record_store.rs

@@ -516,4 +516,35 @@ where
         }
         self.purge_dead_records(false).await;
     }
+
+    pub(super) fn debug_records(&self) -> String {
+        // Dump fields in an abbreviated way
+        let mut out = String::new();
+
+        out += "Record Index:\n";
+        for (rik, rec) in &self.record_index {
+            out += &format!(
+                "  {} @ {} len={}\n",
+                rik.key.to_string(),
+                rec.last_touched().as_u64(),
+                rec.record_data_size()
+            );
+        }
+        out += &format!("Subkey Cache Count: {}\n", self.subkey_cache.len());
+        out += &format!(
+            "Subkey Cache Total Size: {}\n",
+            self.subkey_cache_total_size
+        );
+        out += &format!("Total Storage Space: {}\n", self.total_storage_space);
+        out += &format!("Dead Records: {}\n", self.dead_records.len());
+        for dr in &self.dead_records {
+            out += &format!("  {}\n", dr.0.key.to_string());
+        }
+        out += &format!("Changed Records: {}\n", self.changed_records.len());
+        for cr in &self.changed_records {
+            out += &format!("  {}\n", cr.key.to_string());
+        }
+
+        out
+    }
 }

veilid-core/src/veilid_api/debug.rs

@@ -874,6 +874,39 @@ impl VeilidAPI {
         }
     }
 
+    async fn debug_record_list(&self, args: Vec<String>) -> Result<String, VeilidAPIError> {
+        // <local|remote>
+        let storage_manager = self.storage_manager()?;
+
+        let scope = get_debug_argument_at(&args, 1, "debug_record_list", "scope", get_string)?;
+        let out = match scope.as_str() {
+            "local" => {
+                let mut out = format!("Local Records:\n");
+                out += &storage_manager.debug_local_records().await;
+                out
+            }
+            "remote" => {
+                let mut out = format!("Remote Records:\n");
+                out += &storage_manager.debug_remote_records().await;
+                out
+            }
+            _ => "Invalid scope\n".to_owned(),
+        };
+        return Ok(out);
+    }
+
+    async fn debug_record(&self, args: String) -> Result<String, VeilidAPIError> {
+        let args: Vec<String> = args.split_whitespace().map(|s| s.to_owned()).collect();
+
+        let command = get_debug_argument_at(&args, 0, "debug_record", "command", get_string)?;
+
+        if command == "list" {
+            self.debug_record_list(args).await
+        } else {
+            Ok(">>> Unknown command\n".to_owned())
+        }
+    }
+
     pub async fn debug_help(&self, _args: String) -> Result<String, VeilidAPIError> {
         Ok(r#">>> Debug commands:
         help
@@ -897,6 +930,7 @@ impl VeilidAPI {
               list
               import <blob>
               test <route>
+        record list <local|remote> 
 
         <destination> is:
          * direct:  <node>[+<safety>][<modifiers>]
@@ -953,6 +987,8 @@ impl VeilidAPI {
                 self.debug_restart(rest).await
             } else if arg == "route" {
                 self.debug_route(rest).await
+            } else if arg == "record" {
+                self.debug_record(rest).await
             } else {
                 Err(VeilidAPIError::generic("Unknown debug command"))
             }

veilid-core/src/veilid_api/types/veilid_state.rs

@@ -96,7 +96,7 @@ pub struct VeilidStateNetwork {
     Debug, Clone, PartialEq, Eq, Serialize, Deserialize, RkyvArchive, RkyvSerialize, RkyvDeserialize,
 )]
 #[archive_attr(repr(C), derive(CheckBytes))]
-pub struct VeilidStateRoute {
+pub struct VeilidRouteChange {
     pub dead_routes: Vec<RouteId>,
     pub dead_remote_routes: Vec<RouteId>,
 }
@@ -130,7 +130,7 @@ pub enum VeilidUpdate {
     Attachment(VeilidStateAttachment),
     Network(VeilidStateNetwork),
     Config(VeilidStateConfig),
-    Route(VeilidStateRoute),
+    RouteChange(VeilidRouteChange),
     ValueChange(VeilidValueChange),
     Shutdown,
 }

veilid-flutter/lib/veilid.dart

@@ -234,6 +234,201 @@ Object? veilidApiToEncodable(Object? value) {
   throw UnsupportedError('Cannot convert to JSON: $value');
 }
 
+//////////////////////////////////////
+/// Crypto
+
+typedef CryptoKind = String;
+const cryptoKindVLD0 = "VLD0";
+const cryptoKindNONE = "NONE";
+
+//////////////////////////////////////
+/// DHT Schema
+
+abstract class DHTSchema {
+  factory DHTSchema.fromJson(dynamic json) {
+    switch (json["kind"]) {
+      case "DFLT":
+        {
+          return DHTSchemaDFLT(oCnt: json["o_cnt"]);
+        }
+      case "SMPL":
+        {
+          return DHTSchemaSMPL(
+              oCnt: json["o_cnt"],
+              members: List<DHTSchemaMember>.from(
+                  json['members'].map((j) => DHTSchemaMember.fromJson(j))));
+        }
+      default:
+        {
+          throw VeilidAPIExceptionInternal(
+              "Invalid VeilidAPIException type: ${json['kind']}");
+        }
+    }
+  }
+  Map<String, dynamic> get json;
+}
+
+class DHTSchemaDFLT implements DHTSchema {
+  final int oCnt;
+  //
+  DHTSchemaDFLT({
+    required this.oCnt,
+  }) {
+    if (oCnt < 0 || oCnt > 65535) {
+      throw VeilidAPIExceptionInvalidArgument(
+          "value out of range", "oCnt", oCnt.toString());
+    }
+  }
+
+  @override
+  Map<String, dynamic> get json {
+    return {
+      'kind': "DFLT",
+      'o_cnt': oCnt,
+    };
+  }
+}
+
+class DHTSchemaMember {
+  String mKey;
+  int mCnt;
+
+  DHTSchemaMember({
+    required this.mKey,
+    required this.mCnt,
+  }) {
+    if (mCnt < 0 || mCnt > 65535) {
+      throw VeilidAPIExceptionInvalidArgument(
+          "value out of range", "mCnt", mCnt.toString());
+    }
+  }
+
+  Map<String, dynamic> get json {
+    return {
+      'm_key': mKey,
+      'm_cnt': mCnt,
+    };
+  }
+
+  DHTSchemaMember.fromJson(dynamic json)
+      : mKey = json['m_key'],
+        mCnt = json['m_cnt'];
+}
+
+class DHTSchemaSMPL implements DHTSchema {
+  final int oCnt;
+  final List<DHTSchemaMember> members;
+  //
+  DHTSchemaSMPL({
+    required this.oCnt,
+    required this.members,
+  }) {
+    if (oCnt < 0 || oCnt > 65535) {
+      throw VeilidAPIExceptionInvalidArgument(
+          "value out of range", "oCnt", oCnt.toString());
+    }
+  }
+  @override
+  Map<String, dynamic> get json {
+    return {
+      'kind': "SMPL",
+      'o_cnt': oCnt,
+      'members': members.map((p) => p.json).toList(),
+    };
+  }
+}
+
+//////////////////////////////////////
+/// DHTRecordDescriptor
+
+class DHTRecordDescriptor {
+  String key;
+  String owner;
+  String? ownerSecret;
+  DHTSchema schema;
+
+  DHTRecordDescriptor({
+    required this.key,
+    required this.owner,
+    this.ownerSecret,
+    required this.schema,
+  });
+
+  Map<String, dynamic> get json {
+    return {
+      'key': key,
+      'owner': owner,
+      'owner_secret': ownerSecret,
+      'schema': schema.json,
+    };
+  }
+
+  DHTRecordDescriptor.fromJson(dynamic json)
+      : key = json['key'],
+        owner = json['owner'],
+        ownerSecret = json['owner_secret'],
+        schema = DHTSchema.fromJson(json['schema']);
+}
+
+//////////////////////////////////////
+/// ValueSubkeyRange
+
+class ValueSubkeyRange {
+  final int low;
+  final int high;
+
+  ValueSubkeyRange({
+    required this.low,
+    required this.high,
+  }) {
+    if (low < 0 || low > high) {
+      throw VeilidAPIExceptionInvalidArgument(
+          "invalid range", "low", low.toString());
+    }
+    if (high < 0) {
+      throw VeilidAPIExceptionInvalidArgument(
+          "invalid range", "high", high.toString());
+    }
+  }
+
+  ValueSubkeyRange.fromJson(dynamic json)
+      : low = json[0],
+        high = json[1] {
+    if ((json as List<int>).length != 2) {
+      throw VeilidAPIExceptionInvalidArgument(
+          "not a pair of integers", "json", json.toString());
+    }
+  }
+
+  List<dynamic> get json {
+    return [low, high];
+  }
+}
+
+//////////////////////////////////////
+/// ValueData
+
+class ValueData {
+  final int seq;
+  final Uint8List data;
+  final String writer;
+
+  ValueData({
+    required this.seq,
+    required this.data,
+    required this.writer,
+  });
+
+  ValueData.fromJson(dynamic json)
+      : seq = json['seq'],
+        data = base64UrlNoPadDecode(json['data']),
+        writer = json['writer'];
+
+  Map<String, dynamic> get json {
+    return {'seq': seq, 'data': base64UrlNoPadEncode(data), 'writer': writer};
+  }
+}
+
 //////////////////////////////////////
 /// AttachmentState
 
@@ -1287,9 +1482,21 @@ abstract class VeilidUpdate {
         {
           return VeilidUpdateConfig(state: VeilidStateConfig.fromJson(json));
         }
-      case "Route":
+      case "RouteChange":
         {
-          return VeilidUpdateRoute(state: VeilidStateRoute.fromJson(json));
+          return VeilidUpdateRouteChange(
+              deadRoutes: List<String>.from(json['dead_routes'].map((j) => j)),
+              deadRemoteRoutes:
+                  List<String>.from(json['dead_remote_routes'].map((j) => j)));
+        }
+      case "ValueChange":
+        {
+          return VeilidUpdateValueChange(
+              key: json['key'],
+              subkeys: List<ValueSubkeyRange>.from(
+                  json['subkeys'].map((j) => ValueSubkeyRange.fromJson(j))),
+              count: json['count'],
+              valueData: ValueData.fromJson(json['value_data']));
         }
       default:
         {
@@ -1405,16 +1612,46 @@ class VeilidUpdateConfig implements VeilidUpdate {
   }
 }
 
-class VeilidUpdateRoute implements VeilidUpdate {
+class VeilidUpdateRouteChange implements VeilidUpdate {
-  final VeilidStateRoute state;
+  final List<String> deadRoutes;
+  final List<String> deadRemoteRoutes;
   //
-  VeilidUpdateRoute({required this.state});
+  VeilidUpdateRouteChange({
+    required this.deadRoutes,
+    required this.deadRemoteRoutes,
+  });
 
   @override
   Map<String, dynamic> get json {
-    var jsonRep = state.json;
+    return {
-    jsonRep['kind'] = "Route";
+      'dead_routes': deadRoutes.map((p) => p).toList(),
-    return jsonRep;
+      'dead_remote_routes': deadRemoteRoutes.map((p) => p).toList()
+    };
+  }
+}
+
+class VeilidUpdateValueChange implements VeilidUpdate {
+  final String key;
+  final List<ValueSubkeyRange> subkeys;
+  final int count;
+  final ValueData valueData;
+
+  //
+  VeilidUpdateValueChange({
+    required this.key,
+    required this.subkeys,
+    required this.count,
+    required this.valueData,
+  });
+
+  @override
+  Map<String, dynamic> get json {
+    return {
+      'key': key,
+      'subkeys': subkeys.map((p) => p.json).toList(),
+      'count': count,
+      'value_data': valueData.json,
+    };
   }
 }
 
@@ -1492,31 +1729,6 @@ class VeilidStateConfig {
   }
 }
 
-//////////////////////////////////////
-/// VeilidStateRoute
-
-class VeilidStateRoute {
-  final List<String> deadRoutes;
-  final List<String> deadRemoteRoutes;
-
-  VeilidStateRoute({
-    required this.deadRoutes,
-    required this.deadRemoteRoutes,
-  });
-
-  VeilidStateRoute.fromJson(dynamic json)
-      : deadRoutes = List<String>.from(json['dead_routes'].map((j) => j)),
-        deadRemoteRoutes =
-            List<String>.from(json['dead_remote_routes'].map((j) => j));
-
-  Map<String, dynamic> get json {
-    return {
-      'dead_routes': deadRoutes.map((p) => p).toList(),
-      'dead_remote_routes': deadRemoteRoutes.map((p) => p).toList()
-    };
-  }
-}
-
 //////////////////////////////////////
 /// VeilidState
 
@@ -1893,12 +2105,79 @@ class RouteBlob {
 
 //////////////////////////////////////
 /// VeilidRoutingContext
+
 abstract class VeilidRoutingContext {
   VeilidRoutingContext withPrivacy();
   VeilidRoutingContext withCustomPrivacy(Stability stability);
   VeilidRoutingContext withSequencing(Sequencing sequencing);
   Future<Uint8List> appCall(String target, Uint8List request);
   Future<void> appMessage(String target, Uint8List message);
+
+  Future<DHTRecordDescriptor> createDHTRecord(
+      CryptoKind kind, DHTSchema schema);
+xxx continue here
+  // pub async fn open_dht_record(
+  //     &self,
+  //     key: TypedKey,
+  //     writer: Option<KeyPair>,
+  // ) -> Result<DHTRecordDescriptor, VeilidAPIError> {
+  //     let storage_manager = self.api.storage_manager()?;
+  //     storage_manager
+  //         .open_record(key, writer, self.unlocked_inner.safety_selection)
+  //         .await
+  // }
+
+  // pub async fn close_dht_record(&self, key: TypedKey) -> Result<(), VeilidAPIError> {
+  //     let storage_manager = self.api.storage_manager()?;
+  //     storage_manager.close_record(key).await
+  // }
+
+  // pub async fn delete_dht_record(&self, key: TypedKey) -> Result<(), VeilidAPIError> {
+  //     let storage_manager = self.api.storage_manager()?;
+  //     storage_manager.delete_record(key).await
+  // }
+
+  // pub async fn get_dht_value(
+  //     &self,
+  //     key: TypedKey,
+  //     subkey: ValueSubkey,
+  //     force_refresh: bool,
+  // ) -> Result<Option<ValueData>, VeilidAPIError> {
+  //     let storage_manager = self.api.storage_manager()?;
+  //     storage_manager.get_value(key, subkey, force_refresh).await
+  // }
+
+  // pub async fn set_dht_value(
+  //     &self,
+  //     key: TypedKey,
+  //     subkey: ValueSubkey,
+  //     data: Vec<u8>,
+  // ) -> Result<Option<ValueData>, VeilidAPIError> {
+  //     let storage_manager = self.api.storage_manager()?;
+  //     storage_manager.set_value(key, subkey, data).await
+  // }
+
+  // pub async fn watch_dht_values(
+  //     &self,
+  //     key: TypedKey,
+  //     subkeys: &[ValueSubkeyRange],
+  //     expiration: Timestamp,
+  //     count: u32,
+  // ) -> Result<Timestamp, VeilidAPIError> {
+  //     let storage_manager = self.api.storage_manager()?;
+  //     storage_manager
+  //         .watch_values(key, subkeys, expiration, count)
+  //         .await
+  // }
+
+  // pub async fn cancel_dht_watch(
+  //     &self,
+  //     key: TypedKey,
+  //     subkeys: &[ValueSubkeyRange],
+  // ) -> Result<bool, VeilidAPIError> {
+  //     let storage_manager = self.api.storage_manager()?;
+  //     storage_manager.cancel_watch_values(key, subkeys).await
+  // }
 }
 
 /////////////////////////////////////

veilid-server/src/cmdline.rs

@@ -82,7 +82,7 @@ fn do_clap_matches(default_config_path: &OsStr) -> Result<clap::ArgMatches, clap
                 .long("generate-key-pair")
                 .takes_value(true)
                 .value_name("crypto_kind")
-                .default_missing_value("VLD0")
+                .default_missing_value("")
                 .help("Only generate a new keypair and print it")
                 .long_help("Generate a new keypair for a specific crypto kind and print both the key and its secret to the terminal, then exit immediately."),                
         )

veilid-server/src/main.rs

@@ -45,10 +45,27 @@ fn main() -> EyreResult<()> {
     // --- Generate DHT Key ---
     if matches.occurrences_of("generate-key-pair") != 0 {
         if let Some(ckstr) = matches.get_one::<String>("generate-key-pair") {
-            let ck: veilid_core::CryptoKind =
+            if ckstr == "" {
-                veilid_core::FourCC::from_str(ckstr).wrap_err("couldn't parse crypto kind")?;
+                let mut tks = veilid_core::TypedKeySet::new();
-            let tkp = veilid_core::Crypto::generate_keypair(ck).wrap_err("invalid crypto kind")?;
+                let mut tss = veilid_core::TypedSecretSet::new();
-            println!("{}", tkp.to_string());
+                for ck in veilid_core::VALID_CRYPTO_KINDS {
+                    let tkp = veilid_core::Crypto::generate_keypair(ck)
+                        .wrap_err("invalid crypto kind")?;
+                    tks.add(veilid_core::TypedKey::new(tkp.kind, tkp.value.key));
+                    tss.add(veilid_core::TypedSecret::new(tkp.kind, tkp.value.secret));
+                }
+                println!(
+                    "Public Keys:\n{}\nSecret Keys:\n{}\n",
+                    tks.to_string(),
+                    tss.to_string()
+                );
+            } else {
+                let ck: veilid_core::CryptoKind =
+                    veilid_core::FourCC::from_str(ckstr).wrap_err("couldn't parse crypto kind")?;
+                let tkp =
+                    veilid_core::Crypto::generate_keypair(ck).wrap_err("invalid crypto kind")?;
+                println!("{}", tkp.to_string());
+            }
             return Ok(());
         } else {
             bail!("missing crypto kind");