liz/veilid
1
0
Fork 0
Code Pull Requests Packages Releases Activity

removing dev branch, many changes

Browse Source
This commit is contained in:
John Smith
2023-05-29 19:24:57 +00:00
parent 1430f3f656
commit 0a890c8707
250 changed files with 18084 additions and 8040 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
veilid-core/src/intf/mod.rs
View File

@@ -1,4 +1,4 @@
mod table_db;
use super::*;
#[cfg(target_arch = "wasm32")]
mod wasm;
@@ -8,3 +8,5 @@ pub use wasm::*;
mod native;
#[cfg(not(target_arch = "wasm32"))]
pub use native::*;
pub static KNOWN_PROTECTED_STORE_KEYS: [&'static str; 2] = ["device_encryption_key", "_test_key"];

4
veilid-core/src/intf/native/mod.rs
View File

@@ -1,13 +1,13 @@
mod block_store;
mod protected_store;
mod system;
mod table_store;
pub use block_store::*;
pub use protected_store::*;
pub use system::*;
pub use table_store::*;
#[cfg(target_os = "android")]
pub mod android;
pub mod network_interfaces;
use super::*;

33
veilid-core/src/intf/native/protected_store.rs
View File

@@ -1,7 +1,6 @@
use crate::*;
use super::*;
use data_encoding::BASE64URL_NOPAD;
use keyring_manager::*;
use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
use std::path::Path;
pub struct ProtectedStoreInner {
@@ -30,18 +29,12 @@ impl ProtectedStore {
#[instrument(level = "trace", skip(self), err)]
pub async fn delete_all(&self) -> EyreResult<()> {
// Delete all known keys
if self.remove_user_secret("node_id").await? {
debug!("deleted protected_store key 'node_id'");
}
if self.remove_user_secret("node_id_secret").await? {
debug!("deleted protected_store key 'node_id_secret'");
}
if self.remove_user_secret("_test_key").await? {
debug!("deleted protected_store key '_test_key'");
}
if self.remove_user_secret("RouteSpecStore").await? {
debug!("deleted protected_store key 'RouteSpecStore'");
for kpsk in &KNOWN_PROTECTED_STORE_KEYS {
if let Err(e) = self.remove_user_secret(kpsk).await {
error!("failed to delete '{}': {}", kpsk, e);
} else {
debug!("deleted table '{}'", kpsk);
}
}
Ok(())
}
@@ -65,9 +58,8 @@ impl ProtectedStore {
|| c.protected_store.allow_insecure_fallback)
&& inner.keyring_manager.is_none()
{
let insecure_fallback_directory =
Path::new(&c.protected_store.insecure_fallback_directory);
let insecure_keyring_file = insecure_fallback_directory.to_owned().join(format!(
let directory = Path::new(&c.protected_store.directory);
let insecure_keyring_file = directory.to_owned().join(format!(
"insecure_keyring{}",
if c.namespace.is_empty() {
"".to_owned()
@@ -153,7 +145,7 @@ impl ProtectedStore {
pub async fn save_user_secret_rkyv<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
K: AsRef<str> + fmt::Debug,
T: RkyvSerialize<rkyv::ser::serializers::AllocSerializer<1024>>,
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let v = to_rkyv(value)?;
self.save_user_secret(key, &v).await
@@ -175,9 +167,8 @@ impl ProtectedStore {
K: AsRef<str> + fmt::Debug,
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> bytecheck::CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived:
RkyvDeserialize<T, rkyv::de::deserializers::SharedDeserializeMap>,
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = self.load_user_secret(key).await?;
let b = match out {

2
veilid-core/src/intf/native/system.rs
View File

@@ -2,7 +2,7 @@
use crate::*;
pub async fn get_outbound_relay_peer() -> Option<crate::veilid_api::PeerInfo> {
pub async fn get_outbound_relay_peer() -> Option<crate::routing_table::PeerInfo> {
panic!("Native Veilid should never require an outbound relay");
}

147
veilid-core/src/intf/native/table_store.rs
View File

@@ -1,147 +0,0 @@
use crate::intf::table_db::TableDBInner;
pub use crate::intf::table_db::{TableDB, TableDBTransaction};
use crate::*;
use keyvaluedb_sqlite::*;
use std::path::PathBuf;
struct TableStoreInner {
opened: BTreeMap<String, Weak<Mutex<TableDBInner>>>,
}
/// Veilid Table Storage
/// Database for storing key value pairs persistently across runs
#[derive(Clone)]
pub struct TableStore {
config: VeilidConfig,
inner: Arc<Mutex<TableStoreInner>>,
}
impl TableStore {
fn new_inner() -> TableStoreInner {
TableStoreInner {
opened: BTreeMap::new(),
}
}
pub(crate) fn new(config: VeilidConfig) -> Self {
Self {
config,
inner: Arc::new(Mutex::new(Self::new_inner())),
}
}
/// Delete all known tables
pub async fn delete_all(&self) {
if let Err(e) = self.delete("crypto_caches").await {
error!("failed to delete 'crypto_caches': {}", e);
}
if let Err(e) = self.delete("RouteSpecStore").await {
error!("failed to delete 'RouteSpecStore': {}", e);
}
if let Err(e) = self.delete("routing_table").await {
error!("failed to delete 'routing_table': {}", e);
}
}
pub(crate) async fn init(&self) -> EyreResult<()> {
Ok(())
}
pub(crate) async fn terminate(&self) {
assert!(
self.inner.lock().opened.is_empty(),
"all open databases should have been closed"
);
}
pub(crate) fn on_table_db_drop(&self, table: String) {
let mut inner = self.inner.lock();
if inner.opened.remove(&table).is_none() {
unreachable!("should have removed an item");
}
}
fn get_dbpath(&self, table: &str) -> EyreResult<PathBuf> {
if !table
.chars()
.all(|c| char::is_alphanumeric(c) || c == '_' || c == '-')
{
bail!("table name '{}' is invalid", table);
}
let c = self.config.get();
let tablestoredir = c.table_store.directory.clone();
std::fs::create_dir_all(&tablestoredir).wrap_err("failed to create tablestore path")?;
let dbpath: PathBuf = [tablestoredir, String::from(table)].iter().collect();
Ok(dbpath)
}
fn get_table_name(&self, table: &str) -> EyreResult<String> {
if !table
.chars()
.all(|c| char::is_alphanumeric(c) || c == '_' || c == '-')
{
bail!("table name '{}' is invalid", table);
}
let c = self.config.get();
let namespace = c.namespace.clone();
Ok(if namespace.is_empty() {
table.to_string()
} else {
format!("_ns_{}_{}", namespace, table)
})
}
/// Get or create a TableDB database table. If the column count is greater than an
/// existing TableDB's column count, the database will be upgraded to add the missing columns
pub async fn open(&self, name: &str, column_count: u32) -> EyreResult<TableDB> {
let table_name = self.get_table_name(name)?;
let mut inner = self.inner.lock();
if let Some(table_db_weak_inner) = inner.opened.get(&table_name) {
match TableDB::try_new_from_weak_inner(table_db_weak_inner.clone()) {
Some(tdb) => {
return Ok(tdb);
}
None => {
inner.opened.remove(&table_name);
}
};
}
let dbpath = self.get_dbpath(&table_name)?;
// Ensure permissions are correct
ensure_file_private_owner(&dbpath)?;
let cfg = DatabaseConfig::with_columns(column_count);
let db = Database::open(&dbpath, cfg).wrap_err("failed to open tabledb")?;
// Ensure permissions are correct
ensure_file_private_owner(&dbpath)?;
trace!(
"opened table store '{}' at path '{:?}' with {} columns",
name,
dbpath,
column_count
);
let table_db = TableDB::new(table_name.clone(), self.clone(), db);
inner.opened.insert(table_name, table_db.weak_inner());
Ok(table_db)
}
/// Delete a TableDB table by name
pub async fn delete(&self, name: &str) -> EyreResult<bool> {
let table_name = self.get_table_name(name)?;
let inner = self.inner.lock();
if inner.opened.contains_key(&table_name) {
bail!("Not deleting table that is still opened");
}
let dbpath = self.get_dbpath(&table_name)?;
let ret = std::fs::remove_file(dbpath).is_ok();
Ok(ret)
}
}

276
veilid-core/src/intf/table_db.rs
View File

@@ -1,276 +0,0 @@
use crate::*;
use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
cfg_if! {
if #[cfg(target_arch = "wasm32")] {
use keyvaluedb_web::*;
use keyvaluedb::*;
} else {
use keyvaluedb_sqlite::*;
use keyvaluedb::*;
}
}
pub struct TableDBInner {
table: String,
table_store: TableStore,
database: Database,
}
impl fmt::Debug for TableDBInner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "TableDBInner(table={})", self.table)
}
}
impl Drop for TableDBInner {
fn drop(&mut self) {
self.table_store.on_table_db_drop(self.table.clone());
}
}
#[derive(Debug, Clone)]
pub struct TableDB {
inner: Arc<Mutex<TableDBInner>>,
}
impl TableDB {
pub(super) fn new(table: String, table_store: TableStore, database: Database) -> Self {
Self {
inner: Arc::new(Mutex::new(TableDBInner {
table,
table_store,
database,
})),
}
}
pub(super) fn try_new_from_weak_inner(weak_inner: Weak<Mutex<TableDBInner>>) -> Option<Self> {
weak_inner.upgrade().map(|table_db_inner| Self {
inner: table_db_inner,
})
}
pub(super) fn weak_inner(&self) -> Weak<Mutex<TableDBInner>> {
Arc::downgrade(&self.inner)
}
/// Get the total number of columns in the TableDB
pub fn get_column_count(&self) -> EyreResult<u32> {
let db = &self.inner.lock().database;
db.num_columns().wrap_err("failed to get column count: {}")
}
/// Get the list of keys in a column of the TableDB
pub fn get_keys(&self, col: u32) -> EyreResult<Vec<Box<[u8]>>> {
let db = &self.inner.lock().database;
let mut out: Vec<Box<[u8]>> = Vec::new();
db.iter(col, None, &mut |kv| {
out.push(kv.0.clone().into_boxed_slice());
Ok(true)
})
.wrap_err("failed to get keys for column")?;
Ok(out)
}
/// Start a TableDB write transaction. The transaction object must be committed or rolled back before dropping.
pub fn transact(&self) -> TableDBTransaction {
let dbt = {
let db = &self.inner.lock().database;
db.transaction()
};
TableDBTransaction::new(self.clone(), dbt)
}
/// Store a key with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store(&self, col: u32, key: &[u8], value: &[u8]) -> EyreResult<()> {
let db = self.inner.lock().database.clone();
let mut dbt = db.transaction();
dbt.put(col, key, value);
db.write(dbt).await.wrap_err("failed to store key")
}
/// Store a key in rkyv format with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
where
T: RkyvSerialize<rkyv::ser::serializers::AllocSerializer<1024>>,
{
let v = to_rkyv(value)?;
let db = self.inner.lock().database.clone();
let mut dbt = db.transaction();
dbt.put(col, key, v.as_slice());
db.write(dbt).await.wrap_err("failed to store key")
}
/// Store a key in json format with a value in a column in the TableDB. Performs a single transaction immediately.
pub async fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
where
T: serde::Serialize,
{
let v = serde_json::to_vec(value)?;
let db = self.inner.lock().database.clone();
let mut dbt = db.transaction();
dbt.put(col, key, v.as_slice());
db.write(dbt).await.wrap_err("failed to store key")
}
/// Read a key from a column in the TableDB immediately.
pub fn load(&self, col: u32, key: &[u8]) -> EyreResult<Option<Vec<u8>>> {
let db = self.inner.lock().database.clone();
db.get(col, key).wrap_err("failed to get key")
}
/// Read an rkyv key from a column in the TableDB immediately
pub fn load_rkyv<T>(&self, col: u32, key: &[u8]) -> EyreResult<Option<T>>
where
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> bytecheck::CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived:
RkyvDeserialize<T, rkyv::de::deserializers::SharedDeserializeMap>,
{
let db = self.inner.lock().database.clone();
let out = db.get(col, key).wrap_err("failed to get key")?;
let b = match out {
Some(v) => v,
None => {
return Ok(None);
}
};
let obj = from_rkyv(b)?;
Ok(Some(obj))
}
/// Read an serde-json key from a column in the TableDB immediately
pub fn load_json<T>(&self, col: u32, key: &[u8]) -> EyreResult<Option<T>>
where
T: for<'de> serde::Deserialize<'de>,
{
let db = self.inner.lock().database.clone();
let out = db.get(col, key).wrap_err("failed to get key")?;
let b = match out {
Some(v) => v,
None => {
return Ok(None);
}
};
let obj = serde_json::from_slice(&b)?;
Ok(Some(obj))
}
/// Delete key with from a column in the TableDB
pub async fn delete(&self, col: u32, key: &[u8]) -> EyreResult<bool> {
let db = self.inner.lock().database.clone();
let found = db.get(col, key).wrap_err("failed to get key")?;
match found {
None => Ok(false),
Some(_) => {
let mut dbt = db.transaction();
dbt.delete(col, key);
db.write(dbt).await.wrap_err("failed to delete key")?;
Ok(true)
}
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
struct TableDBTransactionInner {
dbt: Option<DBTransaction>,
}
impl fmt::Debug for TableDBTransactionInner {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"TableDBTransactionInner({})",
match &self.dbt {
Some(dbt) => format!("len={}", dbt.ops.len()),
None => "".to_owned(),
}
)
}
}
/// A TableDB transaction
/// Atomically commits a group of writes or deletes to the TableDB
#[derive(Debug, Clone)]
pub struct TableDBTransaction {
db: TableDB,
inner: Arc<Mutex<TableDBTransactionInner>>,
}
impl TableDBTransaction {
fn new(db: TableDB, dbt: DBTransaction) -> Self {
Self {
db,
inner: Arc::new(Mutex::new(TableDBTransactionInner { dbt: Some(dbt) })),
}
}
/// Commit the transaction. Performs all actions atomically.
pub async fn commit(self) -> EyreResult<()> {
let dbt = {
let mut inner = self.inner.lock();
inner
.dbt
.take()
.ok_or_else(|| eyre!("transaction already completed"))?
};
let db = self.db.inner.lock().database.clone();
db.write(dbt)
.await
.wrap_err("commit failed, transaction lost")
}
/// Rollback the transaction. Does nothing to the TableDB.
pub fn rollback(self) {
let mut inner = self.inner.lock();
inner.dbt = None;
}
/// Store a key with a value in a column in the TableDB
pub fn store(&self, col: u32, key: &[u8], value: &[u8]) {
let mut inner = self.inner.lock();
inner.dbt.as_mut().unwrap().put(col, key, value);
}
/// Store a key in rkyv format with a value in a column in the TableDB
pub fn store_rkyv<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
where
T: RkyvSerialize<rkyv::ser::serializers::AllocSerializer<1024>>,
{
let v = to_rkyv(value)?;
let mut inner = self.inner.lock();
inner.dbt.as_mut().unwrap().put(col, key, v.as_slice());
Ok(())
}
/// Store a key in rkyv format with a value in a column in the TableDB
pub fn store_json<T>(&self, col: u32, key: &[u8], value: &T) -> EyreResult<()>
where
T: serde::Serialize,
{
let v = serde_json::to_vec(value)?;
let mut inner = self.inner.lock();
inner.dbt.as_mut().unwrap().put(col, key, v.as_slice());
Ok(())
}
/// Delete key with from a column in the TableDB
pub fn delete(&self, col: u32, key: &[u8]) {
let mut inner = self.inner.lock();
inner.dbt.as_mut().unwrap().delete(col, key);
}
}
impl Drop for TableDBTransactionInner {
fn drop(&mut self) {
if self.dbt.is_some() {
warn!("Dropped transaction without commit or rollback");
}
}
}

4
veilid-core/src/intf/wasm/mod.rs
View File

@@ -1,9 +1,9 @@
mod block_store;
mod protected_store;
mod system;
mod table_store;
pub use block_store::*;
pub use protected_store::*;
pub use system::*;
pub use table_store::*;
use super::*;

32
veilid-core/src/intf/wasm/protected_store.rs
View File

@@ -1,6 +1,9 @@
use crate::*;
use super::*;
use data_encoding::BASE64URL_NOPAD;
use rkyv::{Archive as RkyvArchive, Deserialize as RkyvDeserialize, Serialize as RkyvSerialize};
use rkyv::{
bytecheck::CheckBytes, Archive as RkyvArchive, Deserialize as RkyvDeserialize,
Serialize as RkyvSerialize,
};
use web_sys::*;
@@ -16,18 +19,12 @@ impl ProtectedStore {
#[instrument(level = "trace", skip(self), err)]
pub async fn delete_all(&self) -> EyreResult<()> {
// Delete all known keys
if self.remove_user_secret("node_id").await? {
debug!("deleted protected_store key 'node_id'");
}
if self.remove_user_secret("node_id_secret").await? {
debug!("deleted protected_store key 'node_id_secret'");
}
if self.remove_user_secret("_test_key").await? {
debug!("deleted protected_store key '_test_key'");
}
if self.remove_user_secret("RouteSpecStore").await? {
debug!("deleted protected_store key 'RouteSpecStore'");
for kpsk in &KNOWN_PROTECTED_STORE_KEYS {
if let Err(e) = self.remove_user_secret(kpsk).await {
error!("failed to delete '{}': {}", kpsk, e);
} else {
debug!("deleted table '{}'", kpsk);
}
}
Ok(())
}
@@ -133,7 +130,7 @@ impl ProtectedStore {
pub async fn save_user_secret_rkyv<K, T>(&self, key: K, value: &T) -> EyreResult<bool>
where
K: AsRef<str> + fmt::Debug,
T: RkyvSerialize<rkyv::ser::serializers::AllocSerializer<1024>>,
T: RkyvSerialize<DefaultVeilidRkyvSerializer>,
{
let v = to_rkyv(value)?;
self.save_user_secret(key, &v).await
@@ -155,9 +152,8 @@ impl ProtectedStore {
K: AsRef<str> + fmt::Debug,
T: RkyvArchive,
<T as RkyvArchive>::Archived:
for<'t> bytecheck::CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived:
RkyvDeserialize<T, rkyv::de::deserializers::SharedDeserializeMap>,
for<'t> CheckBytes<rkyv::validation::validators::DefaultValidator<'t>>,
<T as RkyvArchive>::Archived: RkyvDeserialize<T, VeilidSharedDeserializeMap>,
{
let out = self.load_user_secret(key).await?;
let b = match out {

2
veilid-core/src/intf/wasm/system.rs
View File

@@ -2,7 +2,7 @@ use crate::*;
//use js_sys::*;
pub async fn get_outbound_relay_peer() -> Option<crate::veilid_api::PeerInfo> {
pub async fn get_outbound_relay_peer() -> Option<crate::routing_table::PeerInfo> {
// unimplemented!
None
}

151
veilid-core/src/intf/wasm/table_store.rs
View File

@@ -1,151 +0,0 @@
use crate::intf::table_db::TableDBInner;
pub use crate::intf::table_db::{TableDB, TableDBTransaction};
use crate::*;
use keyvaluedb_web::*;
struct TableStoreInner {
opened: BTreeMap<String, Weak<Mutex<TableDBInner>>>,
}
#[derive(Clone)]
pub struct TableStore {
config: VeilidConfig,
inner: Arc<Mutex<TableStoreInner>>,
async_lock: Arc<AsyncMutex<()>>,
}
impl TableStore {
fn new_inner() -> TableStoreInner {
TableStoreInner {
opened: BTreeMap::new(),
}
}
pub(crate) fn new(config: VeilidConfig) -> Self {
Self {
config,
inner: Arc::new(Mutex::new(Self::new_inner())),
async_lock: Arc::new(AsyncMutex::new(())),
}
}
/// Delete all known tables
pub async fn delete_all(&self) {
if let Err(e) = self.delete("crypto_caches").await {
error!("failed to delete 'crypto_caches': {}", e);
}
if let Err(e) = self.delete("RouteSpecStore").await {
error!("failed to delete 'RouteSpecStore': {}", e);
}
if let Err(e) = self.delete("routing_table").await {
error!("failed to delete 'routing_table': {}", e);
}
}
pub(crate) async fn init(&self) -> EyreResult<()> {
let _async_guard = self.async_lock.lock().await;
Ok(())
}
pub(crate) async fn terminate(&self) {
let _async_guard = self.async_lock.lock().await;
assert!(
self.inner.lock().opened.len() == 0,
"all open databases should have been closed"
);
}
pub(crate) fn on_table_db_drop(&self, table: String) {
let mut inner = self.inner.lock();
match inner.opened.remove(&table) {
Some(_) => (),
None => {
assert!(false, "should have removed an item");
}
}
}
fn get_table_name(&self, table: &str) -> EyreResult<String> {
if !table
.chars()
.all(|c| char::is_alphanumeric(c) || c == '_' || c == '-')
{
bail!("table name '{}' is invalid", table);
}
let c = self.config.get();
let namespace = c.namespace.clone();
Ok(if namespace.len() == 0 {
format!("{}", table)
} else {
format!("_ns_{}_{}", namespace, table)
})
}
/// Get or create a TableDB database table. If the column count is greater than an
/// existing TableDB's column count, the database will be upgraded to add the missing columns
pub async fn open(&self, name: &str, column_count: u32) -> EyreResult<TableDB> {
let _async_guard = self.async_lock.lock().await;
let table_name = self.get_table_name(name)?;
{
let mut inner = self.inner.lock();
if let Some(table_db_weak_inner) = inner.opened.get(&table_name) {
match TableDB::try_new_from_weak_inner(table_db_weak_inner.clone()) {
Some(tdb) => {
return Ok(tdb);
}
None => {
inner.opened.remove(&table_name);
}
};
}
}
let db = Database::open(table_name.clone(), column_count)
.await
.wrap_err("failed to open tabledb")?;
trace!(
"opened table store '{}' with table name '{:?}' with {} columns",
name,
table_name,
column_count
);
let table_db = TableDB::new(table_name.clone(), self.clone(), db);
{
let mut inner = self.inner.lock();
inner.opened.insert(table_name, table_db.weak_inner());
}
Ok(table_db)
}
/// Delete a TableDB table by name
pub async fn delete(&self, name: &str) -> EyreResult<bool> {
let _async_guard = self.async_lock.lock().await;
trace!("TableStore::delete {}", name);
let table_name = self.get_table_name(name)?;
{
let inner = self.inner.lock();
if inner.opened.contains_key(&table_name) {
trace!(
"TableStore::delete {}: Not deleting, still open.",
table_name
);
bail!("Not deleting table that is still opened");
}
}
if is_browser() {
let out = match Database::delete(table_name.clone()).await {
Ok(_) => true,
Err(_) => false,
};
//.map_err(|e| format!("failed to delete tabledb at: {} ({})", table_name, e))?;
trace!("TableStore::deleted {}", table_name);
Ok(out)
} else {
unimplemented!();
}
}
}