2021-11-22 16:28:30 +00:00
|
|
|
use crate::*;
|
2023-05-29 19:24:57 +00:00
|
|
|
use crypto::Crypto;
|
|
|
|
use network_manager::*;
|
|
|
|
use routing_table::*;
|
|
|
|
use storage_manager::*;
|
2021-11-22 16:28:30 +00:00
|
|
|
|
|
|
|
pub struct AttachmentManagerInner {
|
2022-12-26 21:33:48 +00:00
|
|
|
last_attachment_state: AttachmentState,
|
|
|
|
last_routing_table_health: Option<RoutingTableHealth>,
|
2021-11-22 16:28:30 +00:00
|
|
|
maintain_peers: bool,
|
2022-12-17 01:07:28 +00:00
|
|
|
attach_ts: Option<Timestamp>,
|
2022-03-24 14:14:50 +00:00
|
|
|
update_callback: Option<UpdateCallback>,
|
2022-06-13 00:58:02 +00:00
|
|
|
attachment_maintainer_jh: Option<MustJoinHandle<()>>,
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2022-10-10 02:07:15 +00:00
|
|
|
pub struct AttachmentManagerUnlockedInner {
|
|
|
|
config: VeilidConfig,
|
|
|
|
network_manager: NetworkManager,
|
|
|
|
}
|
|
|
|
|
2021-11-22 16:28:30 +00:00
|
|
|
#[derive(Clone)]
|
|
|
|
pub struct AttachmentManager {
|
|
|
|
inner: Arc<Mutex<AttachmentManagerInner>>,
|
2022-10-10 02:07:15 +00:00
|
|
|
unlocked_inner: Arc<AttachmentManagerUnlockedInner>,
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
impl AttachmentManager {
|
2022-10-10 02:07:15 +00:00
|
|
|
fn new_unlocked_inner(
|
2021-11-22 16:28:30 +00:00
|
|
|
config: VeilidConfig,
|
2023-05-29 19:24:57 +00:00
|
|
|
storage_manager: StorageManager,
|
2022-10-10 02:07:15 +00:00
|
|
|
protected_store: ProtectedStore,
|
2021-11-22 16:28:30 +00:00
|
|
|
table_store: TableStore,
|
2023-06-05 01:22:55 +00:00
|
|
|
#[cfg(feature = "unstable-blockstore")] block_store: BlockStore,
|
2021-11-22 16:28:30 +00:00
|
|
|
crypto: Crypto,
|
2022-10-10 02:07:15 +00:00
|
|
|
) -> AttachmentManagerUnlockedInner {
|
|
|
|
AttachmentManagerUnlockedInner {
|
2021-11-22 16:28:30 +00:00
|
|
|
config: config.clone(),
|
2022-10-10 02:07:15 +00:00
|
|
|
network_manager: NetworkManager::new(
|
|
|
|
config,
|
2023-05-29 19:24:57 +00:00
|
|
|
storage_manager,
|
2022-10-10 02:07:15 +00:00
|
|
|
protected_store,
|
|
|
|
table_store,
|
2023-06-05 01:22:55 +00:00
|
|
|
#[cfg(feature = "unstable-blockstore")]
|
2022-10-10 02:07:15 +00:00
|
|
|
block_store,
|
|
|
|
crypto,
|
|
|
|
),
|
|
|
|
}
|
|
|
|
}
|
|
|
|
fn new_inner() -> AttachmentManagerInner {
|
|
|
|
AttachmentManagerInner {
|
2022-12-26 21:33:48 +00:00
|
|
|
last_attachment_state: AttachmentState::Detached,
|
|
|
|
last_routing_table_health: None,
|
2021-11-22 16:28:30 +00:00
|
|
|
maintain_peers: false,
|
2022-12-17 01:07:28 +00:00
|
|
|
attach_ts: None,
|
2022-03-24 14:14:50 +00:00
|
|
|
update_callback: None,
|
2021-11-22 16:28:30 +00:00
|
|
|
attachment_maintainer_jh: None,
|
|
|
|
}
|
|
|
|
}
|
2022-10-10 02:07:15 +00:00
|
|
|
pub fn new(
|
|
|
|
config: VeilidConfig,
|
2023-05-29 19:24:57 +00:00
|
|
|
storage_manager: StorageManager,
|
2022-10-10 02:07:15 +00:00
|
|
|
protected_store: ProtectedStore,
|
|
|
|
table_store: TableStore,
|
2023-06-05 01:22:55 +00:00
|
|
|
#[cfg(feature = "unstable-blockstore")] block_store: BlockStore,
|
2022-10-10 02:07:15 +00:00
|
|
|
crypto: Crypto,
|
|
|
|
) -> Self {
|
2021-11-22 16:28:30 +00:00
|
|
|
Self {
|
2022-10-10 02:07:15 +00:00
|
|
|
inner: Arc::new(Mutex::new(Self::new_inner())),
|
|
|
|
unlocked_inner: Arc::new(Self::new_unlocked_inner(
|
|
|
|
config,
|
2023-05-29 19:24:57 +00:00
|
|
|
storage_manager,
|
2022-10-10 02:07:15 +00:00
|
|
|
protected_store,
|
|
|
|
table_store,
|
2023-06-05 01:22:55 +00:00
|
|
|
#[cfg(feature = "unstable-blockstore")]
|
2022-10-10 02:07:15 +00:00
|
|
|
block_store,
|
|
|
|
crypto,
|
|
|
|
)),
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
pub fn config(&self) -> VeilidConfig {
|
2022-10-10 02:07:15 +00:00
|
|
|
self.unlocked_inner.config.clone()
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn network_manager(&self) -> NetworkManager {
|
2022-10-10 02:07:15 +00:00
|
|
|
self.unlocked_inner.network_manager.clone()
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
pub fn is_attached(&self) -> bool {
|
2022-12-26 21:33:48 +00:00
|
|
|
let s = self.inner.lock().last_attachment_state;
|
2021-11-27 17:44:21 +00:00
|
|
|
!matches!(s, AttachmentState::Detached | AttachmentState::Detaching)
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
pub fn is_detached(&self) -> bool {
|
2022-12-26 21:33:48 +00:00
|
|
|
let s = self.inner.lock().last_attachment_state;
|
2021-11-27 17:44:21 +00:00
|
|
|
matches!(s, AttachmentState::Detached)
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2022-12-17 01:07:28 +00:00
|
|
|
pub fn get_attach_timestamp(&self) -> Option<Timestamp> {
|
|
|
|
self.inner.lock().attach_ts
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2022-03-24 14:14:50 +00:00
|
|
|
fn translate_routing_table_health(
|
2022-12-26 21:33:48 +00:00
|
|
|
health: &RoutingTableHealth,
|
2022-03-24 14:14:50 +00:00
|
|
|
config: &VeilidConfigRoutingTable,
|
2022-12-26 21:33:48 +00:00
|
|
|
) -> AttachmentState {
|
2022-03-24 14:14:50 +00:00
|
|
|
if health.reliable_entry_count >= config.limit_over_attached.try_into().unwrap() {
|
2022-12-26 21:33:48 +00:00
|
|
|
return AttachmentState::OverAttached;
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
2022-03-24 14:14:50 +00:00
|
|
|
if health.reliable_entry_count >= config.limit_fully_attached.try_into().unwrap() {
|
2022-12-26 21:33:48 +00:00
|
|
|
return AttachmentState::FullyAttached;
|
2022-03-24 14:14:50 +00:00
|
|
|
}
|
|
|
|
if health.reliable_entry_count >= config.limit_attached_strong.try_into().unwrap() {
|
2022-12-26 21:33:48 +00:00
|
|
|
return AttachmentState::AttachedStrong;
|
2022-03-24 14:14:50 +00:00
|
|
|
}
|
|
|
|
if health.reliable_entry_count >= config.limit_attached_good.try_into().unwrap() {
|
2022-12-26 21:33:48 +00:00
|
|
|
return AttachmentState::AttachedGood;
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
2022-03-24 14:14:50 +00:00
|
|
|
if health.reliable_entry_count >= config.limit_attached_weak.try_into().unwrap()
|
|
|
|
|| health.unreliable_entry_count >= config.limit_attached_weak.try_into().unwrap()
|
|
|
|
{
|
2022-12-26 21:33:48 +00:00
|
|
|
return AttachmentState::AttachedWeak;
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
2022-12-26 21:33:48 +00:00
|
|
|
AttachmentState::Attaching
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2022-12-26 21:33:48 +00:00
|
|
|
/// Update attachment and network readiness state
|
|
|
|
/// and possibly send a VeilidUpdate::Attachment
|
|
|
|
fn update_attachment(&self) {
|
|
|
|
// update the routing table health
|
|
|
|
let routing_table = self.network_manager().routing_table();
|
|
|
|
let health = routing_table.get_routing_table_health();
|
|
|
|
let opt_update = {
|
|
|
|
let mut inner = self.inner.lock();
|
|
|
|
|
|
|
|
// Check if the routing table health is different
|
|
|
|
if let Some(last_routing_table_health) = &inner.last_routing_table_health {
|
|
|
|
// If things are the same, just return
|
|
|
|
if last_routing_table_health == &health {
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
}
|
2021-11-22 16:28:30 +00:00
|
|
|
|
2022-12-26 21:33:48 +00:00
|
|
|
// Swap in new health numbers
|
|
|
|
let opt_previous_health = inner.last_routing_table_health.take();
|
|
|
|
inner.last_routing_table_health = Some(health.clone());
|
2021-11-22 16:28:30 +00:00
|
|
|
|
2022-12-26 21:33:48 +00:00
|
|
|
// Calculate new attachment state
|
2022-10-10 02:07:15 +00:00
|
|
|
let config = self.config();
|
|
|
|
let routing_table_config = &config.get().network.routing_table;
|
2022-12-26 21:33:48 +00:00
|
|
|
let previous_attachment_state = inner.last_attachment_state;
|
|
|
|
inner.last_attachment_state =
|
|
|
|
AttachmentManager::translate_routing_table_health(&health, routing_table_config);
|
|
|
|
|
|
|
|
// If we don't have an update callback yet for some reason, just return now
|
|
|
|
let Some(update_callback) = inner.update_callback.clone() else {
|
|
|
|
return;
|
|
|
|
};
|
|
|
|
|
|
|
|
// Send update if one of:
|
|
|
|
// * the attachment state has changed
|
|
|
|
// * routing domain readiness has changed
|
|
|
|
// * this is our first routing table health check
|
|
|
|
let send_update = previous_attachment_state != inner.last_attachment_state
|
|
|
|
|| opt_previous_health
|
|
|
|
.map(|x| {
|
|
|
|
x.public_internet_ready != health.public_internet_ready
|
|
|
|
|| x.local_network_ready != health.local_network_ready
|
|
|
|
})
|
|
|
|
.unwrap_or(true);
|
|
|
|
if send_update {
|
|
|
|
Some((update_callback, Self::get_veilid_state_inner(&*inner)))
|
2021-11-22 16:28:30 +00:00
|
|
|
} else {
|
2022-12-26 21:33:48 +00:00
|
|
|
None
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
};
|
2022-12-26 21:33:48 +00:00
|
|
|
|
|
|
|
// Send the update outside of the lock
|
|
|
|
if let Some(update) = opt_update {
|
|
|
|
(update.0)(VeilidUpdate::Attachment(update.1));
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-06-29 03:46:29 +00:00
|
|
|
fn update_attaching_detaching_state(&self, state: AttachmentState) {
|
|
|
|
let update_callback = {
|
2022-12-26 21:33:48 +00:00
|
|
|
let mut inner = self.inner.lock();
|
2023-06-29 03:46:29 +00:00
|
|
|
inner.last_attachment_state = state;
|
|
|
|
if state == AttachmentState::Attaching {
|
|
|
|
inner.attach_ts = Some(get_aligned_timestamp());
|
|
|
|
} else if state == AttachmentState::Detached {
|
|
|
|
inner.attach_ts = None;
|
|
|
|
} else if state == AttachmentState::Detaching {
|
|
|
|
// ok
|
|
|
|
} else {
|
|
|
|
unreachable!("don't use this for attached states, use update_attachment()");
|
|
|
|
}
|
|
|
|
inner.update_callback.clone()
|
|
|
|
};
|
|
|
|
|
|
|
|
if let Some(update_callback) = update_callback {
|
|
|
|
update_callback(VeilidUpdate::Attachment(VeilidStateAttachment {
|
|
|
|
state,
|
|
|
|
public_internet_ready: false,
|
|
|
|
local_network_ready: false,
|
|
|
|
}))
|
2022-12-26 21:33:48 +00:00
|
|
|
}
|
2023-06-29 03:46:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#[instrument(level = "debug", skip(self))]
|
|
|
|
async fn attachment_maintainer(self) {
|
|
|
|
debug!("attachment starting");
|
|
|
|
self.update_attaching_detaching_state(AttachmentState::Attaching);
|
|
|
|
|
2022-10-10 02:07:15 +00:00
|
|
|
let netman = self.network_manager();
|
2021-11-22 16:28:30 +00:00
|
|
|
|
2022-07-07 03:15:51 +00:00
|
|
|
let mut restart;
|
|
|
|
loop {
|
|
|
|
restart = false;
|
|
|
|
if let Err(err) = netman.startup().await {
|
|
|
|
error!("network startup failed: {}", err);
|
|
|
|
netman.shutdown().await;
|
|
|
|
break;
|
|
|
|
}
|
2021-11-22 16:28:30 +00:00
|
|
|
|
2022-07-07 03:15:51 +00:00
|
|
|
debug!("started maintaining peers");
|
2021-11-22 16:28:30 +00:00
|
|
|
while self.inner.lock().maintain_peers {
|
|
|
|
// tick network manager
|
|
|
|
if let Err(err) = netman.tick().await {
|
|
|
|
error!("Error in network manager: {}", err);
|
|
|
|
self.inner.lock().maintain_peers = false;
|
2022-07-07 03:15:51 +00:00
|
|
|
restart = true;
|
2021-11-22 16:28:30 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2022-07-22 17:05:28 +00:00
|
|
|
// see if we need to restart the network
|
|
|
|
if netman.needs_restart() {
|
|
|
|
info!("Restarting network");
|
|
|
|
restart = true;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
2022-12-26 21:33:48 +00:00
|
|
|
// Update attachment and network readiness state
|
|
|
|
// and possibly send a VeilidUpdate::Attachment
|
|
|
|
self.update_attachment();
|
2021-11-22 16:28:30 +00:00
|
|
|
|
|
|
|
// sleep should be at the end in case maintain_peers changes state
|
2022-11-27 02:37:23 +00:00
|
|
|
sleep(1000).await;
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
2022-07-07 03:15:51 +00:00
|
|
|
debug!("stopped maintaining peers");
|
2021-11-22 16:28:30 +00:00
|
|
|
|
2022-12-26 21:33:48 +00:00
|
|
|
if !restart {
|
2023-06-29 03:46:29 +00:00
|
|
|
self.update_attaching_detaching_state(AttachmentState::Detaching);
|
2022-12-26 21:33:48 +00:00
|
|
|
debug!("attachment stopping");
|
|
|
|
}
|
|
|
|
|
2022-07-07 03:15:51 +00:00
|
|
|
debug!("stopping network");
|
2021-11-22 16:28:30 +00:00
|
|
|
netman.shutdown().await;
|
2022-07-07 03:15:51 +00:00
|
|
|
|
|
|
|
if !restart {
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
debug!("completely restarting attachment");
|
|
|
|
// chill out for a second first, give network stack time to settle out
|
2022-11-27 02:37:23 +00:00
|
|
|
sleep(1000).await;
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2023-06-29 03:46:29 +00:00
|
|
|
self.update_attaching_detaching_state(AttachmentState::Detached);
|
|
|
|
debug!("attachment stopped");
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 21:07:10 +00:00
|
|
|
#[instrument(level = "debug", skip_all, err)]
|
2022-07-10 21:36:50 +00:00
|
|
|
pub async fn init(&self, update_callback: UpdateCallback) -> EyreResult<()> {
|
2022-03-09 03:32:12 +00:00
|
|
|
trace!("init");
|
2022-10-10 02:07:15 +00:00
|
|
|
{
|
2022-03-24 14:14:50 +00:00
|
|
|
let mut inner = self.inner.lock();
|
|
|
|
inner.update_callback = Some(update_callback.clone());
|
2022-12-26 21:33:48 +00:00
|
|
|
}
|
2021-11-22 16:28:30 +00:00
|
|
|
|
2022-10-10 02:07:15 +00:00
|
|
|
self.network_manager().init(update_callback).await?;
|
2021-11-22 16:28:30 +00:00
|
|
|
|
|
|
|
Ok(())
|
|
|
|
}
|
2022-06-10 21:07:10 +00:00
|
|
|
|
|
|
|
#[instrument(level = "debug", skip(self))]
|
2021-11-22 16:28:30 +00:00
|
|
|
pub async fn terminate(&self) {
|
|
|
|
// Ensure we detached
|
|
|
|
self.detach().await;
|
2022-10-10 02:07:15 +00:00
|
|
|
self.network_manager().terminate().await;
|
|
|
|
self.inner.lock().update_callback = None;
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 21:07:10 +00:00
|
|
|
#[instrument(level = "trace", skip(self))]
|
2022-12-26 21:33:48 +00:00
|
|
|
pub async fn attach(&self) -> bool {
|
2021-11-22 16:28:30 +00:00
|
|
|
// Create long-running connection maintenance routine
|
2022-11-02 01:05:48 +00:00
|
|
|
let mut inner = self.inner.lock();
|
2022-10-10 02:07:15 +00:00
|
|
|
if inner.attachment_maintainer_jh.is_some() {
|
2022-12-26 21:33:48 +00:00
|
|
|
return false;
|
2022-10-10 02:07:15 +00:00
|
|
|
}
|
|
|
|
inner.maintain_peers = true;
|
2022-11-27 02:37:23 +00:00
|
|
|
inner.attachment_maintainer_jh = Some(spawn(self.clone().attachment_maintainer()));
|
2022-12-26 21:33:48 +00:00
|
|
|
|
|
|
|
true
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2022-06-10 21:07:10 +00:00
|
|
|
#[instrument(level = "trace", skip(self))]
|
2022-12-26 21:33:48 +00:00
|
|
|
pub async fn detach(&self) -> bool {
|
2022-10-10 02:07:15 +00:00
|
|
|
let attachment_maintainer_jh = {
|
|
|
|
let mut inner = self.inner.lock();
|
|
|
|
let attachment_maintainer_jh = inner.attachment_maintainer_jh.take();
|
|
|
|
if attachment_maintainer_jh.is_some() {
|
|
|
|
// Terminate long-running connection maintenance routine
|
|
|
|
inner.maintain_peers = false;
|
|
|
|
}
|
|
|
|
attachment_maintainer_jh
|
|
|
|
};
|
2021-11-22 16:28:30 +00:00
|
|
|
if let Some(jh) = attachment_maintainer_jh {
|
|
|
|
jh.await;
|
2022-12-26 21:33:48 +00:00
|
|
|
true
|
|
|
|
} else {
|
|
|
|
false
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-12-26 21:33:48 +00:00
|
|
|
pub fn get_attachment_state(&self) -> AttachmentState {
|
|
|
|
self.inner.lock().last_attachment_state
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
|
2022-12-26 21:33:48 +00:00
|
|
|
fn get_veilid_state_inner(inner: &AttachmentManagerInner) -> VeilidStateAttachment {
|
|
|
|
VeilidStateAttachment {
|
|
|
|
state: inner.last_attachment_state,
|
|
|
|
public_internet_ready: inner
|
|
|
|
.last_routing_table_health
|
|
|
|
.as_ref()
|
|
|
|
.map(|x| x.public_internet_ready)
|
|
|
|
.unwrap_or(false),
|
|
|
|
local_network_ready: inner
|
|
|
|
.last_routing_table_health
|
|
|
|
.as_ref()
|
|
|
|
.map(|x| x.local_network_ready)
|
|
|
|
.unwrap_or(false),
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-05-16 15:52:48 +00:00
|
|
|
pub fn get_veilid_state(&self) -> VeilidStateAttachment {
|
2022-12-26 21:33:48 +00:00
|
|
|
let inner = self.inner.lock();
|
|
|
|
Self::get_veilid_state_inner(&*inner)
|
2022-05-16 15:52:48 +00:00
|
|
|
}
|
2021-11-22 16:28:30 +00:00
|
|
|
}
|