From befb100ba408696b4abdb7accb87185eee9de851 Mon Sep 17 00:00:00 2001 From: Christien Rioux Date: Sat, 15 Jul 2023 21:44:36 -0400 Subject: [PATCH] compress envelopes with lz4 --- Cargo.lock | 7 +++++ veilid-core/Cargo.toml | 1 + veilid-core/src/crypto/envelope.rs | 23 ++++++++++++-- veilid-core/src/lib.rs | 1 + veilid-core/src/table_store/table_db.rs | 40 +++++++++++++------------ 5 files changed, 51 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88f2f1ae..4d29034d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3167,6 +3167,12 @@ dependencies = [ "linked-hash-map", ] +[[package]] +name = "lz4_flex" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8" + [[package]] name = "malloc_buf" version = "0.0.6" @@ -6414,6 +6420,7 @@ dependencies = [ "keyvaluedb-web", "lazy_static", "libc", + "lz4_flex", "ndk", "ndk-glue", "netlink-packet-route 0.15.0", diff --git a/veilid-core/Cargo.toml b/veilid-core/Cargo.toml index 1b7848c0..07799ab4 100644 --- a/veilid-core/Cargo.toml +++ b/veilid-core/Cargo.toml @@ -92,6 +92,7 @@ serde-big-array = "^0" json = "^0" data-encoding = { version = "^2" } schemars = "0.8.12" +lz4_flex = { version = "0.11.1", default-features = false, features = ["safe-encode", "safe-decode"] } # Dependencies for native builds only # Linux, Windows, Mac, iOS, Android diff --git a/veilid-core/src/crypto/envelope.rs b/veilid-core/src/crypto/envelope.rs index 21c940f4..f7116e59 100644 --- a/veilid-core/src/crypto/envelope.rs +++ b/veilid-core/src/crypto/envelope.rs @@ -213,6 +213,10 @@ impl Envelope { &dh_secret, ); + // Decompress body + let body = decompress_size_prepended(&body) + .map_err(|e| VeilidAPIError::parse_error("failed to decompress", e))?; + Ok(body) } @@ -223,10 +227,25 @@ impl Envelope { node_id_secret: &SecretKey, network_key: &Option, ) -> VeilidAPIResult> { + // Ensure body isn't too long + let uncompressed_body_size: usize = body.len() + MIN_ENVELOPE_SIZE; + if uncompressed_body_size > MAX_ENVELOPE_SIZE { + apibail_parse_error!( + "envelope size before compression is too large", + uncompressed_body_size + ); + } + + // Compress body + let body = compress_prepend_size(&body); + // Ensure body isn't too long let envelope_size: usize = body.len() + MIN_ENVELOPE_SIZE; if envelope_size > MAX_ENVELOPE_SIZE { - apibail_parse_error!("envelope size is too large", envelope_size); + apibail_parse_error!( + "envelope size after compression is too large", + envelope_size + ); } // Generate dh secret let vcrypto = crypto @@ -271,7 +290,7 @@ impl Envelope { } // Encrypt and authenticate message - let encrypted_body = vcrypto.crypt_no_auth_unaligned(body, &self.nonce.bytes, &dh_secret); + let encrypted_body = vcrypto.crypt_no_auth_unaligned(&body, &self.nonce.bytes, &dh_secret); // Write body if !encrypted_body.is_empty() { diff --git a/veilid-core/src/lib.rs b/veilid-core/src/lib.rs index 2446cfe4..0c973e7d 100644 --- a/veilid-core/src/lib.rs +++ b/veilid-core/src/lib.rs @@ -89,6 +89,7 @@ use cfg_if::*; use enumset::*; use eyre::{bail, eyre, Report as EyreReport, Result as EyreResult, WrapErr}; use futures_util::stream::FuturesUnordered; +use lz4_flex::block::{compress_prepend_size, decompress_size_prepended}; use parking_lot::*; use schemars::{schema_for, JsonSchema}; use serde::*; diff --git a/veilid-core/src/table_store/table_db.rs b/veilid-core/src/table_store/table_db.rs index 41c5129a..a95d3298 100644 --- a/veilid-core/src/table_store/table_db.rs +++ b/veilid-core/src/table_store/table_db.rs @@ -102,13 +102,14 @@ impl TableDB { /// but if the contents are guaranteed to be unique, then a nonce /// can be generated from the hash of the contents and the encryption key itself fn maybe_encrypt(&self, data: &[u8], keyed_nonce: bool) -> Vec { + let data = compress_prepend_size(data); if let Some(ei) = &self.unlocked_inner.encrypt_info { let mut out = unsafe { unaligned_u8_vec_uninit(NONCE_LENGTH + data.len()) }; if keyed_nonce { // Key content nonce let mut noncedata = Vec::with_capacity(data.len() + PUBLIC_KEY_LENGTH); - noncedata.extend_from_slice(data); + noncedata.extend_from_slice(&data); noncedata.extend_from_slice(&ei.key.bytes); let noncehash = ei.vcrypto.generate_hash(&noncedata); out[0..NONCE_LENGTH].copy_from_slice(&noncehash[0..NONCE_LENGTH]) @@ -119,23 +120,23 @@ impl TableDB { let (nonce, encout) = out.split_at_mut(NONCE_LENGTH); ei.vcrypto.crypt_b2b_no_auth( - data, + &data, encout, (nonce as &[u8]).try_into().unwrap(), &ei.key, ); out } else { - data.to_vec() + data } } /// Decrypt buffer using decrypt key with nonce prepended to input - fn maybe_decrypt(&self, data: &[u8]) -> Vec { + fn maybe_decrypt(&self, data: &[u8]) -> std::io::Result> { if let Some(di) = &self.unlocked_inner.decrypt_info { assert!(data.len() >= NONCE_LENGTH); if data.len() == NONCE_LENGTH { - return Vec::new(); + return Ok(Vec::new()); } let mut out = unsafe { unaligned_u8_vec_uninit(data.len() - NONCE_LENGTH) }; @@ -146,9 +147,11 @@ impl TableDB { (&data[0..NONCE_LENGTH]).try_into().unwrap(), &di.key, ); - out + decompress_size_prepended(&out) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())) } else { - data.to_vec() + decompress_size_prepended(data) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string())) } } @@ -163,7 +166,8 @@ impl TableDB { let db = self.unlocked_inner.database.clone(); let mut out = Vec::new(); db.iter_keys(col, None, |k| { - out.push(self.maybe_decrypt(k)); + let key = self.maybe_decrypt(k)?; + out.push(key); Ok(Option::<()>::None) }) .await @@ -214,11 +218,10 @@ impl TableDB { } let db = self.unlocked_inner.database.clone(); let key = self.maybe_encrypt(key, true); - Ok(db - .get(col, &key) - .await - .map_err(VeilidAPIError::from)? - .map(|v| self.maybe_decrypt(&v))) + match db.get(col, &key).await.map_err(VeilidAPIError::from)? { + Some(v) => Ok(Some(self.maybe_decrypt(&v).map_err(VeilidAPIError::from)?)), + None => Ok(None), + } } /// Read an serde-json key from a column in the TableDB immediately @@ -244,12 +247,11 @@ impl TableDB { let key = self.maybe_encrypt(key, true); let db = self.unlocked_inner.database.clone(); - let old_value = db - .delete(col, &key) - .await - .map_err(VeilidAPIError::from)? - .map(|v| self.maybe_decrypt(&v)); - Ok(old_value) + + match db.delete(col, &key).await.map_err(VeilidAPIError::from)? { + Some(v) => Ok(Some(self.maybe_decrypt(&v).map_err(VeilidAPIError::from)?)), + None => Ok(None), + } } /// Delete serde-json key with from a column in the TableDB