compress envelopes with lz4

Cargo.lock

@@ -3167,6 +3167,12 @@ dependencies = [
  "linked-hash-map",
 ]
 
+[[package]]
+name = "lz4_flex"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ea9b256699eda7b0387ffbc776dd625e28bde3918446381781245b7a50349d8"
+
 [[package]]
 name = "malloc_buf"
 version = "0.0.6"
@@ -6414,6 +6420,7 @@ dependencies = [
  "keyvaluedb-web",
  "lazy_static",
  "libc",
+ "lz4_flex",
  "ndk",
  "ndk-glue",
  "netlink-packet-route 0.15.0",

veilid-core/Cargo.toml

@@ -92,6 +92,7 @@ serde-big-array = "^0"
 json = "^0"
 data-encoding = { version = "^2" }
 schemars = "0.8.12"
+lz4_flex = { version = "0.11.1", default-features = false, features = ["safe-encode", "safe-decode"] }
 
 # Dependencies for native builds only
 # Linux, Windows, Mac, iOS, Android

veilid-core/src/crypto/envelope.rs

@@ -213,6 +213,10 @@ impl Envelope {
             &dh_secret,
         );
 
+        // Decompress body
+        let body = decompress_size_prepended(&body)
+            .map_err(|e| VeilidAPIError::parse_error("failed to decompress", e))?;
+
         Ok(body)
     }
 
@@ -223,10 +227,25 @@ impl Envelope {
         node_id_secret: &SecretKey,
         network_key: &Option<SharedSecret>,
     ) -> VeilidAPIResult<Vec<u8>> {
+        // Ensure body isn't too long
+        let uncompressed_body_size: usize = body.len() + MIN_ENVELOPE_SIZE;
+        if uncompressed_body_size > MAX_ENVELOPE_SIZE {
+            apibail_parse_error!(
+                "envelope size before compression is too large",
+                uncompressed_body_size
+            );
+        }
+
+        // Compress body
+        let body = compress_prepend_size(&body);
+
         // Ensure body isn't too long
         let envelope_size: usize = body.len() + MIN_ENVELOPE_SIZE;
         if envelope_size > MAX_ENVELOPE_SIZE {
-            apibail_parse_error!("envelope size is too large", envelope_size);
+            apibail_parse_error!(
+                "envelope size after compression is too large",
+                envelope_size
+            );
         }
         // Generate dh secret
         let vcrypto = crypto
@@ -271,7 +290,7 @@ impl Envelope {
         }
 
         // Encrypt and authenticate message
-        let encrypted_body = vcrypto.crypt_no_auth_unaligned(body, &self.nonce.bytes, &dh_secret);
+        let encrypted_body = vcrypto.crypt_no_auth_unaligned(&body, &self.nonce.bytes, &dh_secret);
 
         // Write body
         if !encrypted_body.is_empty() {

veilid-core/src/lib.rs

@@ -89,6 +89,7 @@ use cfg_if::*;
 use enumset::*;
 use eyre::{bail, eyre, Report as EyreReport, Result as EyreResult, WrapErr};
 use futures_util::stream::FuturesUnordered;
+use lz4_flex::block::{compress_prepend_size, decompress_size_prepended};
 use parking_lot::*;
 use schemars::{schema_for, JsonSchema};
 use serde::*;

veilid-core/src/table_store/table_db.rs

@@ -102,13 +102,14 @@ impl TableDB {
     /// but if the contents are guaranteed to be unique, then a nonce
     /// can be generated from the hash of the contents and the encryption key itself
     fn maybe_encrypt(&self, data: &[u8], keyed_nonce: bool) -> Vec<u8> {
+        let data = compress_prepend_size(data);
         if let Some(ei) = &self.unlocked_inner.encrypt_info {
             let mut out = unsafe { unaligned_u8_vec_uninit(NONCE_LENGTH + data.len()) };
 
             if keyed_nonce {
                 // Key content nonce
                 let mut noncedata = Vec::with_capacity(data.len() + PUBLIC_KEY_LENGTH);
-                noncedata.extend_from_slice(data);
+                noncedata.extend_from_slice(&data);
                 noncedata.extend_from_slice(&ei.key.bytes);
                 let noncehash = ei.vcrypto.generate_hash(&noncedata);
                 out[0..NONCE_LENGTH].copy_from_slice(&noncehash[0..NONCE_LENGTH])
@@ -119,23 +120,23 @@ impl TableDB {
 
             let (nonce, encout) = out.split_at_mut(NONCE_LENGTH);
             ei.vcrypto.crypt_b2b_no_auth(
-                data,
+                &data,
                 encout,
                 (nonce as &[u8]).try_into().unwrap(),
                 &ei.key,
             );
             out
         } else {
-            data.to_vec()
+            data
         }
     }
 
     /// Decrypt buffer using decrypt key with nonce prepended to input
-    fn maybe_decrypt(&self, data: &[u8]) -> Vec<u8> {
+    fn maybe_decrypt(&self, data: &[u8]) -> std::io::Result<Vec<u8>> {
         if let Some(di) = &self.unlocked_inner.decrypt_info {
             assert!(data.len() >= NONCE_LENGTH);
             if data.len() == NONCE_LENGTH {
-                return Vec::new();
+                return Ok(Vec::new());
             }
 
             let mut out = unsafe { unaligned_u8_vec_uninit(data.len() - NONCE_LENGTH) };
@@ -146,9 +147,11 @@ impl TableDB {
                 (&data[0..NONCE_LENGTH]).try_into().unwrap(),
                 &di.key,
             );
-            out
+            decompress_size_prepended(&out)
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))
         } else {
-            data.to_vec()
+            decompress_size_prepended(data)
+                .map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e.to_string()))
         }
     }
 
@@ -163,7 +166,8 @@ impl TableDB {
         let db = self.unlocked_inner.database.clone();
         let mut out = Vec::new();
         db.iter_keys(col, None, |k| {
-            out.push(self.maybe_decrypt(k));
+            let key = self.maybe_decrypt(k)?;
+            out.push(key);
             Ok(Option::<()>::None)
         })
         .await
@@ -214,11 +218,10 @@ impl TableDB {
         }
         let db = self.unlocked_inner.database.clone();
         let key = self.maybe_encrypt(key, true);
-        Ok(db
+        match db.get(col, &key).await.map_err(VeilidAPIError::from)? {
-            .get(col, &key)
+            Some(v) => Ok(Some(self.maybe_decrypt(&v).map_err(VeilidAPIError::from)?)),
-            .await
+            None => Ok(None),
-            .map_err(VeilidAPIError::from)?
+        }
-            .map(|v| self.maybe_decrypt(&v)))
     }
 
     /// Read an serde-json key from a column in the TableDB immediately
@@ -244,12 +247,11 @@ impl TableDB {
         let key = self.maybe_encrypt(key, true);
 
         let db = self.unlocked_inner.database.clone();
-        let old_value = db
+
-            .delete(col, &key)
+        match db.delete(col, &key).await.map_err(VeilidAPIError::from)? {
-            .await
+            Some(v) => Ok(Some(self.maybe_decrypt(&v).map_err(VeilidAPIError::from)?)),
-            .map_err(VeilidAPIError::from)?
+            None => Ok(None),
-            .map(|v| self.maybe_decrypt(&v));
+        }
-        Ok(old_value)
     }
 
     /// Delete serde-json key with from a column in the TableDB