better error handling

veilid-python/pyproject.toml

@@ -4,7 +4,7 @@ version = "0.1.0"
 description = ""
 authors = ["Christien Rioux <chris@veilid.org>"]
 readme = "README.md"
-packages = [{include = "veilid_python"}]
+packages = [{include = "veilid"}]
 
 [tool.poetry.dependencies]
 python = "^3.11"

veilid-python/tests/__init__.py

@@ -1,7 +1,10 @@
+from typing import Callable, Awaitable
+import os
 import pytest
 pytest_plugins = ('pytest_asyncio',)
 
-import os
+import veilid
+
 
 ##################################################################
 VEILID_SERVER = os.getenv("VEILID_SERVER")
@@ -18,5 +21,10 @@ else:
 
 ##################################################################
 
-async def simple_update_callback(update):
+async def simple_connect_and_run(func: Callable[[veilid.VeilidAPI], Awaitable]):
+    api = await veilid.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback)
+    async with api:
+        await func(api)
+
+async def simple_update_callback(update: veilid.VeilidUpdate):
     print("VeilidUpdate: {}".format(update))

veilid-python/tests/test_basic.py

@@ -1,6 +1,6 @@
-# Basic veilid_python tests
+# Basic veilid tests
 
-import veilid_python
+import veilid
 import pytest
 from . import *
 
@@ -8,19 +8,22 @@ from . import *
 
 @pytest.mark.asyncio
 async def test_connect():
-    async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api:
+    async def func(api: veilid.VeilidAPI):
         pass
+    await simple_connect_and_run(func)
 
 @pytest.mark.asyncio
 async def test_fail_connect():
     with pytest.raises(Exception):
-        async with await veilid_python.json_api_connect("fuahwelifuh32luhwafluehawea", 1, simple_update_callback) as api:
+        api = await veilid.json_api_connect("fuahwelifuh32luhwafluehawea", 1, simple_update_callback)
+        async with api:
             pass
 
 @pytest.mark.asyncio
 async def test_version():
-    async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api:
+    async def func(api: veilid.VeilidAPI):
         v = await api.veilid_version()
         print("veilid_version: {}".format(v.__dict__))
         vstr = await api.veilid_version_string()
         print("veilid_version_string: {}".format(vstr))
+    await simple_connect_and_run(func)

veilid-python/tests/test_crypto.py

@@ -1,6 +1,6 @@
-# Crypto veilid_python tests
+# Crypto veilid tests
 
-import veilid_python
+import veilid
 import pytest
 from . import *
 
@@ -8,21 +8,35 @@ from . import *
 
 @pytest.mark.asyncio
 async def test_best_crypto_system():
-    async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api:
+    async def func(api: veilid.VeilidAPI):
         bcs = await api.best_crypto_system()
-        # let handle dangle for test
-        # del bcs
+    await simple_connect_and_run(func)
         
 @pytest.mark.asyncio
 async def test_get_crypto_system():
-    async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api:
-        cs = await api.get_crypto_system(veilid_python.CryptoKind.CRYPTO_KIND_VLD0)
+    async def func(api: veilid.VeilidAPI):
+        cs = await api.get_crypto_system(veilid.CryptoKind.CRYPTO_KIND_VLD0)
         # clean up handle early
         del cs
+    await simple_connect_and_run(func)
         
 @pytest.mark.asyncio
 async def test_get_crypto_system_invalid():
-    async with await veilid_python.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, simple_update_callback) as api:
-        with pytest.raises(veilid_python.VeilidAPIError):
-            cs = await api.get_crypto_system(veilid_python.CryptoKind.CRYPTO_KIND_NONE)
+    async def func(api: veilid.VeilidAPI):
+        with pytest.raises(veilid.VeilidAPIError):
+            cs = await api.get_crypto_system(veilid.CryptoKind.CRYPTO_KIND_NONE)
+    await simple_connect_and_run(func)
 
+@pytest.mark.asyncio
+async def test_hash_and_verify_password():
+    async def func(api: veilid.VeilidAPI):
+        bcs = await api.best_crypto_system()
+        nonce = await bcs.random_nonce()
+        salt = nonce.to_bytes()
+        # Password match
+        phash = await bcs.hash_password(b"abc123", salt)
+        assert await bcs.verify_password(b"abc123", phash)
+        # Password mismatch
+        phash2 = await bcs.hash_password(b"abc1234", salt)
+        assert not await bcs.verify_password(b"abc12345", phash)
+    await simple_connect_and_run(func)

veilid-python/tests/test_routing_context.py

@@ -0,0 +1,47 @@
+# Routing context veilid tests
+
+import veilid
+import pytest
+import asyncio
+import json
+from . import *
+
+##################################################################
+
+@pytest.mark.asyncio
+async def test_routing_contexts():
+    async def func(api: veilid.VeilidAPI):
+        rc = await api.new_routing_context()
+        rcp = await rc.with_privacy()
+        rcps = await rcp.with_sequencing(veilid.Sequencing.ENSURE_ORDERED)
+        rcpsr = await rcps.with_custom_privacy(veilid.Stability.RELIABLE)
+    await simple_connect_and_run(func)
+
+@pytest.mark.asyncio
+async def test_routing_context_app_message_loopback():
+
+    app_message_queue = asyncio.Queue()
+
+    async def app_message_queue_update_callback(update: veilid.VeilidUpdate):
+        if update.kind == veilid.VeilidUpdateKind.APP_MESSAGE:
+            await app_message_queue.put(update)
+
+    api = await veilid.json_api_connect(VEILID_SERVER, VEILID_SERVER_PORT, app_message_queue_update_callback)
+    async with api:
+    
+        # make a routing context that uses a safety route
+        rc = await (await api.new_routing_context()).with_privacy()
+
+        # get our own node id
+        state = await api.get_state()
+        node_id = state.config.config.network.routing_table.node_id.pop()
+
+        # send an app message to our node id
+        message = b"abcd1234"
+        await rc.app_message(node_id, message)
+
+        # we should get the same message back
+        #update: veilid.VeilidUpdate = await asyncio.wait_for(app_message_queue.get(), timeout=10)
+        #appmsg: veilid.VeilidAppMessage = update.detail
+        #assert appmsg.message == message
+

veilid-python/update_schema.sh

@@ -11,7 +11,7 @@ if [ ! -f "$VEILID_SERVER" ]; then
 fi
 
 # Produce schema from veilid-server
-$VEILID_SERVER --emit-schema Request > $SCRIPTDIR/veilid_python/schema/Request.json
-$VEILID_SERVER --emit-schema RecvMessage > $SCRIPTDIR/veilid_python/schema/RecvMessage.json
+$VEILID_SERVER --emit-schema Request > $SCRIPTDIR/veilid/schema/Request.json
+$VEILID_SERVER --emit-schema RecvMessage > $SCRIPTDIR/veilid/schema/RecvMessage.json
 
 

veilid-python/veilid/config.py

@@ -2,6 +2,8 @@ from typing import Self, Optional
 from enum import StrEnum
 from json import dumps
 
+from .types import *
+
 class VeilidConfigLogLevel(StrEnum):
     OFF = 'Off'
     ERROR = 'Error'
@@ -96,8 +98,8 @@ class VeilidConfigBlockStore:
         return self.__dict__
 
 class VeilidConfigRoutingTable:
-    node_id: list[str]
-    node_id_secret: list[str]
+    node_id: list[TypedKey]
+    node_id_secret: list[TypedSecret]
     bootstrap: list[str]
     limit_over_attached: int
     limit_fully_attached: int
@@ -105,7 +107,7 @@ class VeilidConfigRoutingTable:
     limit_attached_good: int
     limit_attached_weak: int
 
-    def __init__(self, node_id: list[str], node_id_secret: list[str], bootstrap: list[str], limit_over_attached: int,
+    def __init__(self, node_id: list[TypedKey], node_id_secret: list[TypedSecret], bootstrap: list[str], limit_over_attached: int,
         limit_fully_attached: int, limit_attached_strong: int, limit_attached_good: int, limit_attached_weak: int):
 
         self.node_id = node_id
@@ -120,8 +122,8 @@ class VeilidConfigRoutingTable:
     @staticmethod
     def from_json(j: dict) -> Self:
         return VeilidConfigRoutingTable(
-            j['node_id'],
-            j['node_id_secret'],
+            list(map(lambda x: TypedKey(x), j['node_id'])), 
+            list(map(lambda x: TypedSecret(x), j['node_id_secret'])),
             j['bootstrap'],
             j['limit_over_attached'],
             j['limit_fully_attached'],

veilid-python/veilid/json_api.py

@@ -2,7 +2,7 @@ import json
 import asyncio
 from jsonschema import validators, exceptions
 
-from typing import Callable, Awaitable
+from typing import Callable, Awaitable, Mapping
 
 from .api import *
 from .state import *
@@ -42,7 +42,7 @@ class _JsonVeilidAPI(VeilidAPI):
     # Shared Mutable State
     lock: asyncio.Lock
     next_id: int
-    in_flight_requests: dict
+    in_flight_requests: Mapping[str, asyncio.Future]
     
     def __init__(self, reader: asyncio.StreamReader, writer: asyncio.StreamWriter, update_callback: Callable[[VeilidUpdate], Awaitable], validate_schema: bool = True):
         self.reader = reader
@@ -54,7 +54,7 @@ class _JsonVeilidAPI(VeilidAPI):
         self.next_id = 1
         self.in_flight_requests = dict()
 
-    async def __aenter__(self):
+    async def __aenter__(self) -> Self:
         return self
 
     async def __aexit__(self, *excinfo):
@@ -67,6 +67,10 @@ class _JsonVeilidAPI(VeilidAPI):
             self.writer.close()
             await self.writer.wait_closed()
             self.writer = None
+
+            for (reqid, reqfuture) in self.in_flight_requests.items():
+                reqfuture.cancel()
+
         finally:
             self.lock.release()
 
@@ -103,7 +107,9 @@ class _JsonVeilidAPI(VeilidAPI):
         finally:
             self.lock.release()
         # Resolve the request's future to the response json
-        reqfuture.set_result(j)
+        if reqfuture is not None:
+            reqfuture.set_result(j)
+
 
     async def handle_recv_messages(self):
         # Read lines until we're done
@@ -124,8 +130,6 @@ class _JsonVeilidAPI(VeilidAPI):
                     await self.handle_recv_message_response(j)
                 elif j['type'] == "Update":
                     await self.update_callback(VeilidUpdate.from_json(j))
-        except:
-            pass
         finally:
             await self._cleanup_close()
             
@@ -263,17 +267,17 @@ class _JsonVeilidAPI(VeilidAPI):
         cs_id = raise_api_result(await self.send_ndjson_request(Operation.BEST_CRYPTO_SYSTEM))
         return _JsonCryptoSystem(self, cs_id)
     async def verify_signatures(self, node_ids: list[TypedKey], data: bytes, signatures: list[TypedSignature]) -> list[TypedKey]:
-        return map(lambda x: TypedKey(x), raise_api_result(await self.send_ndjson_request(Operation.VERIFY_SIGNATURES,
+        return list(map(lambda x: TypedKey(x), raise_api_result(await self.send_ndjson_request(Operation.VERIFY_SIGNATURES,
             node_ids = node_ids,
             data = data,
-            signatures = signatures)))
+            signatures = signatures))))
     async def generate_signatures(self, data: bytes, key_pairs: list[TypedKeyPair]) -> list[TypedSignature]:
-        return map(lambda x: TypedSignature(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_SIGNATURES,
+        return list(map(lambda x: TypedSignature(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_SIGNATURES,
             data = data,
-            key_pairs = key_pairs)))
+            key_pairs = key_pairs))))
     async def generate_key_pair(self, kind: CryptoKind) -> list[TypedKeyPair]:
-        return map(lambda x: TypedKeyPair(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_KEY_PAIR,
-            kind = kind)))
+        return list(map(lambda x: TypedKeyPair(x), raise_api_result(await self.send_ndjson_request(Operation.GENERATE_KEY_PAIR,
+            kind = kind))))
     async def now(self) -> Timestamp:
         return Timestamp(raise_api_result(await self.send_ndjson_request(Operation.NOW)))
     async def debug(self, command: str) -> str:
@@ -456,10 +460,10 @@ class _JsonTableDb(TableDb):
             db_id = self.db_id, 
             db_op = TableDbOperation.GET_COLUMN_COUNT))
     async def get_keys(self, col: int) -> list[bytes]:
-        return map(lambda x: urlsafe_b64decode_no_pad(x), raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op,
+        return list(map(lambda x: urlsafe_b64decode_no_pad(x), raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op,
             db_id = self.db_id, 
             db_op = TableDbOperation.GET_KEYS,
-            col = col)))
+            col = col))))
     async def transact(self) -> TableDbTransaction:
         tx_id = raise_api_result(await self.api.send_ndjson_request(Operation.TABLE_DB, validate=validate_db_op,
             db_id = self.db_id, 
@@ -627,6 +631,6 @@ class _JsonCryptoSystem(CryptoSystem):
 
 ######################################################
 
-def json_api_connect(host:str, port:int, update_callback: Callable[[VeilidUpdate], Awaitable]) -> VeilidAPI:
-    return _JsonVeilidAPI.connect(host, port, update_callback)
+async def json_api_connect(host:str, port:int, update_callback: Callable[[VeilidUpdate], Awaitable]) -> VeilidAPI:
+    return await _JsonVeilidAPI.connect(host, port, update_callback)
 

veilid-python/veilid/state.py

@@ -181,7 +181,7 @@ class VeilidStateNetwork:
             j['started'],
             ByteCount(j['bps_down']),
             ByteCount(j['bps_up']),
-            map(lambda x: PeerTableData.from_json(x), j['peers']))
+            list(map(lambda x: PeerTableData.from_json(x), j['peers'])))
 
 class VeilidStateConfig:
     config: VeilidConfig
@@ -193,7 +193,8 @@ class VeilidStateConfig:
     def from_json(j: dict) -> Self:
         '''JSON object hook'''
         return VeilidStateConfig(
-            j['config'])
+            VeilidConfig.from_json(j['config'])
+        )
 
 class VeilidState:
     attachment: VeilidStateAttachment
@@ -227,7 +228,7 @@ class VeilidLog:
     def from_json(j: dict) -> Self:
         '''JSON object hook'''
         return VeilidLog(
-            VeilidLogLevel(j['attachment']), 
+            VeilidLogLevel(j['log_level']), 
             j['message'], 
             j['backtrace'])
     
@@ -276,8 +277,8 @@ class VeilidRouteChange:
     def from_json(j: dict) -> Self:
         '''JSON object hook'''
         return VeilidRouteChange(
-            map(lambda x: RouteId(x), j['dead_routes']),
-            map(lambda x: RouteId(x), j['dead_remote_routes']))
+            list(map(lambda x: RouteId(x), j['dead_routes'])),
+            list(map(lambda x: RouteId(x), j['dead_remote_routes'])))
 
 class VeilidValueChange:
     key: TypedKey
@@ -296,7 +297,7 @@ class VeilidValueChange:
         '''JSON object hook'''
         return VeilidValueChange(
             TypedKey(j['key']),
-            map(lambda x: ValueSubkey(x), j['subkeys']),
+            list(map(lambda x: ValueSubkey(x), j['subkeys'])),
             j['count'],
             ValueData.from_json(j['value']))
 
@@ -346,4 +347,4 @@ class VeilidUpdate:
                 detail = None
             case _:
                 raise ValueError("Unknown VeilidUpdateKind")
-                
+        return VeilidUpdate(kind, detail)

veilid-python/veilid/types.py

@@ -11,8 +11,7 @@ def urlsafe_b64encode_no_pad(b: bytes) -> str:
     """
     Removes any `=` used as padding from the encoded string.
     """
-    encoded = str(base64.urlsafe_b64encode(b))
-    return encoded.rstrip("=")
+    return base64.urlsafe_b64encode(b).decode().rstrip("=")
 
 
 def urlsafe_b64decode_no_pad(s: str) -> bytes:
@@ -20,7 +19,7 @@ def urlsafe_b64decode_no_pad(s: str) -> bytes:
     Adds back in the required padding before decoding.
     """
     padding = 4 - (len(s) % 4)
-    string = string + ("=" * padding)
+    s = s + ("=" * padding)
     return base64.urlsafe_b64decode(s)
 
 class VeilidJSONEncoder(json.JSONEncoder):
@@ -248,7 +247,7 @@ class DHTSchema:
         if DHTSchemaKind(j['kind']) == DHTSchemaKind.SMPL:
             return DHTSchema.smpl(
                 j['o_cnt'],
-                map(lambda x: DHTSchemaSMPLMember.from_json(x), j['members']))
+                list(map(lambda x: DHTSchemaSMPLMember.from_json(x), j['members'])))
         raise Exception("Unknown DHTSchema kind", j['kind'])
 
     def to_json(self) -> dict: