Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2025-05-03 15:14:56 -04:00
Code Issues Projects Releases Packages Wiki Activity

add better dht debugging

Browse source
This commit is contained in:
John Smith 2023-06-26 21:29:02 -04:00
parent 62aeec6faf
commit 291e3ef2fe
27 changed files with 817 additions and 95 deletions
Download patch file Download diff file Expand all files Collapse all files

2
veilid-python/tests/conftest.py
View file

@ -22,7 +22,7 @@ def server_info() -> tuple[str, int]:
return hostname, 5959
async def simple_update_callback(api: veilid.VeilidAPI, update: veilid.VeilidUpdate):
async def simple_update_callback(update: veilid.VeilidUpdate):
print(f"VeilidUpdate: {update}")

5
veilid-python/tests/test_dht.py
View file

@ -67,10 +67,15 @@ async def test_set_get_dht_value(api_connection: veilid.VeilidAPI):
vd2 = await rc.get_dht_value(rec.key, 0, False)
assert vd2 != None
vd3 = await rc.get_dht_value(rec.key, 0, True)
assert vd3 != None
print("vd: {}", vd.__dict__)
print("vd2: {}", vd2.__dict__)
print("vd3: {}", vd3.__dict__)
assert vd == vd2
assert vd2 == vd3
await rc.close_dht_record(rec.key)
await rc.delete_dht_record(rec.key)

85
veilid-python/tests/test_routing_context.py
View file

@ -3,6 +3,7 @@
import asyncio
import random
import sys
import os
import pytest
import veilid
@ -30,7 +31,7 @@ async def test_routing_context_app_message_loopback():
# Seriously, mypy?
app_message_queue: asyncio.Queue = asyncio.Queue()
async def app_message_queue_update_callback(api: veilid.VeilidAPI, update: veilid.VeilidUpdate):
async def app_message_queue_update_callback(update: veilid.VeilidUpdate):
if update.kind == veilid.VeilidUpdateKind.APP_MESSAGE:
await app_message_queue.put(update)
@ -69,7 +70,7 @@ async def test_routing_context_app_message_loopback():
async def test_routing_context_app_call_loopback():
app_call_queue: asyncio.Queue = asyncio.Queue()
async def app_call_queue_update_callback(api: veilid.VeilidAPI, update: veilid.VeilidUpdate):
async def app_call_queue_update_callback(update: veilid.VeilidUpdate):
if update.kind == veilid.VeilidUpdateKind.APP_CALL:
await app_call_queue.put(update)
@ -120,7 +121,7 @@ async def test_routing_context_app_message_loopback_big_packets():
global got_message
got_message = 0
async def app_message_queue_update_callback(api: veilid.VeilidAPI, update: veilid.VeilidUpdate):
async def app_message_queue_update_callback(update: veilid.VeilidUpdate):
if update.kind == veilid.VeilidUpdateKind.APP_MESSAGE:
global got_message
got_message += 1
@ -147,8 +148,8 @@ async def test_routing_context_app_message_loopback_big_packets():
# import it as a remote route as well so we can send to it
prr = await api.import_remote_private_route(blob)
# do this test 100 times
for _ in range(100):
# do this test 1000 times
for _ in range(1000):
# send a random sized random app message to our own private route
message = random.randbytes(random.randint(0, 32768))
@ -169,27 +170,39 @@ async def test_routing_context_app_message_loopback_big_packets():
@pytest.mark.asyncio
async def test_routing_context_app_call_loopback_big_packets():
print("")
global got_message
got_message = 0
async def app_message_queue_update_callback(api: veilid.VeilidAPI, update: veilid.VeilidUpdate):
app_call_queue: asyncio.Queue = asyncio.Queue()
async def app_call_queue_update_callback(update: veilid.VeilidUpdate):
if update.kind == veilid.VeilidUpdateKind.APP_CALL:
await app_call_queue.put(update)
async def app_call_queue_task_handler(api: veilid.VeilidAPI):
while True:
update = await app_call_queue.get()
global got_message
got_message += 1
sys.stdout.write("{} ".format(got_message))
sys.stdout.flush()
await api.app_call_reply(update.detail.call_id, update.detail.message)
await api.app_call_reply(update.detail.call_id, update.detail.message)
hostname, port = server_info()
api = await veilid.json_api_connect(
hostname, port, app_message_queue_update_callback
hostname, port, app_call_queue_update_callback
)
async with api:
# purge routes to ensure we start fresh
await api.debug("purge routes")
app_call_task = asyncio.create_task(
app_call_queue_task_handler(api), name="app call task"
)
# make a routing context that uses a safety route
rc = await (await (await api.new_routing_context()).with_privacy()).with_sequencing(veilid.Sequencing.ENSURE_ORDERED)
async with rc:
@ -200,11 +213,57 @@ async def test_routing_context_app_call_loopback_big_packets():
# import it as a remote route as well so we can send to it
prr = await api.import_remote_private_route(blob)
# do this test 100 times
for _ in range(100):
# do this test 10 times
for _ in range(10):
# send a random sized random app message to our own private route
message = random.randbytes(random.randint(0, 32768))
out_message = await rc.app_call(prr, message)
assert message == out_message
app_call_task.cancel()
@pytest.mark.skipif(os.getenv("NOSKIP")!="1", reason="unneeded test, only for performance check")
@pytest.mark.asyncio
async def test_routing_context_app_message_loopback_bandwidth():
app_message_queue: asyncio.Queue = asyncio.Queue()
async def app_message_queue_update_callback(update: veilid.VeilidUpdate):
if update.kind == veilid.VeilidUpdateKind.APP_MESSAGE:
await app_message_queue.put(True)
hostname, port = server_info()
api = await veilid.json_api_connect(
hostname, port, app_message_queue_update_callback
)
async with api:
# purge routes to ensure we start fresh
await api.debug("purge routes")
# make a routing context that uses a safety route
#rc = await (await (await api.new_routing_context()).with_privacy()).with_sequencing(veilid.Sequencing.ENSURE_ORDERED)
#rc = await (await api.new_routing_context()).with_privacy()
rc = await api.new_routing_context()
async with rc:
# make a new local private route
prl, blob = await api.new_private_route()
# import it as a remote route as well so we can send to it
prr = await api.import_remote_private_route(blob)
# do this test 1000 times
message = random.randbytes(16384)
for _ in range(10000):
# send a random sized random app message to our own private route
await rc.app_message(prr, message)
# we should get the same number of messages back (not storing all that data)
for _ in range(10000):
await asyncio.wait_for(
app_message_queue.get(), timeout=10
)

2
veilid-python/veilid/api.py
View file

@ -166,7 +166,7 @@ class TableDb(ABC):
class CryptoSystem(ABC):
async def __aenter__(self) -> Self:
return self

16
veilid-python/veilid/json_api.py
View file

@ -51,7 +51,7 @@ _VALIDATOR_RECV_MESSAGE = _get_schema_validator(
class _JsonVeilidAPI(VeilidAPI):
reader: Optional[asyncio.StreamReader]
writer: Optional[asyncio.StreamWriter]
update_callback: Callable[[VeilidAPI, VeilidUpdate], Awaitable]
update_callback: Callable[[VeilidUpdate], Awaitable]
handle_recv_messages_task: Optional[asyncio.Task]
validate_schema: bool
done: bool
@ -64,7 +64,7 @@ class _JsonVeilidAPI(VeilidAPI):
self,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter,
update_callback: Callable[[VeilidAPI, VeilidUpdate], Awaitable],
update_callback: Callable[[VeilidUpdate], Awaitable],
validate_schema: bool = True,
):
self.reader = reader
@ -115,7 +115,7 @@ class _JsonVeilidAPI(VeilidAPI):
@classmethod
async def connect(
cls, host: str, port: int, update_callback: Callable[[VeilidAPI, VeilidUpdate], Awaitable]
cls, host: str, port: int, update_callback: Callable[[VeilidUpdate], Awaitable]
) -> Self:
reader, writer = await asyncio.open_connection(host, port)
veilid_api = cls(reader, writer, update_callback)
@ -135,6 +135,8 @@ class _JsonVeilidAPI(VeilidAPI):
# Resolve the request's future to the response json
if reqfuture is not None:
reqfuture.set_result(j)
else:
print("Missing id: {}", id)
async def handle_recv_messages(self):
# Read lines until we're done
@ -155,7 +157,7 @@ class _JsonVeilidAPI(VeilidAPI):
if j["type"] == "Response":
await self.handle_recv_message_response(j)
elif j["type"] == "Update":
await self.update_callback(self, VeilidUpdate.from_json(j))
await self.update_callback(VeilidUpdate.from_json(j))
finally:
await self._cleanup_close()
@ -485,7 +487,7 @@ class _JsonRoutingContext(RoutingContext):
await self.release()
return self.__class__(self.api, new_rc_id)
async def app_call(self, target: TypedKey | RouteId, request: bytes) -> bytes:
async def app_call(self, target: TypedKey | RouteId, message: bytes) -> bytes:
return urlsafe_b64decode_no_pad(
raise_api_result(
await self.api.send_ndjson_request(
@ -494,7 +496,7 @@ class _JsonRoutingContext(RoutingContext):
rc_id=self.rc_id,
rc_op=RoutingContextOperation.APP_CALL,
target=target,
request=request,
message=message,
)
)
)
@ -1162,6 +1164,6 @@ class _JsonCryptoSystem(CryptoSystem):
async def json_api_connect(
host: str, port: int, update_callback: Callable[[VeilidAPI, VeilidUpdate], Awaitable]
host: str, port: int, update_callback: Callable[[VeilidUpdate], Awaitable]
) -> _JsonVeilidAPI:
return await _JsonVeilidAPI.connect(host, port, update_callback)

8
veilid-python/veilid/schema/Request.json
View file

@ -259,20 +259,20 @@
{
"type": "object",
"required": [
"message",
"rc_op",
"request",
"target"
],
"properties": {
"message": {
"type": "string"
},
"rc_op": {
"type": "string",
"enum": [
"AppCall"
]
},
"request": {
"type": "string"
},
"target": {
"type": "string"
}