Git-Mirrors/veilid
1
0
Fork 0
mirror of https://gitlab.com/veilid/veilid.git synced 2024-10-01 01:26:08 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dht-testing' into 'main'

Browse Source 
More bug fixes for tabledb and veilid-python

See merge request veilid/veilid!28
This commit is contained in:
John Smith 2023-06-19 01:35:29 +00:00
commit 9e496e1625
6 changed files with 140 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.vscode/settings.json vendored
View File

@ -1,3 +1,6 @@
{
"cmake.configureOnOpen": false
"cmake.configureOnOpen": false,
"python.analysis.extraPaths": [
"veilid-python/.venv/lib/python3.11/site-packages"
],
}

23
veilid-core/src/storage_manager/mod.rs
View File

@ -233,8 +233,11 @@ impl StorageManager {
force_refresh: bool,
) -> VeilidAPIResult<Option<ValueData>> {
let mut inner = self.lock().await?;
let Some(opened_record) = inner.opened_records.remove(&key) else {
apibail_generic!("record not open");
let safety_selection = {
let Some(opened_record) = inner.opened_records.get(&key) else {
apibail_generic!("record not open");
};
opened_record.safety_selection()
};
// See if the requested subkey is our local record store
@ -269,7 +272,7 @@ impl StorageManager {
rpc_processor,
key,
subkey,
opened_record.safety_selection(),
safety_selection,
last_subkey_result,
)
.await?;
@ -307,12 +310,18 @@ impl StorageManager {
apibail_generic!("unsupported cryptosystem");
};
let Some(opened_record) = inner.opened_records.remove(&key) else {
apibail_generic!("record not open");
let (safety_selection, opt_writer) = {
let Some(opened_record) = inner.opened_records.get(&key) else {
apibail_generic!("record not open");
};
(
opened_record.safety_selection(),
opened_record.writer().cloned(),
)
};
// If we don't have a writer then we can't write
let Some(writer) = opened_record.writer().cloned() else {
let Some(writer) = opt_writer else {
apibail_generic!("value is not writable");
};
@ -371,7 +380,7 @@ impl StorageManager {
rpc_processor,
key,
subkey,
opened_record.safety_selection(),
safety_selection,
signed_value_data,
descriptor,
)

14
veilid-core/src/table_store/table_store.rs
View File

@ -428,7 +428,7 @@ impl TableStore {
}
pub(crate) fn on_table_db_drop(&self, table: String) {
log_rtab!(debug "dropping table db: {}", table);
log_rtab!("dropping table db: {}", table);
let mut inner = self.inner.lock();
if inner.opened.remove(&table).is_none() {
unreachable!("should have removed an item");
@ -557,15 +557,15 @@ impl TableStore {
let deleted = self.table_store_driver.delete(&table_name).await?;
if !deleted {
// Table missing? Just remove name
self.name_delete(&name)
.await
.expect("failed to delete name");
warn!(
"table existed in name table but not in storage: {} : {}",
name, table_name
);
return Ok(false);
}
self.name_delete(&name)
.await
.expect("failed to delete name");
self.flush().await;
Ok(true)
}
@ -581,6 +581,8 @@ impl TableStore {
}
}
trace!("TableStore::rename {} -> {}", old_name, new_name);
self.name_rename(old_name, new_name).await
self.name_rename(old_name, new_name).await?;
self.flush().await;
Ok(())
}
}

45
veilid-python/tests/test_dht.py
View File

@ -1,13 +1,13 @@
# Routing context veilid tests
# # Routing context veilid tests
import veilid
import pytest
import asyncio
import json
from . import *
# import veilid
# import pytest
# import asyncio
# import json
# from . import *
##################################################################
BOGUS_KEY = veilid.TypedKey.from_value(veilid.CryptoKind.CRYPTO_KIND_VLD0, veilid.PublicKey.from_bytes(b' '))
# ##################################################################
# BOGUS_KEY = veilid.TypedKey.from_value(veilid.CryptoKind.CRYPTO_KIND_VLD0, veilid.PublicKey.from_bytes(b' '))
# @pytest.mark.asyncio
# async def test_get_dht_value_unopened(api_connection: veilid.VeilidAPI):
@ -46,6 +46,29 @@ BOGUS_KEY = veilid.TypedKey.from_value(veilid.CryptoKind.CRYPTO_KIND_VLD0, veili
# await rc.close_dht_record(rec.key)
# await rc.delete_dht_record(rec.key)
# xxx make tests for tabledb api first
# xxx then make a test that creates a record, stores it in a table
# xxx then make another test that gets the keys from the table and closes/deletes them
# @pytest.mark.asyncio
# async def test_get_dht_value_nonexistent(api_connection: veilid.VeilidAPI):
# rc = await api_connection.new_routing_context()
# async with rc:
# rec = await rc.create_dht_record(veilid.CryptoKind.CRYPTO_KIND_VLD0, veilid.DHTSchema.dflt(1))
# assert await rc.get_dht_value(rec.key, 0, False) == None
# await rc.close_dht_record(rec.key)
# await rc.delete_dht_record(rec.key)
# @pytest.mark.asyncio
# async def test_set_get_dht_value(api_connection: veilid.VeilidAPI):
# rc = await api_connection.new_routing_context()
# async with rc:
# rec = await rc.create_dht_record(veilid.CryptoKind.CRYPTO_KIND_VLD0, veilid.DHTSchema.dflt(1))
# vd = await rc.set_dht_value(rec.key, 0, b"BLAH BLAH BLAH")
# assert vd != None
# vd2 = await rc.get_dht_value(rec.key, 0, False)
# assert vd2 != None
# assert vd == vd2
# await rc.close_dht_record(rec.key)
# await rc.delete_dht_record(rec.key)

61
veilid-python/veilid/api.py
View File

@ -6,6 +6,18 @@ from .state import VeilidState
class RoutingContext(ABC):
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.is_done():
await self.release()
@abstractmethod
def is_done(self) -> bool:
pass
@abstractmethod
async def release(self):
pass
@ -84,6 +96,17 @@ class RoutingContext(ABC):
class TableDbTransaction(ABC):
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.is_done():
await self.rollback()
@abstractmethod
def is_done(self) -> bool:
pass
@abstractmethod
async def commit(self):
pass
@ -102,6 +125,17 @@ class TableDbTransaction(ABC):
class TableDb(ABC):
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.is_done():
await self.release()
@abstractmethod
def is_done(self) -> bool:
pass
@abstractmethod
async def release(self):
pass
@ -132,6 +166,18 @@ class TableDb(ABC):
class CryptoSystem(ABC):
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.is_done():
await self.release()
@abstractmethod
def is_done(self) -> bool:
pass
@abstractmethod
async def release(self):
pass
@ -246,6 +292,21 @@ class CryptoSystem(ABC):
class VeilidAPI(ABC):
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.is_done():
await self.release()
@abstractmethod
def is_done(self) -> bool:
pass
@abstractmethod
async def release(self):
pass
@abstractmethod
async def control(self, args: list[str]) -> str:
pass

50
veilid-python/veilid/json_api.py
View File

@ -53,7 +53,8 @@ class _JsonVeilidAPI(VeilidAPI):
writer: Optional[asyncio.StreamWriter]
update_callback: Callable[[VeilidUpdate], Awaitable]
handle_recv_messages_task: Optional[asyncio.Task]
validate_schemas: bool
validate_schema: bool
done: bool
# Shared Mutable State
lock: asyncio.Lock
next_id: int
@ -70,17 +71,12 @@ class _JsonVeilidAPI(VeilidAPI):
self.writer = writer
self.update_callback = update_callback
self.validate_schema = validate_schema
self.done = False
self.handle_recv_messages_task = None
self.lock = asyncio.Lock()
self.next_id = 1
self.in_flight_requests = dict()
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
await self.close()
async def _cleanup_close(self):
await self.lock.acquire()
try:
@ -96,7 +92,10 @@ class _JsonVeilidAPI(VeilidAPI):
finally:
self.lock.release()
async def close(self):
def is_done(self) -> bool:
return self.done
async def release(self):
# Take the task
await self.lock.acquire()
try:
@ -112,6 +111,7 @@ class _JsonVeilidAPI(VeilidAPI):
await handle_recv_messages_task
except asyncio.CancelledError:
pass
self.done = True
@classmethod
async def connect(
@ -430,12 +430,8 @@ class _JsonRoutingContext(RoutingContext):
# complain
raise AssertionError("Should have released routing context before dropping object")
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.done:
await self.release()
def is_done(self) -> bool:
return self.done
async def release(self):
if self.done:
@ -668,12 +664,8 @@ class _JsonTableDbTransaction(TableDbTransaction):
# complain
raise AssertionError("Should have committed or rolled back transaction before dropping object")
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.done:
await self.rollback()
def is_done(self) -> bool:
return self.done
async def commit(self):
if self.done:
@ -753,12 +745,8 @@ class _JsonTableDb(TableDb):
# complain
raise AssertionError("Should have released table db before dropping object")
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.done:
await self.release()
def is_done(self) -> bool:
return self.done
async def release(self):
if self.done:
@ -880,13 +868,9 @@ class _JsonCryptoSystem(CryptoSystem):
# complain
raise AssertionError("Should have released crypto system before dropping object")
async def __aenter__(self) -> Self:
return self
async def __aexit__(self, *excinfo):
if not self.done:
await self.release()
def is_done(self) -> bool:
return self.done
async def release(self):
if self.done:
return