mirror of
https://git.anonymousland.org/anonymousland/synapse-product.git
synced 2024-12-24 22:49:29 -05:00
Rejig structure given the appservice_handler already filters the correct ASes to use.
This commit is contained in:
parent
0c838f9f5e
commit
d516d68b29
@ -16,16 +16,10 @@
|
|||||||
This module controls the reliability for application service transactions.
|
This module controls the reliability for application service transactions.
|
||||||
|
|
||||||
The nominal flow through this module looks like:
|
The nominal flow through this module looks like:
|
||||||
___________
|
_________
|
||||||
\O/ --- event -->| | +--------------+
|
---ASa[e]-->| Event |
|
||||||
| - event ---->| event_pool|<-- poll 1/s for events ---| EventSorter |
|
----ASb[e]->| Grouper |<-poll 1/s--+
|
||||||
/ \ ---- event ->|___________| +--------------+
|
--ASa[e]--->|_________| | ASa[e,e] ASb[e]
|
||||||
USERS ____________________________|
|
|
||||||
| | |
|
|
||||||
V V V
|
|
||||||
ASa ASb ASc
|
|
||||||
[e,e] [e] [e,e,e]
|
|
||||||
|
|
|
||||||
V
|
V
|
||||||
-````````- +------------+
|
-````````- +------------+
|
||||||
|````````|<--StoreTxn-|Transaction |
|
|````````|<--StoreTxn-|Transaction |
|
||||||
@ -43,11 +37,11 @@ Recoverer attempts to recover ASes who have died. The flow for this looks like:
|
|||||||
V |
|
V |
|
||||||
START ---> Wait exp ------> Get oldest txn ID from ----> FAILURE
|
START ---> Wait exp ------> Get oldest txn ID from ----> FAILURE
|
||||||
backoff DB and try to send it
|
backoff DB and try to send it
|
||||||
^ |__________
|
^ |___________
|
||||||
Mark AS as | V
|
Mark AS as | V
|
||||||
UP & quit +---------- YES SUCCESS
|
UP & quit +---------- YES SUCCESS
|
||||||
| | |
|
| | |
|
||||||
NO <--- Have more txns? <------ Mark txn success & nuke -+
|
NO <--- Have more txns? <------ Mark txn success & nuke <-+
|
||||||
from db; incr AS pos.
|
from db; incr AS pos.
|
||||||
Reset backoff.
|
Reset backoff.
|
||||||
|
|
||||||
@ -62,24 +56,28 @@ class AppServiceScheduler(object):
|
|||||||
case is a simple array.
|
case is a simple array.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
def __init__(self, store, as_api, services):
|
def __init__(self, clock, store, as_api):
|
||||||
self.app_services = services
|
self.clock = clock
|
||||||
self.event_pool = []
|
self.store = store
|
||||||
|
self.as_api = as_api
|
||||||
|
self.event_grouper = _EventGrouper()
|
||||||
|
|
||||||
def create_recoverer(service):
|
def create_recoverer(service, callback):
|
||||||
return _Recoverer(store, as_api, service)
|
return _Recoverer(clock, store, as_api, service, callback)
|
||||||
self.txn_ctrl = _TransactionController(store, as_api, create_recoverer)
|
|
||||||
|
|
||||||
self.event_sorter = _EventSorter(self, self.txn_ctrl, services)
|
self.txn_ctrl = _TransactionController(
|
||||||
|
clock, store, as_api, self.event_grouper, create_recoverer
|
||||||
|
)
|
||||||
|
|
||||||
def start(self):
|
def start(self):
|
||||||
self.event_sorter.start_polling()
|
# check for any DOWN ASes and start recoverers for them.
|
||||||
|
_Recoverer.start(
|
||||||
|
self.clock, self.store, self.as_api, self.txn_ctrl.on_recovered
|
||||||
|
)
|
||||||
|
self.txn_ctrl.start_polling()
|
||||||
|
|
||||||
def store_event(self, event): # event_pool
|
def submit_event_for_as(self, service, event):
|
||||||
self.event_pool.append(event)
|
self.event_grouper.on_receive(service, event)
|
||||||
|
|
||||||
def drain_events(self): # event_pool
|
|
||||||
return self.event_pool
|
|
||||||
|
|
||||||
|
|
||||||
class AppServiceTransaction(object):
|
class AppServiceTransaction(object):
|
||||||
@ -99,71 +97,99 @@ class AppServiceTransaction(object):
|
|||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
class _EventSorter(object):
|
class _EventGrouper(object):
|
||||||
|
"""Groups events for the same application service together.
|
||||||
|
"""
|
||||||
|
|
||||||
def __init__(self, event_pool, txn_ctrl, services):
|
def __init__(self):
|
||||||
self.event_pool = event_pool
|
self.groups = {} # dict of {service: [events]}
|
||||||
self.txn_ctrl = txn_ctrl
|
|
||||||
self.services = services
|
|
||||||
|
|
||||||
def start_polling(self):
|
def on_receive(self, service, event):
|
||||||
events = self.event_pool.drain_events()
|
# TODO group this
|
||||||
if events:
|
|
||||||
self._process(events)
|
|
||||||
# TODO repoll later on
|
|
||||||
|
|
||||||
def _process(self, events):
|
|
||||||
# TODO sort events
|
|
||||||
# TODO fe (AS, events) => poke transaction controller on_receive_events
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
def drain_groups(self):
|
||||||
|
return self.groups
|
||||||
|
|
||||||
|
|
||||||
class _TransactionController(object):
|
class _TransactionController(object):
|
||||||
|
|
||||||
def __init__(self, store, as_api, recoverer_fn):
|
def __init__(self, clock, store, as_api, event_grouper, recoverer_fn):
|
||||||
|
self.clock = clock
|
||||||
self.store = store
|
self.store = store
|
||||||
self.as_api = as_api
|
self.as_api = as_api
|
||||||
|
self.event_grouper = event_grouper
|
||||||
self.recoverer_fn = recoverer_fn
|
self.recoverer_fn = recoverer_fn
|
||||||
|
|
||||||
def on_receive_events(self, service, events):
|
def start_polling(self):
|
||||||
txn = self._store_txn(service, events)
|
groups = self.event_grouper.drain_groups()
|
||||||
|
for service in groups:
|
||||||
|
txn_id = self._get_next_txn_id(service)
|
||||||
|
txn = AppServiceTransaction(service, txn_id, groups[service])
|
||||||
|
self._store_txn(txn)
|
||||||
|
if self._is_service_up(service):
|
||||||
if txn.send(self.as_api):
|
if txn.send(self.as_api):
|
||||||
txn.complete(self.store)
|
txn.complete(self.store)
|
||||||
else:
|
else:
|
||||||
|
# TODO mark AS as down
|
||||||
self._start_recoverer(service)
|
self._start_recoverer(service)
|
||||||
|
self.clock.call_later(1000, self.start_polling)
|
||||||
|
|
||||||
|
|
||||||
|
def on_recovered(self, service):
|
||||||
|
# TODO mark AS as UP
|
||||||
|
pass
|
||||||
|
|
||||||
def _start_recoverer(self, service):
|
def _start_recoverer(self, service):
|
||||||
recoverer = self.recoverer_fn(service)
|
recoverer = self.recoverer_fn(service, self.on_recovered)
|
||||||
recoverer.recover()
|
recoverer.recover()
|
||||||
|
|
||||||
def _store_txn(self, service, events):
|
def _is_service_up(self, service):
|
||||||
pass # returns AppServiceTransaction
|
pass
|
||||||
|
|
||||||
|
def _get_next_txn_id(self, service):
|
||||||
|
pass # TODO work out the next txn_id for this service
|
||||||
|
|
||||||
|
def _store_txn(self, txn):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
class _Recoverer(object):
|
class _Recoverer(object):
|
||||||
|
|
||||||
def __init__(self, store, as_api, service):
|
@staticmethod
|
||||||
|
def start(clock, store, as_api, callback):
|
||||||
|
# TODO check for DOWN ASes and init recoverers
|
||||||
|
pass
|
||||||
|
|
||||||
|
def __init__(self, clock, store, as_api, service, callback):
|
||||||
|
self.clock = clock
|
||||||
self.store = store
|
self.store = store
|
||||||
self.as_api = as_api
|
self.as_api = as_api
|
||||||
self.service = service
|
self.service = service
|
||||||
|
self.callback = callback
|
||||||
self.backoff_counter = 1
|
self.backoff_counter = 1
|
||||||
|
|
||||||
def recover(self):
|
def recover(self):
|
||||||
# TODO wait a bit
|
self.clock.call_later(2000 ** self.backoff_counter, self.retry)
|
||||||
|
|
||||||
|
def retry(self):
|
||||||
txn = self._get_oldest_txn()
|
txn = self._get_oldest_txn()
|
||||||
if txn:
|
if txn:
|
||||||
if txn.send(self.as_api):
|
if txn.send(self.as_api):
|
||||||
txn.complete(self.store)
|
txn.complete(self.store)
|
||||||
|
# reset the backoff counter and retry immediately
|
||||||
self.backoff_counter = 1
|
self.backoff_counter = 1
|
||||||
else:
|
self.retry()
|
||||||
self.backoff_counter += 1
|
|
||||||
self.recover(self.service)
|
|
||||||
return
|
return
|
||||||
else:
|
else:
|
||||||
self._set_service_recovered(self.service)
|
self.backoff_counter += 1
|
||||||
|
self.recover()
|
||||||
|
return
|
||||||
|
else:
|
||||||
|
self._set_service_recovered()
|
||||||
|
|
||||||
def _set_service_recovered(self, service):
|
def _set_service_recovered(self):
|
||||||
pass
|
self.callback(self.service)
|
||||||
|
|
||||||
def _get_oldest_txn(self):
|
def _get_oldest_txn(self):
|
||||||
pass # returns AppServiceTransaction
|
pass # returns AppServiceTransaction
|
||||||
|
Loading…
Reference in New Issue
Block a user