Rejig structure given the appservice_handler already filters the correct ASes to use.

This commit is contained in:
Kegan Dougal 2015-03-06 10:25:50 +00:00
parent 0c838f9f5e
commit d516d68b29

View File

@ -16,16 +16,10 @@
This module controls the reliability for application service transactions. This module controls the reliability for application service transactions.
The nominal flow through this module looks like: The nominal flow through this module looks like:
___________ _________
\O/ --- event -->| | +--------------+ ---ASa[e]-->| Event |
| - event ---->| event_pool|<-- poll 1/s for events ---| EventSorter | ----ASb[e]->| Grouper |<-poll 1/s--+
/ \ ---- event ->|___________| +--------------+ --ASa[e]--->|_________| | ASa[e,e] ASb[e]
USERS ____________________________|
| | |
V V V
ASa ASb ASc
[e,e] [e] [e,e,e]
|
V V
-````````- +------------+ -````````- +------------+
|````````|<--StoreTxn-|Transaction | |````````|<--StoreTxn-|Transaction |
@ -43,11 +37,11 @@ Recoverer attempts to recover ASes who have died. The flow for this looks like:
V | V |
START ---> Wait exp ------> Get oldest txn ID from ----> FAILURE START ---> Wait exp ------> Get oldest txn ID from ----> FAILURE
backoff DB and try to send it backoff DB and try to send it
^ |__________ ^ |___________
Mark AS as | V Mark AS as | V
UP & quit +---------- YES SUCCESS UP & quit +---------- YES SUCCESS
| | | | | |
NO <--- Have more txns? <------ Mark txn success & nuke -+ NO <--- Have more txns? <------ Mark txn success & nuke <-+
from db; incr AS pos. from db; incr AS pos.
Reset backoff. Reset backoff.
@ -62,24 +56,28 @@ class AppServiceScheduler(object):
case is a simple array. case is a simple array.
""" """
def __init__(self, store, as_api, services): def __init__(self, clock, store, as_api):
self.app_services = services self.clock = clock
self.event_pool = [] self.store = store
self.as_api = as_api
self.event_grouper = _EventGrouper()
def create_recoverer(service): def create_recoverer(service, callback):
return _Recoverer(store, as_api, service) return _Recoverer(clock, store, as_api, service, callback)
self.txn_ctrl = _TransactionController(store, as_api, create_recoverer)
self.event_sorter = _EventSorter(self, self.txn_ctrl, services) self.txn_ctrl = _TransactionController(
clock, store, as_api, self.event_grouper, create_recoverer
)
def start(self): def start(self):
self.event_sorter.start_polling() # check for any DOWN ASes and start recoverers for them.
_Recoverer.start(
self.clock, self.store, self.as_api, self.txn_ctrl.on_recovered
)
self.txn_ctrl.start_polling()
def store_event(self, event): # event_pool def submit_event_for_as(self, service, event):
self.event_pool.append(event) self.event_grouper.on_receive(service, event)
def drain_events(self): # event_pool
return self.event_pool
class AppServiceTransaction(object): class AppServiceTransaction(object):
@ -99,71 +97,99 @@ class AppServiceTransaction(object):
pass pass
class _EventSorter(object): class _EventGrouper(object):
"""Groups events for the same application service together.
"""
def __init__(self, event_pool, txn_ctrl, services): def __init__(self):
self.event_pool = event_pool self.groups = {} # dict of {service: [events]}
self.txn_ctrl = txn_ctrl
self.services = services
def start_polling(self): def on_receive(self, service, event):
events = self.event_pool.drain_events() # TODO group this
if events:
self._process(events)
# TODO repoll later on
def _process(self, events):
# TODO sort events
# TODO fe (AS, events) => poke transaction controller on_receive_events
pass pass
def drain_groups(self):
return self.groups
class _TransactionController(object): class _TransactionController(object):
def __init__(self, store, as_api, recoverer_fn): def __init__(self, clock, store, as_api, event_grouper, recoverer_fn):
self.clock = clock
self.store = store self.store = store
self.as_api = as_api self.as_api = as_api
self.event_grouper = event_grouper
self.recoverer_fn = recoverer_fn self.recoverer_fn = recoverer_fn
def on_receive_events(self, service, events): def start_polling(self):
txn = self._store_txn(service, events) groups = self.event_grouper.drain_groups()
for service in groups:
txn_id = self._get_next_txn_id(service)
txn = AppServiceTransaction(service, txn_id, groups[service])
self._store_txn(txn)
if self._is_service_up(service):
if txn.send(self.as_api): if txn.send(self.as_api):
txn.complete(self.store) txn.complete(self.store)
else: else:
# TODO mark AS as down
self._start_recoverer(service) self._start_recoverer(service)
self.clock.call_later(1000, self.start_polling)
def on_recovered(self, service):
# TODO mark AS as UP
pass
def _start_recoverer(self, service): def _start_recoverer(self, service):
recoverer = self.recoverer_fn(service) recoverer = self.recoverer_fn(service, self.on_recovered)
recoverer.recover() recoverer.recover()
def _store_txn(self, service, events): def _is_service_up(self, service):
pass # returns AppServiceTransaction pass
def _get_next_txn_id(self, service):
pass # TODO work out the next txn_id for this service
def _store_txn(self, txn):
pass
class _Recoverer(object): class _Recoverer(object):
def __init__(self, store, as_api, service): @staticmethod
def start(clock, store, as_api, callback):
# TODO check for DOWN ASes and init recoverers
pass
def __init__(self, clock, store, as_api, service, callback):
self.clock = clock
self.store = store self.store = store
self.as_api = as_api self.as_api = as_api
self.service = service self.service = service
self.callback = callback
self.backoff_counter = 1 self.backoff_counter = 1
def recover(self): def recover(self):
# TODO wait a bit self.clock.call_later(2000 ** self.backoff_counter, self.retry)
def retry(self):
txn = self._get_oldest_txn() txn = self._get_oldest_txn()
if txn: if txn:
if txn.send(self.as_api): if txn.send(self.as_api):
txn.complete(self.store) txn.complete(self.store)
# reset the backoff counter and retry immediately
self.backoff_counter = 1 self.backoff_counter = 1
else: self.retry()
self.backoff_counter += 1
self.recover(self.service)
return return
else: else:
self._set_service_recovered(self.service) self.backoff_counter += 1
self.recover()
return
else:
self._set_service_recovered()
def _set_service_recovered(self, service): def _set_service_recovered(self):
pass self.callback(self.service)
def _get_oldest_txn(self): def _get_oldest_txn(self):
pass # returns AppServiceTransaction pass # returns AppServiceTransaction