Add a sleep to the Limiter to fix stack overflows.

Fixes #3570
This commit is contained in:
Richard van der Hoff 2018-07-20 12:37:12 +01:00
parent 7044af3298
commit d7275eecf3
2 changed files with 24 additions and 7 deletions

View File

@ -248,11 +248,15 @@ class Limiter(object):
# do some work. # do some work.
""" """
def __init__(self, max_count): def __init__(self, max_count, clock=None):
""" """
Args: Args:
max_count(int): The maximum number of concurrent access max_count(int): The maximum number of concurrent access
""" """
if not clock:
from twisted.internet import reactor
clock = Clock(reactor)
self._clock = clock
self.max_count = max_count self.max_count = max_count
# key_to_defer is a map from the key to a 2 element list where # key_to_defer is a map from the key to a 2 element list where
@ -277,9 +281,22 @@ class Limiter(object):
with PreserveLoggingContext(): with PreserveLoggingContext():
yield new_defer yield new_defer
logger.info("Acquired limiter lock for key %r", key) logger.info("Acquired limiter lock for key %r", key)
entry[0] += 1
# if the code holding the lock completes synchronously, then it
# will recursively run the next claimant on the list. That can
# relatively rapidly lead to stack exhaustion. This is essentially
# the same problem as http://twistedmatrix.com/trac/ticket/9304.
#
# In order to break the cycle, we add a cheeky sleep(0) here to
# ensure that we fall back to the reactor between each iteration.
#
# (This needs to happen while we hold the lock, and the context manager's exit
# code must be synchronous, so this is the only sensible place.)
yield self._clock.sleep(0)
else: else:
logger.info("Acquired uncontended limiter lock for key %r", key) logger.info("Acquired uncontended limiter lock for key %r", key)
entry[0] += 1 entry[0] += 1
@contextmanager @contextmanager

View File

@ -48,21 +48,21 @@ class LimiterTestCase(unittest.TestCase):
self.assertFalse(d4.called) self.assertFalse(d4.called)
self.assertFalse(d5.called) self.assertFalse(d5.called)
self.assertTrue(d4.called) cm4 = yield d4
self.assertFalse(d5.called) self.assertFalse(d5.called)
with cm3: with cm3:
self.assertFalse(d5.called) self.assertFalse(d5.called)
self.assertTrue(d5.called) cm5 = yield d5
with cm2: with cm2:
pass pass
with (yield d4): with cm4:
pass pass
with (yield d5): with cm5:
pass pass
d6 = limiter.queue(key) d6 = limiter.queue(key)