Catch any exceptions in the pusher loop. Use a lower timeout for pushers so we can see if they're actually still running.

synapse/push/__init__.py

@@ -24,6 +24,7 @@ import baserules
 import logging
 import simplejson as json
 import re
+import random
 
 logger = logging.getLogger(__name__)
 
@@ -256,12 +257,31 @@ class Pusher(object):
             logger.info("Pusher %s for user %s starting from token %s",
                         self.pushkey, self.user_name, self.last_token)
 
+        wait = 0
         while self.alive:
+            try:
+                if wait > 0:
+                    yield synapse.util.async.sleep(wait)
+                yield self.get_and_dispatch()
+                wait = 0
+            except:
+                if wait == 0:
+                    wait = 1
+                else:
+                    wait = min(wait * 2, 1800)
+                logger.exception(
+                    "Exception in pusher loop for pushkey %s. Pausing for %ds",
+                    self.pushkey, wait
+                )
+
+    @defer.inlineCallbacks
+    def get_and_dispatch(self):
         from_tok = StreamToken.from_string(self.last_token)
         config = PaginationConfig(from_token=from_tok, limit='1')
+        timeout = (300 + random.randint(-60, 60)) * 1000
         chunk = yield self.evStreamHandler.get_stream(
             self.user_name, config,
-                timeout=100*365*24*60*60*1000, affect_presence=False
+            timeout=timeout, affect_presence=False
         )
 
         # limiting to 1 may get 1 event plus 1 presence event, so
@@ -273,10 +293,11 @@ class Pusher(object):
                 break
         if not single_event:
             self.last_token = chunk['end']
-                continue
+            logger.debug("Event stream timeout for pushkey %s", self.pushkey)
+            return
 
         if not self.alive:
-                continue
+            return
 
         processed = False
         actions = yield self._actions_for_event(single_event)
@@ -319,7 +340,7 @@ class Pusher(object):
                         )
 
         if not self.alive:
-                continue
+            return
 
         if processed:
             self.backoff_delay = Pusher.INITIAL_BACKOFF