diff --git a/synapse/notifier.py b/synapse/notifier.py
index 344dd0317..1f7f0a143 100644
--- a/synapse/notifier.py
+++ b/synapse/notifier.py
@@ -145,8 +145,8 @@ class Notifier(object):
 
             for x in self.room_to_user_streams.values():
                 all_user_streams |= x
-            for x in self.user_to_user_streams.values():
-                all_user_streams |= x
+            for x in self.user_to_user_stream:
+                all_user_streams.add(x)
             for x in self.appservice_to_user_streams.values():
                 all_user_streams |= x