diff --git a/synapse/storage/stream.py b/synapse/storage/stream.py
index 47f05a41b..bb56f0763 100644
--- a/synapse/storage/stream.py
+++ b/synapse/storage/stream.py
@@ -89,10 +89,14 @@ class StreamStore(SQLBaseStore):
 
         ret = [self._parse_event_from_row(r) for r in rows]
 
-        if from_key < to_key:
-            key = max([r["ordering"] for r in rows])
+
+        if rows:
+            if from_key < to_key:
+                key = max([r["ordering"] for r in rows])
+            else:
+                key = min([r["ordering"] for r in rows])
         else:
-            key = min([r["ordering"] for r in rows])
+            key = to_key
 
         defer.returnValue((ret, key))