diff --git a/synapse/handlers/sync.py b/synapse/handlers/sync.py
index 91934b0c8..14f2032af 100644
--- a/synapse/handlers/sync.py
+++ b/synapse/handlers/sync.py
@@ -565,21 +565,26 @@ class SyncHandler(object):
         if sync_result_builder.since_token is not None:
             since_stream_id = int(sync_result_builder.since_token.to_device_key)
 
-        if since_stream_id:
+        if since_stream_id != int(now_token.to_device_key):
+            # We only delete messages when a new message comes in, but that's
+            # fine so long as we delete them at some point.
+
             logger.debug("Deleting messages up to %d", since_stream_id)
             yield self.store.delete_messages_for_device(
                 user_id, device_id, since_stream_id
             )
 
-        logger.debug("Getting messages up to %d", now_token.to_device_key)
-        messages, stream_id = yield self.store.get_new_messages_for_device(
-            user_id, device_id, now_token.to_device_key
-        )
-        logger.debug("Got messages up to %d: %r", stream_id, messages)
-        sync_result_builder.now_token = now_token.copy_and_replace(
-            "to_device_key", stream_id
-        )
-        sync_result_builder.to_device = messages
+            logger.debug("Getting messages up to %d", now_token.to_device_key)
+            messages, stream_id = yield self.store.get_new_messages_for_device(
+                user_id, device_id, now_token.to_device_key
+            )
+            logger.debug("Got messages up to %d: %r", stream_id, messages)
+            sync_result_builder.now_token = now_token.copy_and_replace(
+                "to_device_key", stream_id
+            )
+            sync_result_builder.to_device = messages
+        else:
+            sync_result_builder.to_device = []
 
     @defer.inlineCallbacks
     def _generate_sync_entry_for_account_data(self, sync_result_builder):