diff --git a/synapse/storage/devices.py b/synapse/storage/devices.py
index 2b2cebacf..89c7bc0cc 100644
--- a/synapse/storage/devices.py
+++ b/synapse/storage/devices.py
@@ -486,6 +486,8 @@ class DeviceStore(SQLBaseStore):
         defer.returnValue(stream_id)
 
     def _add_device_change_txn(self, txn, user_id, device_ids, hosts, stream_id):
+        now = self._clock.time_msec()
+
         txn.call_after(
             self._device_list_stream_cache.entity_has_changed,
             user_id, stream_id,
@@ -519,6 +521,7 @@ class DeviceStore(SQLBaseStore):
                     "user_id": user_id,
                     "device_id": device_id,
                     "sent": False,
+                    "ts": now,
                 }
                 for destination in hosts
                 for device_id in device_ids
diff --git a/synapse/storage/schema/delta/40/device_list_streams.sql b/synapse/storage/schema/delta/40/device_list_streams.sql
index 8348c143c..54841b384 100644
--- a/synapse/storage/schema/delta/40/device_list_streams.sql
+++ b/synapse/storage/schema/delta/40/device_list_streams.sql
@@ -51,7 +51,8 @@ CREATE TABLE device_lists_outbound_pokes (
     stream_id BIGINT NOT NULL,
     user_id TEXT NOT NULL,
     device_id TEXT NOT NULL,
-    sent BOOLEAN NOT NULL
+    sent BOOLEAN NOT NULL,
+    ts BIGINT NOT NULL  -- So that in future we can clear out pokes to dead servers
 );
 
 CREATE INDEX device_lists_outbound_pokes_id ON device_lists_outbound_pokes(destination, stream_id);