diff --git a/pantalaimon/client.py b/pantalaimon/client.py
index 3f419ef..8023634 100644
--- a/pantalaimon/client.py
+++ b/pantalaimon/client.py
@@ -811,12 +811,9 @@ class PanClient(AsyncClient):
             return
 
         for event in to_device_events["events"]:
-            if event.get("type") != "m.room.encrypted":
-                continue
-
             event = ToDeviceEvent.parse_encrypted_event(event)
 
-            if not isinstance(event, OlmEvent):
+            if not isinstance(event, ToDeviceEvent):
                 continue
 
             self.olm.handle_to_device_event(event)