Go back to storing JSON in TEXT

synapse/storage/_base.py

@@ -818,9 +818,6 @@ class SQLBaseStore(object):
 
         internal_metadata, js, redacted, rejected_reason = res
 
-        internal_metadata = self.database_engine.load_unicode(internal_metadata)
-        js = self.database_engine.load_unicode(js)
-
         start_time = update_counter("select_event", start_time)
 
         result = self._get_event_from_row_txn(

synapse/storage/appservice.py

@@ -366,9 +366,7 @@ class ApplicationServiceTransactionStore(SQLBaseStore):
         new_txn_id = max(highest_txn_id, last_txn_id) + 1
 
         # Insert new txn into txn table
-        event_ids = buffer(
+        event_ids = json.dumps([e.event_id for e in events])
-            json.dumps([e.event_id for e in events]).encode("utf8")
-        )
         txn.execute(
             "INSERT INTO application_services_txns(as_id, txn_id, event_ids) "
             "VALUES(?,?,?)",

synapse/storage/engines/postgres.py

@@ -39,6 +39,3 @@ class PostgresEngine(object):
         if isinstance(error, self.module.DatabaseError):
             return error.pgcode in ["40001", "40P01"]
         return False
-
-    def load_unicode(self, v):
-        return bytes(v).decode("UTF8")

synapse/storage/engines/sqlite3.py

@@ -37,8 +37,3 @@ class Sqlite3Engine(object):
 
     def is_deadlock(self, error):
         return False
-
-    def load_unicode(self, v):
-        if isinstance(v, types.UnicodeType):
-            return v
-        return bytes(v).decode("UTF8")

synapse/storage/events.py

@@ -168,7 +168,7 @@ class EventsStore(SQLBaseStore):
 
         metadata_json = encode_canonical_json(
             event.internal_metadata.get_dict()
-        )
+        ).decode("UTF-8")
 
         # If we have already persisted this event, we don't need to do any
         # more processing.
@@ -184,7 +184,7 @@ class EventsStore(SQLBaseStore):
                 )
                 txn.execute(
                     sql,
-                    (buffer(metadata_json), event.event_id,)
+                    (metadata_json, event.event_id,)
                 )
 
                 sql = (
@@ -229,14 +229,14 @@ class EventsStore(SQLBaseStore):
             values={
                 "event_id": event.event_id,
                 "room_id": event.room_id,
-                "internal_metadata": buffer(metadata_json),
+                "internal_metadata": metadata_json,
-                "json": buffer(encode_canonical_json(event_dict)),
+                "json": encode_canonical_json(event_dict).decode("UTF-8"),
             },
         )
 
-        content = buffer(encode_canonical_json(
+        content = encode_canonical_json(
             event.content
-        ))
+        ).decode("UTF-8")
 
         vals = {
             "topological_ordering": event.depth,
@@ -261,9 +261,9 @@ class EventsStore(SQLBaseStore):
             ]
         }
 
-        vals["unrecognized_keys"] = buffer(encode_canonical_json(
+        vals["unrecognized_keys"] = encode_canonical_json(
             unrec
-        ))
+        ).decode("UTF-8")
 
         sql = (
             "INSERT INTO events"

synapse/storage/profile.py

@@ -35,16 +35,13 @@ class ProfileStore(SQLBaseStore):
             desc="get_profile_displayname",
         )
 
-        if name:
-            name = self.database_engine.load_unicode(name)
-
         defer.returnValue(name)
 
     def set_profile_displayname(self, user_localpart, new_displayname):
         return self._simple_update_one(
             table="profiles",
             keyvalues={"user_id": user_localpart},
-            updatevalues={"displayname": new_displayname.encode("utf8")},
+            updatevalues={"displayname": new_displayname},
             desc="set_profile_displayname",
         )
 

synapse/storage/registration.py

@@ -98,11 +98,6 @@ class RegistrationStore(SQLBaseStore):
             allow_none=True,
         )
 
-        if user_info:
-            user_info["password_hash"] = self.database_engine.load_unicode(
-                user_info["password_hash"]
-            )
-
         defer.returnValue(user_info)
 
     @cached()

synapse/storage/schema/delta/15/appservice_txns.sql

@@ -22,7 +22,7 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids LONGBLOB NOT NULL,
+    event_ids TEXT NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 

synapse/storage/schema/full_schemas/11/im.sql

@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events(
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content bytea NOT NULL,
+    content TEXT NOT NULL,
-    unrecognized_keys bytea,
+    unrecognized_keys TEXT,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth BIGINT DEFAULT 0 NOT NULL,
@@ -35,8 +35,8 @@ CREATE INDEX events_room_id ON events (room_id);
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata bytea NOT NULL,
+    internal_metadata TEXT NOT NULL,
-    json bytea NOT NULL,
+    json TEXT NOT NULL,
     UNIQUE (event_id)
 );
 

synapse/storage/schema/full_schemas/16/application_services.sql

@@ -39,7 +39,7 @@ CREATE TABLE IF NOT EXISTS application_services_state(
 CREATE TABLE IF NOT EXISTS application_services_txns(
     as_id VARCHAR(150) NOT NULL,
     txn_id INTEGER NOT NULL,
-    event_ids bytea NOT NULL,
+    event_ids TEXT NOT NULL,
     UNIQUE(as_id, txn_id)
 );
 

synapse/storage/schema/full_schemas/16/im.sql

@@ -19,8 +19,8 @@ CREATE TABLE IF NOT EXISTS events(
     event_id VARCHAR(150) NOT NULL,
     type VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    content bytea NOT NULL,
+    content TEXT NOT NULL,
-    unrecognized_keys bytea,
+    unrecognized_keys TEXT,
     processed BOOL NOT NULL,
     outlier BOOL NOT NULL,
     depth BIGINT DEFAULT 0 NOT NULL,
@@ -39,8 +39,8 @@ CREATE INDEX events_order_room ON events (
 CREATE TABLE IF NOT EXISTS event_json(
     event_id VARCHAR(150) NOT NULL,
     room_id VARCHAR(150) NOT NULL,
-    internal_metadata bytea NOT NULL,
+    internal_metadata TEXT NOT NULL,
-    json bytea NOT NULL,
+    json TEXT NOT NULL,
     UNIQUE (event_id)
 );