Matrix-Mirrors/anonymousland-synapse
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse.git synced 2024-12-25 06:59:22 -05:00
Code Issues Packages Projects Releases Wiki Activity

Make the port script drop NUL values in all tables

Browse Source 
Postgres doesn't support NULs in strings so it makes the script
throw an exception and stop if any values contain \0. Drop them
with appropriate warning.
This commit is contained in:
David Baker 2017-10-31 16:58:49 +00:00
parent c31a7c3ff6
commit 9d419f48e6
1 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
scripts/synapse_port_db
View File

@ -320,7 +320,7 @@ class Porter(object):
backward_chunk = min(row[0] for row in brows) - 1 backward_chunk = min(row[0] for row in brows) - 1
rows = frows + brows rows = frows + brows
self._convert_rows(table, headers, rows) rows = self._convert_rows(table, headers, rows)
def insert(txn): def insert(txn):
self.postgres_store.insert_many_txn( self.postgres_store.insert_many_txn(
@ -556,17 +556,29 @@ class Porter(object):
i for i, h in enumerate(headers) if h in bool_col_names i for i, h in enumerate(headers) if h in bool_col_names
] ]
class BadValueException(Exception):
pass
def conv(j, col): def conv(j, col):
if j in bool_cols: if j in bool_cols:
return bool(col) return bool(col)
elif isinstance(col, basestring) and "\0" in col:
logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
raise BadValueException();
return col return col
outrows = []
for i, row in enumerate(rows): for i, row in enumerate(rows):
rows[i] = tuple( try:
conv(j, col) outrows.append(tuple(
for j, col in enumerate(row) conv(j, col)
if j > 0 for j, col in enumerate(row)
) if j > 0
))
except BadValueException:
pass
return outrows
@defer.inlineCallbacks @defer.inlineCallbacks
def _setup_sent_transactions(self): def _setup_sent_transactions(self):
@ -594,7 +606,7 @@ class Porter(object):
"select", r, "select", r,
) )
self._convert_rows("sent_transactions", headers, rows) rows = self._convert_rows("sent_transactions", headers, rows)
inserted_rows = len(rows) inserted_rows = len(rows)
if inserted_rows: if inserted_rows: