Matrix-Mirrors/anonymousland-synapse
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse.git synced 2025-01-14 01:49:24 -05:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2611 from matrix-org/dbkr/port_script_drop_nuls

Browse Source 
Make the port script drop NUL values in all tables
This commit is contained in:
Richard van der Hoff 2017-10-31 17:16:08 +00:00 committed by GitHub
commit e025aec028
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 19 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
scripts/synapse_port_db
View File

@ -320,7 +320,7 @@ class Porter(object):
backward_chunk = min(row[0] for row in brows) - 1 backward_chunk = min(row[0] for row in brows) - 1
rows = frows + brows rows = frows + brows
self._convert_rows(table, headers, rows) rows = self._convert_rows(table, headers, rows)
def insert(txn): def insert(txn):
self.postgres_store.insert_many_txn( self.postgres_store.insert_many_txn(
@ -556,17 +556,29 @@ class Porter(object):
i for i, h in enumerate(headers) if h in bool_col_names i for i, h in enumerate(headers) if h in bool_col_names
] ]
class BadValueException(Exception):
pass
def conv(j, col): def conv(j, col):
if j in bool_cols: if j in bool_cols:
return bool(col) return bool(col)
elif isinstance(col, basestring) and "\0" in col:
logger.warn("DROPPING ROW: NUL value in table %s col %s: %r", table, headers[j], col)
raise BadValueException();
return col return col
outrows = []
for i, row in enumerate(rows): for i, row in enumerate(rows):
rows[i] = tuple( try:
outrows.append(tuple(
conv(j, col) conv(j, col)
for j, col in enumerate(row) for j, col in enumerate(row)
if j > 0 if j > 0
) ))
except BadValueException:
pass
return outrows
@defer.inlineCallbacks @defer.inlineCallbacks
def _setup_sent_transactions(self): def _setup_sent_transactions(self):
@ -594,7 +606,7 @@ class Porter(object):
"select", r, "select", r,
) )
self._convert_rows("sent_transactions", headers, rows) rows = self._convert_rows("sent_transactions", headers, rows)
inserted_rows = len(rows) inserted_rows = len(rows)
if inserted_rows: if inserted_rows: