From a3cb24475577c31fa2c16a26fccddb76daf2f6ae Mon Sep 17 00:00:00 2001 From: Erik Johnston Date: Fri, 14 Jun 2024 16:40:29 +0100 Subject: [PATCH] Automatically apply SQL for inconsistent sequence (#17305) Rather than forcing the server operator to apply the SQL manually. This should be safe, as there should be only one writer for these sequences. --- changelog.d/17305.misc | 1 + docs/postgres.md | 10 -------- synapse/storage/util/sequence.py | 37 ++++++++++++----------------- tests/storage/test_id_generators.py | 14 +++++++---- 4 files changed, 25 insertions(+), 37 deletions(-) create mode 100644 changelog.d/17305.misc diff --git a/changelog.d/17305.misc b/changelog.d/17305.misc new file mode 100644 index 000000000..cb6b9504b --- /dev/null +++ b/changelog.d/17305.misc @@ -0,0 +1 @@ +When rolling back to a previous Synapse version and then forwards again to this release, don't require server operators to manually run SQL. diff --git a/docs/postgres.md b/docs/postgres.md index 4b2ba3827..d06f0cda1 100644 --- a/docs/postgres.md +++ b/docs/postgres.md @@ -255,13 +255,3 @@ however extreme care must be taken to avoid database corruption. Note that the above may fail with an error about duplicate rows if corruption has already occurred, and such duplicate rows will need to be manually removed. - -### Fixing inconsistent sequences error - -Synapse uses Postgres sequences to generate IDs for various tables. A sequence -and associated table can get out of sync if, for example, Synapse has been -downgraded and then upgraded again. - -To fix the issue shut down Synapse (including any and all workers) and run the -SQL command included in the error message. Once done Synapse should start -successfully. diff --git a/synapse/storage/util/sequence.py b/synapse/storage/util/sequence.py index f57e7ec41..c4c0602b2 100644 --- a/synapse/storage/util/sequence.py +++ b/synapse/storage/util/sequence.py @@ -36,21 +36,6 @@ if TYPE_CHECKING: logger = logging.getLogger(__name__) -_INCONSISTENT_SEQUENCE_ERROR = """ -Postgres sequence '%(seq)s' is inconsistent with associated -table '%(table)s'. This can happen if Synapse has been downgraded and -then upgraded again, or due to a bad migration. - -To fix this error, shut down Synapse (including any and all workers) -and run the following SQL: - - SELECT setval('%(seq)s', ( - %(max_id_sql)s - )); - -See docs/postgres.md for more information. -""" - _INCONSISTENT_STREAM_ERROR = """ Postgres sequence '%(seq)s' is inconsistent with associated stream position of '%(stream_name)s' in the 'stream_positions' table. @@ -169,25 +154,33 @@ class PostgresSequenceGenerator(SequenceGenerator): if row: max_in_stream_positions = row[0] - txn.close() - # If `is_called` is False then `last_value` is actually the value that # will be generated next, so we decrement to get the true "last value". if not is_called: last_value -= 1 if max_stream_id > last_value: + # The sequence is lagging behind the tables. This is probably due to + # rolling back to a version before the sequence was used and then + # forwards again. We resolve this by setting the sequence to the + # right value. logger.warning( - "Postgres sequence %s is behind table %s: %d < %d", + "Postgres sequence %s is behind table %s: %d < %d. Updating sequence.", self._sequence_name, table, last_value, max_stream_id, ) - raise IncorrectDatabaseSetup( - _INCONSISTENT_SEQUENCE_ERROR - % {"seq": self._sequence_name, "table": table, "max_id_sql": table_sql} - ) + + sql = f""" + SELECT setval('{self._sequence_name}', GREATEST( + (SELECT last_value FROM {self._sequence_name}), + ({table_sql}) + )); + """ + txn.execute(sql) + + txn.close() # If we have values in the stream positions table then they have to be # less than or equal to `last_value` diff --git a/tests/storage/test_id_generators.py b/tests/storage/test_id_generators.py index f0307252f..9be2923e6 100644 --- a/tests/storage/test_id_generators.py +++ b/tests/storage/test_id_generators.py @@ -28,7 +28,6 @@ from synapse.storage.database import ( LoggingDatabaseConnection, LoggingTransaction, ) -from synapse.storage.engines import IncorrectDatabaseSetup from synapse.storage.types import Cursor from synapse.storage.util.id_generators import MultiWriterIdGenerator from synapse.storage.util.sequence import ( @@ -525,7 +524,7 @@ class WorkerMultiWriterIdGeneratorTestCase(MultiWriterIdGeneratorBase): self.assertEqual(id_gen_5.get_current_token_for_writer("third"), 6) def test_sequence_consistency(self) -> None: - """Test that we error out if the table and sequence diverges.""" + """Test that we correct the sequence if the table and sequence diverges.""" # Prefill with some rows self._insert_row_with_id("master", 3) @@ -536,9 +535,14 @@ class WorkerMultiWriterIdGeneratorTestCase(MultiWriterIdGeneratorBase): self.get_success(self.db_pool.runInteraction("_insert", _insert)) - # Creating the ID gen should error - with self.assertRaises(IncorrectDatabaseSetup): - self._create_id_generator("first") + # Creating the ID gen should now fix the inconsistency + id_gen = self._create_id_generator() + + async def _get_next_async() -> None: + async with id_gen.get_next() as stream_id: + self.assertEqual(stream_id, 27) + + self.get_success(_get_next_async()) def test_minimal_local_token(self) -> None: self._insert_rows("first", 3)