mirror of
https://git.anonymousland.org/anonymousland/synapse-product.git
synced 2024-09-08 05:31:57 +00:00
Fix UPSERTs on SQLite 3.24+ (#4477)
This commit is contained in:
parent
88f4df85ca
commit
7072fe3084
@ -1 +1 @@
|
|||||||
Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+.
|
Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+.
|
||||||
|
@ -1 +1 @@
|
|||||||
Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+.
|
Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+.
|
||||||
|
1
changelog.d/4477.misc
Normal file
1
changelog.d/4477.misc
Normal file
@ -0,0 +1 @@
|
|||||||
|
Synapse will now take advantage of native UPSERT functionality in PostgreSQL 9.5+ and SQLite 3.24+.
|
@ -27,7 +27,7 @@ from twisted.internet import defer
|
|||||||
|
|
||||||
from synapse.api.errors import StoreError
|
from synapse.api.errors import StoreError
|
||||||
from synapse.metrics.background_process_metrics import run_as_background_process
|
from synapse.metrics.background_process_metrics import run_as_background_process
|
||||||
from synapse.storage.engines import PostgresEngine
|
from synapse.storage.engines import PostgresEngine, Sqlite3Engine
|
||||||
from synapse.util.caches.descriptors import Cache
|
from synapse.util.caches.descriptors import Cache
|
||||||
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
|
from synapse.util.logcontext import LoggingContext, PreserveLoggingContext
|
||||||
from synapse.util.stringutils import exception_to_unicode
|
from synapse.util.stringutils import exception_to_unicode
|
||||||
@ -196,6 +196,12 @@ class SQLBaseStore(object):
|
|||||||
# A set of tables that are not safe to use native upserts in.
|
# A set of tables that are not safe to use native upserts in.
|
||||||
self._unsafe_to_upsert_tables = {"user_ips"}
|
self._unsafe_to_upsert_tables = {"user_ips"}
|
||||||
|
|
||||||
|
# We add the user_directory_search table to the blacklist on SQLite
|
||||||
|
# because the existing search table does not have an index, making it
|
||||||
|
# unsafe to use native upserts.
|
||||||
|
if isinstance(self.database_engine, Sqlite3Engine):
|
||||||
|
self._unsafe_to_upsert_tables.add("user_directory_search")
|
||||||
|
|
||||||
if self.database_engine.can_native_upsert:
|
if self.database_engine.can_native_upsert:
|
||||||
# Check ASAP (and then later, every 1s) to see if we have finished
|
# Check ASAP (and then later, every 1s) to see if we have finished
|
||||||
# background updates of tables that aren't safe to update.
|
# background updates of tables that aren't safe to update.
|
||||||
@ -230,7 +236,7 @@ class SQLBaseStore(object):
|
|||||||
self._unsafe_to_upsert_tables.discard("user_ips")
|
self._unsafe_to_upsert_tables.discard("user_ips")
|
||||||
|
|
||||||
# If there's any tables left to check, reschedule to run.
|
# If there's any tables left to check, reschedule to run.
|
||||||
if self._unsafe_to_upsert_tables:
|
if self.updates:
|
||||||
self._clock.call_later(
|
self._clock.call_later(
|
||||||
15.0,
|
15.0,
|
||||||
run_as_background_process,
|
run_as_background_process,
|
||||||
|
@ -33,14 +33,10 @@ class Sqlite3Engine(object):
|
|||||||
@property
|
@property
|
||||||
def can_native_upsert(self):
|
def can_native_upsert(self):
|
||||||
"""
|
"""
|
||||||
Do we support native UPSERTs?
|
Do we support native UPSERTs? This requires SQLite3 3.24+, plus some
|
||||||
|
more work we haven't done yet to tell what was inserted vs updated.
|
||||||
"""
|
"""
|
||||||
# SQLite3 3.24+ supports them, but empirically the unit tests don't work
|
return self.module.sqlite_version_info >= (3, 24, 0)
|
||||||
# when its enabled.
|
|
||||||
# FIXME: Figure out what is wrong so we can re-enable native upserts
|
|
||||||
|
|
||||||
# return self.module.sqlite_version_info >= (3, 24, 0)
|
|
||||||
return False
|
|
||||||
|
|
||||||
def check_database(self, txn):
|
def check_database(self, txn):
|
||||||
pass
|
pass
|
||||||
|
@ -197,15 +197,21 @@ class MonthlyActiveUsersStore(SQLBaseStore):
|
|||||||
if is_support:
|
if is_support:
|
||||||
return
|
return
|
||||||
|
|
||||||
is_insert = yield self.runInteraction(
|
yield self.runInteraction(
|
||||||
"upsert_monthly_active_user", self.upsert_monthly_active_user_txn,
|
"upsert_monthly_active_user", self.upsert_monthly_active_user_txn,
|
||||||
user_id
|
user_id
|
||||||
)
|
)
|
||||||
|
|
||||||
if is_insert:
|
user_in_mau = self.user_last_seen_monthly_active.cache.get(
|
||||||
self.user_last_seen_monthly_active.invalidate((user_id,))
|
(user_id,),
|
||||||
|
None,
|
||||||
|
update_metrics=False
|
||||||
|
)
|
||||||
|
if user_in_mau is None:
|
||||||
self.get_monthly_active_count.invalidate(())
|
self.get_monthly_active_count.invalidate(())
|
||||||
|
|
||||||
|
self.user_last_seen_monthly_active.invalidate((user_id,))
|
||||||
|
|
||||||
def upsert_monthly_active_user_txn(self, txn, user_id):
|
def upsert_monthly_active_user_txn(self, txn, user_id):
|
||||||
"""Updates or inserts monthly active user member
|
"""Updates or inserts monthly active user member
|
||||||
|
|
||||||
|
@ -49,14 +49,17 @@ class SQLBaseStoreTestCase(unittest.TestCase):
|
|||||||
self.db_pool.runWithConnection = runWithConnection
|
self.db_pool.runWithConnection = runWithConnection
|
||||||
|
|
||||||
config = Mock()
|
config = Mock()
|
||||||
config._enable_native_upserts = False
|
config._disable_native_upserts = True
|
||||||
config.event_cache_size = 1
|
config.event_cache_size = 1
|
||||||
config.database_config = {"name": "sqlite3"}
|
config.database_config = {"name": "sqlite3"}
|
||||||
|
engine = create_engine(config.database_config)
|
||||||
|
fake_engine = Mock(wraps=engine)
|
||||||
|
fake_engine.can_native_upsert = False
|
||||||
hs = TestHomeServer(
|
hs = TestHomeServer(
|
||||||
"test",
|
"test",
|
||||||
db_pool=self.db_pool,
|
db_pool=self.db_pool,
|
||||||
config=config,
|
config=config,
|
||||||
database_engine=create_engine(config.database_config),
|
database_engine=fake_engine,
|
||||||
)
|
)
|
||||||
|
|
||||||
self.datastore = SQLBaseStore(None, hs)
|
self.datastore = SQLBaseStore(None, hs)
|
||||||
|
@ -18,12 +18,12 @@ from twisted.internet import defer
|
|||||||
|
|
||||||
from synapse.api.constants import UserTypes
|
from synapse.api.constants import UserTypes
|
||||||
|
|
||||||
from tests.unittest import HomeserverTestCase
|
from tests import unittest
|
||||||
|
|
||||||
FORTY_DAYS = 40 * 24 * 60 * 60
|
FORTY_DAYS = 40 * 24 * 60 * 60
|
||||||
|
|
||||||
|
|
||||||
class MonthlyActiveUsersTestCase(HomeserverTestCase):
|
class MonthlyActiveUsersTestCase(unittest.HomeserverTestCase):
|
||||||
def make_homeserver(self, reactor, clock):
|
def make_homeserver(self, reactor, clock):
|
||||||
|
|
||||||
hs = self.setup_test_homeserver()
|
hs = self.setup_test_homeserver()
|
||||||
|
Loading…
Reference in New Issue
Block a user