Merge remote-tracking branch 'origin/develop' into neilj/fix_reap_users_in_postgres

This commit is contained in:
Amber Brown 2018-08-14 20:56:23 +10:00
commit 591bf87c6a
56 changed files with 404 additions and 292 deletions

View File

@ -8,6 +8,9 @@ before_script:
- git remote set-branches --add origin develop
- git fetch origin develop
services:
- postgresql
matrix:
fast_finish: true
include:
@ -20,6 +23,9 @@ matrix:
- python: 2.7
env: TOX_ENV=py27
- python: 2.7
env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
- python: 3.6
env: TOX_ENV=py36
@ -29,6 +35,10 @@ matrix:
- python: 3.6
env: TOX_ENV=check-newsfragment
allow_failures:
- python: 2.7
env: TOX_ENV=py27-postgres TRIAL_FLAGS="-j 4"
install:
- pip install tox

1
changelog.d/3423.misc Normal file
View File

@ -0,0 +1 @@
The test suite now can run under PostgreSQL.

1
changelog.d/3669.misc Normal file
View File

@ -0,0 +1 @@
Update docker base image from alpine 3.7 to 3.8.

1
changelog.d/3677.bugfix Normal file
View File

@ -0,0 +1 @@
Dont ship recaptcha_ajax.js, use it directly from Google

1
changelog.d/3684.misc Normal file
View File

@ -0,0 +1 @@
Implemented a new testing base class to reduce test boilerplate.

View File

@ -1,4 +1,4 @@
FROM docker.io/python:2-alpine3.7
FROM docker.io/python:2-alpine3.8
RUN apk add --no-cache --virtual .nacl_deps \
build-base \

View File

@ -95,6 +95,7 @@ class PresenceHandler(object):
Args:
hs (synapse.server.HomeServer):
"""
self.hs = hs
self.is_mine = hs.is_mine
self.is_mine_id = hs.is_mine_id
self.clock = hs.get_clock()
@ -230,6 +231,10 @@ class PresenceHandler(object):
earlier than they should when synapse is restarted. This affect of this
is some spurious presence changes that will self-correct.
"""
# If the DB pool has already terminated, don't try updating
if not self.hs.get_db_pool().running:
return
logger.info(
"Performing _on_shutdown. Persisting %d unpersisted changes",
len(self.user_to_current_state)

View File

@ -4,7 +4,7 @@
<meta name='viewport' content='width=device-width, initial-scale=1, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0'>
<link rel="stylesheet" href="style.css">
<script src="js/jquery-2.1.3.min.js"></script>
<script src="js/recaptcha_ajax.js"></script>
<script src="https://www.google.com/recaptcha/api/js/recaptcha_ajax.js"></script>
<script src="register_config.js"></script>
<script src="js/register.js"></script>
</head>

File diff suppressed because one or more lines are too long

View File

@ -96,6 +96,11 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
self._batch_row_update[key] = (user_agent, device_id, now)
def _update_client_ips_batch(self):
# If the DB pool has already terminated, don't try updating
if not self.hs.get_db_pool().running:
return
def update():
to_update = self._batch_row_update
self._batch_row_update = {}

View File

@ -15,4 +15,7 @@
from twisted.trial import util
from tests import utils
util.DEFAULT_TIMEOUT_DURATION = 10
utils.setupdb()

View File

@ -39,7 +39,7 @@ class AuthTestCase(unittest.TestCase):
self.state_handler = Mock()
self.store = Mock()
self.hs = yield setup_test_homeserver(handlers=None)
self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
self.hs.get_datastore = Mock(return_value=self.store)
self.hs.handlers = TestHandlers(self.hs)
self.auth = Auth(self.hs)

View File

@ -46,7 +46,10 @@ class FilteringTestCase(unittest.TestCase):
self.mock_http_client.put_json = DeferredMockCallable()
hs = yield setup_test_homeserver(
handlers=None, http_client=self.mock_http_client, keyring=Mock()
self.addCleanup,
handlers=None,
http_client=self.mock_http_client,
keyring=Mock(),
)
self.filtering = hs.get_filtering()

View File

@ -58,12 +58,10 @@ class KeyringTestCase(unittest.TestCase):
self.mock_perspective_server = MockPerspectiveServer()
self.http_client = Mock()
self.hs = yield utils.setup_test_homeserver(
handlers=None, http_client=self.http_client
self.addCleanup, handlers=None, http_client=self.http_client
)
keys = self.mock_perspective_server.get_verify_keys()
self.hs.config.perspectives = {
self.mock_perspective_server.server_name: keys
}
self.hs.config.perspectives = {self.mock_perspective_server.server_name: keys}
def check_context(self, _, expected):
self.assertEquals(

View File

@ -35,7 +35,7 @@ class AuthHandlers(object):
class AuthTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield setup_test_homeserver(handlers=None)
self.hs = yield setup_test_homeserver(self.addCleanup, handlers=None)
self.hs.handlers = AuthHandlers(self.hs)
self.auth_handler = self.hs.handlers.auth_handler
self.macaroon_generator = self.hs.get_macaroon_generator()

View File

@ -34,7 +34,7 @@ class DeviceTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield utils.setup_test_homeserver()
hs = yield utils.setup_test_homeserver(self.addCleanup)
self.handler = hs.get_device_handler()
self.store = hs.get_datastore()
self.clock = hs.get_clock()

View File

@ -46,6 +46,7 @@ class DirectoryTestCase(unittest.TestCase):
self.mock_registry.register_query_handler = register_query_handler
hs = yield setup_test_homeserver(
self.addCleanup,
http_client=None,
resource_for_federation=Mock(),
federation_client=self.mock_federation,

View File

@ -34,7 +34,7 @@ class E2eKeysHandlerTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield utils.setup_test_homeserver(
handlers=None, federation_client=mock.Mock()
self.addCleanup, handlers=None, federation_client=mock.Mock()
)
self.handler = synapse.handlers.e2e_keys.E2eKeysHandler(self.hs)

View File

@ -48,6 +48,7 @@ class ProfileTestCase(unittest.TestCase):
self.mock_registry.register_query_handler = register_query_handler
hs = yield setup_test_homeserver(
self.addCleanup,
http_client=None,
handlers=None,
resource_for_federation=Mock(),

View File

@ -40,6 +40,7 @@ class RegistrationTestCase(unittest.TestCase):
self.mock_distributor.declare("registered_user")
self.mock_captcha_client = Mock()
self.hs = yield setup_test_homeserver(
self.addCleanup,
handlers=None,
http_client=None,
expire_access_token=True,

View File

@ -67,6 +67,7 @@ class TypingNotificationsTestCase(unittest.TestCase):
self.state_handler = Mock()
hs = yield setup_test_homeserver(
self.addCleanup,
"test",
auth=self.auth,
clock=self.clock,

View File

@ -54,6 +54,7 @@ class BaseSlavedStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield setup_test_homeserver(
self.addCleanup,
"blue",
http_client=None,
federation_client=Mock(),

View File

@ -51,7 +51,7 @@ class UserRegisterTestCase(unittest.TestCase):
self.secrets = Mock()
self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock
self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.hs.config.registration_shared_secret = u"shared"

View File

@ -41,6 +41,7 @@ class EventStreamPermissionsTestCase(RestTestCase):
self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
hs = yield setup_test_homeserver(
self.addCleanup,
http_client=None,
federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["send_message"]),

View File

@ -46,6 +46,7 @@ class ProfileTestCase(unittest.TestCase):
)
hs = yield setup_test_homeserver(
self.addCleanup,
"test",
http_client=None,
resource_for_client=self.mock_resource,

View File

@ -49,7 +49,7 @@ class CreateUserServletTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock)
self.hs = self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock
self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.hs.get_datastore = Mock(return_value=self.datastore)
self.hs.get_handlers = Mock(return_value=handlers)

View File

@ -50,6 +50,7 @@ class RoomBase(unittest.TestCase):
self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver(
self.addCleanup,
"red",
http_client=None,
clock=self.hs_clock,

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2018 New Vector
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -17,40 +18,32 @@
from mock import Mock, NonCallableMock
# twisted imports
from twisted.internet import defer
import synapse.rest.client.v1.room
from synapse.rest.client.v1 import room
from synapse.types import UserID
from ....utils import MockClock, MockHttpResource, setup_test_homeserver
from .utils import RestTestCase
from tests import unittest
PATH_PREFIX = "/_matrix/client/api/v1"
class RoomTypingTestCase(RestTestCase):
class RoomTypingTestCase(unittest.HomeserverTestCase):
""" Tests /rooms/$room_id/typing/$user_id REST API. """
user_id = "@sid:red"
user = UserID.from_string(user_id)
servlets = [room.register_servlets]
@defer.inlineCallbacks
def setUp(self):
self.clock = MockClock()
def make_homeserver(self, reactor, clock):
self.mock_resource = MockHttpResource(prefix=PATH_PREFIX)
self.auth_user_id = self.user_id
hs = yield setup_test_homeserver(
hs = self.setup_test_homeserver(
"red",
clock=self.clock,
http_client=None,
federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["send_message"]),
)
self.hs = hs
self.event_source = hs.get_event_sources().sources["typing"]
@ -99,25 +92,24 @@ class RoomTypingTestCase(RestTestCase):
fetch_room_distributions_into
)
synapse.rest.client.v1.room.register_servlets(hs, self.mock_resource)
return hs
self.room_id = yield self.create_room_as(self.user_id)
def prepare(self, reactor, clock, hs):
self.room_id = self.helper.create_room_as(self.user_id)
# Need another user to make notifications actually work
yield self.join(self.room_id, user="@jim:red")
self.helper.join(self.room_id, user="@jim:red")
@defer.inlineCallbacks
def test_set_typing(self):
(code, _) = yield self.mock_resource.trigger(
request, channel = self.make_request(
"PUT",
"/rooms/%s/typing/%s" % (self.room_id, self.user_id),
'{"typing": true, "timeout": 30000}',
b'{"typing": true, "timeout": 30000}',
)
self.assertEquals(200, code)
self.render(request)
self.assertEquals(200, channel.code)
self.assertEquals(self.event_source.get_current_key(), 1)
events = yield self.event_source.get_new_events(
from_key=0, room_ids=[self.room_id]
)
events = self.event_source.get_new_events(from_key=0, room_ids=[self.room_id])
self.assertEquals(
events[0],
[
@ -129,35 +121,36 @@ class RoomTypingTestCase(RestTestCase):
],
)
@defer.inlineCallbacks
def test_set_not_typing(self):
(code, _) = yield self.mock_resource.trigger(
request, channel = self.make_request(
"PUT",
"/rooms/%s/typing/%s" % (self.room_id, self.user_id),
'{"typing": false}',
b'{"typing": false}',
)
self.assertEquals(200, code)
self.render(request)
self.assertEquals(200, channel.code)
@defer.inlineCallbacks
def test_typing_timeout(self):
(code, _) = yield self.mock_resource.trigger(
request, channel = self.make_request(
"PUT",
"/rooms/%s/typing/%s" % (self.room_id, self.user_id),
'{"typing": true, "timeout": 30000}',
b'{"typing": true, "timeout": 30000}',
)
self.assertEquals(200, code)
self.render(request)
self.assertEquals(200, channel.code)
self.assertEquals(self.event_source.get_current_key(), 1)
self.clock.advance_time(36)
self.reactor.advance(36)
self.assertEquals(self.event_source.get_current_key(), 2)
(code, _) = yield self.mock_resource.trigger(
request, channel = self.make_request(
"PUT",
"/rooms/%s/typing/%s" % (self.room_id, self.user_id),
'{"typing": true, "timeout": 30000}',
b'{"typing": true, "timeout": 30000}',
)
self.assertEquals(200, code)
self.render(request)
self.assertEquals(200, channel.code)
self.assertEquals(self.event_source.get_current_key(), 3)

View File

@ -43,7 +43,7 @@ class FilterTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock
self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.auth = self.hs.get_auth()

View File

@ -47,7 +47,7 @@ class RegisterRestServletTestCase(unittest.TestCase):
login_handler=self.login_handler,
)
self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock
self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.hs.get_auth = Mock(return_value=self.auth)
self.hs.get_handlers = Mock(return_value=self.handlers)

View File

@ -40,7 +40,7 @@ class FilterTestCase(unittest.TestCase):
self.hs_clock = Clock(self.clock)
self.hs = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.clock
self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.clock
)
self.auth = self.hs.get_auth()

View File

@ -147,12 +147,15 @@ class ThreadedMemoryReactorClock(MemoryReactorClock):
return d
def setup_test_homeserver(*args, **kwargs):
def setup_test_homeserver(cleanup_func, *args, **kwargs):
"""
Set up a synchronous test server, driven by the reactor used by
the homeserver.
"""
d = _sth(*args, **kwargs).result
d = _sth(cleanup_func, *args, **kwargs).result
if isinstance(d, Failure):
d.raiseException()
# Make the thread pool synchronous.
clock = d.get_clock()
@ -189,6 +192,9 @@ def setup_test_homeserver(*args, **kwargs):
def start(self):
pass
def stop(self):
pass
def callInThreadWithCallback(self, onResult, function, *args, **kwargs):
def _(res):
if isinstance(res, Failure):

View File

@ -43,7 +43,10 @@ class ApplicationServiceStoreTestCase(unittest.TestCase):
password_providers=[],
)
hs = yield setup_test_homeserver(
config=config, federation_sender=Mock(), federation_client=Mock()
self.addCleanup,
config=config,
federation_sender=Mock(),
federation_client=Mock(),
)
self.as_token = "token1"
@ -108,7 +111,10 @@ class ApplicationServiceTransactionStoreTestCase(unittest.TestCase):
password_providers=[],
)
hs = yield setup_test_homeserver(
config=config, federation_sender=Mock(), federation_client=Mock()
self.addCleanup,
config=config,
federation_sender=Mock(),
federation_client=Mock(),
)
self.db_pool = hs.get_db_pool()
@ -392,6 +398,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
)
hs = yield setup_test_homeserver(
self.addCleanup,
config=config,
datastore=Mock(),
federation_sender=Mock(),
@ -409,6 +416,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
)
hs = yield setup_test_homeserver(
self.addCleanup,
config=config,
datastore=Mock(),
federation_sender=Mock(),
@ -432,6 +440,7 @@ class ApplicationServiceStoreConfigTestCase(unittest.TestCase):
app_service_config_files=[f1, f2], event_cache_size=1, password_providers=[]
)
hs = yield setup_test_homeserver(
self.addCleanup,
config=config,
datastore=Mock(),
federation_sender=Mock(),

View File

@ -9,7 +9,9 @@ from tests.utils import setup_test_homeserver
class BackgroundUpdateTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver() # type: synapse.server.HomeServer
hs = yield setup_test_homeserver(
self.addCleanup
) # type: synapse.server.HomeServer
self.store = hs.get_datastore()
self.clock = hs.get_clock()

View File

@ -28,7 +28,7 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield tests.utils.setup_test_homeserver()
self.hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = self.hs.get_datastore()
self.clock = self.hs.get_clock()

View File

@ -28,7 +28,7 @@ class DeviceStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield tests.utils.setup_test_homeserver()
hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()

View File

@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class DirectoryStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver()
hs = yield setup_test_homeserver(self.addCleanup)
self.store = DirectoryStore(None, hs)

View File

@ -26,8 +26,7 @@ class EndToEndKeyStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield tests.utils.setup_test_homeserver()
hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
@defer.inlineCallbacks

View File

@ -22,7 +22,7 @@ import tests.utils
class EventFederationWorkerStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield tests.utils.setup_test_homeserver()
hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
@defer.inlineCallbacks

View File

@ -33,7 +33,7 @@ HIGHLIGHT = [
class EventPushActionsStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield tests.utils.setup_test_homeserver()
hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
@defer.inlineCallbacks

View File

@ -28,7 +28,7 @@ class KeyStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield tests.utils.setup_test_homeserver()
hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
@defer.inlineCallbacks

View File

@ -28,7 +28,7 @@ class MonthlyActiveUsersTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield setup_test_homeserver()
self.hs = yield setup_test_homeserver(self.addCleanup)
self.store = self.hs.get_datastore()
@defer.inlineCallbacks

View File

@ -26,7 +26,7 @@ from tests.utils import MockClock, setup_test_homeserver
class PresenceStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver(clock=MockClock())
hs = yield setup_test_homeserver(self.addCleanup, clock=MockClock())
self.store = PresenceStore(None, hs)

View File

@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class ProfileStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver()
hs = yield setup_test_homeserver(self.addCleanup)
self.store = ProfileStore(None, hs)

View File

@ -29,7 +29,7 @@ class RedactionTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver(
resource_for_federation=Mock(), http_client=None
self.addCleanup, resource_for_federation=Mock(), http_client=None
)
self.store = hs.get_datastore()

View File

@ -23,7 +23,7 @@ from tests.utils import setup_test_homeserver
class RegistrationStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver()
hs = yield setup_test_homeserver(self.addCleanup)
self.db_pool = hs.get_db_pool()
self.store = hs.get_datastore()

View File

@ -26,7 +26,7 @@ from tests.utils import setup_test_homeserver
class RoomStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver()
hs = yield setup_test_homeserver(self.addCleanup)
# We can't test RoomStore on its own without the DirectoryStore, for
# management of the 'room_aliases' table
@ -57,7 +57,7 @@ class RoomStoreTestCase(unittest.TestCase):
class RoomEventsStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = setup_test_homeserver()
hs = setup_test_homeserver(self.addCleanup)
# Room events need the full datastore, for persist_event() and
# get_room_state()

View File

@ -29,7 +29,7 @@ class RoomMemberStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield setup_test_homeserver(
resource_for_federation=Mock(), http_client=None
self.addCleanup, resource_for_federation=Mock(), http_client=None
)
# We can't test the RoomMemberStore on its own without the other event
# storage logic

View File

@ -33,7 +33,7 @@ class StateStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
hs = yield tests.utils.setup_test_homeserver()
hs = yield tests.utils.setup_test_homeserver(self.addCleanup)
self.store = hs.get_datastore()
self.event_builder_factory = hs.get_event_builder_factory()

View File

@ -29,7 +29,7 @@ BOBBY = "@bobby:a"
class UserDirectoryStoreTestCase(unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield setup_test_homeserver()
self.hs = yield setup_test_homeserver(self.addCleanup)
self.store = UserDirectoryStore(None, self.hs)
# alice and bob are both in !room_id. bobby is not but shares

View File

@ -18,7 +18,10 @@ class MessageAcceptTests(unittest.TestCase):
self.reactor = ThreadedMemoryReactorClock()
self.hs_clock = Clock(self.reactor)
self.homeserver = setup_test_homeserver(
http_client=self.http_client, clock=self.hs_clock, reactor=self.reactor
self.addCleanup,
http_client=self.http_client,
clock=self.hs_clock,
reactor=self.reactor,
)
user_id = UserID("us", "test")

View File

@ -16,7 +16,7 @@ class JsonResourceTests(unittest.TestCase):
self.reactor = MemoryReactorClock()
self.hs_clock = Clock(self.reactor)
self.homeserver = setup_test_homeserver(
http_client=None, clock=self.hs_clock, reactor=self.reactor
self.addCleanup, http_client=None, clock=self.hs_clock, reactor=self.reactor
)
def test_handler_for_request(self):

View File

@ -31,7 +31,7 @@ TEST_ROOM_ID = "!TEST:ROOM"
class FilterEventsForServerTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks
def setUp(self):
self.hs = yield setup_test_homeserver()
self.hs = yield setup_test_homeserver(self.addCleanup)
self.event_creation_handler = self.hs.get_event_creation_handler()
self.event_builder_factory = self.hs.get_event_builder_factory()
self.store = self.hs.get_datastore()

View File

@ -1,5 +1,6 @@
# -*- coding: utf-8 -*-
# Copyright 2014-2016 OpenMarket Ltd
# Copyright 2018 New Vector
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -15,12 +16,19 @@
import logging
from mock import Mock
import twisted
import twisted.logger
from twisted.trial import unittest
from synapse.http.server import JsonResource
from synapse.server import HomeServer
from synapse.types import UserID, create_requester
from synapse.util.logcontext import LoggingContextFilter
from tests.server import get_clock, make_request, render, setup_test_homeserver
# Set up putting Synapse's logs into Trial's.
rootLogger = logging.getLogger()
@ -129,3 +137,139 @@ def DEBUG(target):
Can apply to either a TestCase or an individual test method."""
target.loglevel = logging.DEBUG
return target
class HomeserverTestCase(TestCase):
"""
A base TestCase that reduces boilerplate for HomeServer-using test cases.
Attributes:
servlets (list[function]): List of servlet registration function.
user_id (str): The user ID to assume if auth is hijacked.
hijack_auth (bool): Whether to hijack auth to return the user specified
in user_id.
"""
servlets = []
hijack_auth = True
def setUp(self):
"""
Set up the TestCase by calling the homeserver constructor, optionally
hijacking the authentication system to return a fixed user, and then
calling the prepare function.
"""
self.reactor, self.clock = get_clock()
self._hs_args = {"clock": self.clock, "reactor": self.reactor}
self.hs = self.make_homeserver(self.reactor, self.clock)
if self.hs is None:
raise Exception("No homeserver returned from make_homeserver.")
if not isinstance(self.hs, HomeServer):
raise Exception("A homeserver wasn't returned, but %r" % (self.hs,))
# Register the resources
self.resource = JsonResource(self.hs)
for servlet in self.servlets:
servlet(self.hs, self.resource)
if hasattr(self, "user_id"):
from tests.rest.client.v1.utils import RestHelper
self.helper = RestHelper(self.hs, self.resource, self.user_id)
if self.hijack_auth:
def get_user_by_access_token(token=None, allow_guest=False):
return {
"user": UserID.from_string(self.helper.auth_user_id),
"token_id": 1,
"is_guest": False,
}
def get_user_by_req(request, allow_guest=False, rights="access"):
return create_requester(
UserID.from_string(self.helper.auth_user_id), 1, False, None
)
self.hs.get_auth().get_user_by_req = get_user_by_req
self.hs.get_auth().get_user_by_access_token = get_user_by_access_token
self.hs.get_auth().get_access_token_from_request = Mock(
return_value="1234"
)
if hasattr(self, "prepare"):
self.prepare(self.reactor, self.clock, self.hs)
def make_homeserver(self, reactor, clock):
"""
Make and return a homeserver.
Args:
reactor: A Twisted Reactor, or something that pretends to be one.
clock (synapse.util.Clock): The Clock, associated with the reactor.
Returns:
A homeserver (synapse.server.HomeServer) suitable for testing.
Function to be overridden in subclasses.
"""
raise NotImplementedError()
def prepare(self, reactor, clock, homeserver):
"""
Prepare for the test. This involves things like mocking out parts of
the homeserver, or building test data common across the whole test
suite.
Args:
reactor: A Twisted Reactor, or something that pretends to be one.
clock (synapse.util.Clock): The Clock, associated with the reactor.
homeserver (synapse.server.HomeServer): The HomeServer to test
against.
Function to optionally be overridden in subclasses.
"""
def make_request(self, method, path, content=b""):
"""
Create a SynapseRequest at the path using the method and containing the
given content.
Args:
method (bytes/unicode): The HTTP request method ("verb").
path (bytes/unicode): The HTTP path, suitably URL encoded (e.g.
escaped UTF-8 & spaces and such).
content (bytes): The body of the request.
Returns:
A synapse.http.site.SynapseRequest.
"""
return make_request(method, path, content)
def render(self, request):
"""
Render a request against the resources registered by the test class's
servlets.
Args:
request (synapse.http.site.SynapseRequest): The request to render.
"""
render(request, self.resource, self.reactor)
def setup_test_homeserver(self, *args, **kwargs):
"""
Set up the test homeserver, meant to be called by the overridable
make_homeserver. It automatically passes through the test class's
clock & reactor.
Args:
See tests.utils.setup_test_homeserver.
Returns:
synapse.server.HomeServer
"""
kwargs = dict(kwargs)
kwargs.update(self._hs_args)
return setup_test_homeserver(self.addCleanup, *args, **kwargs)

View File

@ -13,7 +13,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import atexit
import hashlib
import os
import uuid
from inspect import getcallargs
from mock import Mock, patch
@ -27,23 +30,80 @@ from synapse.http.server import HttpServer
from synapse.server import HomeServer
from synapse.storage import PostgresEngine
from synapse.storage.engines import create_engine
from synapse.storage.prepare_database import prepare_database
from synapse.storage.prepare_database import (
_get_or_create_schema_state,
_setup_new_database,
prepare_database,
)
from synapse.util.logcontext import LoggingContext
from synapse.util.ratelimitutils import FederationRateLimiter
# set this to True to run the tests against postgres instead of sqlite.
# It requires you to have a local postgres database called synapse_test, within
# which ALL TABLES WILL BE DROPPED
USE_POSTGRES_FOR_TESTS = False
USE_POSTGRES_FOR_TESTS = os.environ.get("SYNAPSE_POSTGRES", False)
POSTGRES_USER = os.environ.get("SYNAPSE_POSTGRES_USER", "postgres")
POSTGRES_BASE_DB = "_synapse_unit_tests_base_%s" % (os.getpid(),)
def setupdb():
# If we're using PostgreSQL, set up the db once
if USE_POSTGRES_FOR_TESTS:
pgconfig = {
"name": "psycopg2",
"args": {
"database": POSTGRES_BASE_DB,
"user": POSTGRES_USER,
"cp_min": 1,
"cp_max": 5,
},
}
config = Mock()
config.password_providers = []
config.database_config = pgconfig
db_engine = create_engine(pgconfig)
db_conn = db_engine.module.connect(user=POSTGRES_USER)
db_conn.autocommit = True
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
cur.execute("CREATE DATABASE %s;" % (POSTGRES_BASE_DB,))
cur.close()
db_conn.close()
# Set up in the db
db_conn = db_engine.module.connect(
database=POSTGRES_BASE_DB, user=POSTGRES_USER
)
cur = db_conn.cursor()
_get_or_create_schema_state(cur, db_engine)
_setup_new_database(cur, db_engine)
db_conn.commit()
cur.close()
db_conn.close()
def _cleanup():
db_conn = db_engine.module.connect(user=POSTGRES_USER)
db_conn.autocommit = True
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (POSTGRES_BASE_DB,))
cur.close()
db_conn.close()
atexit.register(_cleanup)
@defer.inlineCallbacks
def setup_test_homeserver(
name="test", datastore=None, config=None, reactor=None, **kargs
cleanup_func, name="test", datastore=None, config=None, reactor=None, **kargs
):
"""Setup a homeserver suitable for running tests against. Keyword arguments
are passed to the Homeserver constructor. If no datastore is supplied a
datastore backed by an in-memory sqlite db will be given to the HS.
"""
Setup a homeserver suitable for running tests against. Keyword arguments
are passed to the Homeserver constructor.
If no datastore is supplied, one is created and given to the homeserver.
Args:
cleanup_func : The function used to register a cleanup routine for
after the test.
"""
if reactor is None:
from twisted.internet import reactor
@ -95,9 +155,11 @@ def setup_test_homeserver(
kargs["clock"] = MockClock()
if USE_POSTGRES_FOR_TESTS:
test_db = "synapse_test_%s" % uuid.uuid4().hex
config.database_config = {
"name": "psycopg2",
"args": {"database": "synapse_test", "cp_min": 1, "cp_max": 5},
"args": {"database": test_db, "cp_min": 1, "cp_max": 5},
}
else:
config.database_config = {
@ -107,6 +169,21 @@ def setup_test_homeserver(
db_engine = create_engine(config.database_config)
# Create the database before we actually try and connect to it, based off
# the template database we generate in setupdb()
if datastore is None and isinstance(db_engine, PostgresEngine):
db_conn = db_engine.module.connect(
database=POSTGRES_BASE_DB, user=POSTGRES_USER
)
db_conn.autocommit = True
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
cur.execute(
"CREATE DATABASE %s WITH TEMPLATE %s;" % (test_db, POSTGRES_BASE_DB)
)
cur.close()
db_conn.close()
# we need to configure the connection pool to run the on_new_connection
# function, so that we can test code that uses custom sqlite functions
# (like rank).
@ -125,15 +202,35 @@ def setup_test_homeserver(
reactor=reactor,
**kargs
)
db_conn = hs.get_db_conn()
# make sure that the database is empty
if isinstance(db_engine, PostgresEngine):
cur = db_conn.cursor()
cur.execute("SELECT tablename FROM pg_tables where schemaname='public'")
rows = cur.fetchall()
for r in rows:
cur.execute("DROP TABLE %s CASCADE" % r[0])
yield prepare_database(db_conn, db_engine, config)
# Prepare the DB on SQLite -- PostgreSQL is a copy of an already up to
# date db
if not isinstance(db_engine, PostgresEngine):
db_conn = hs.get_db_conn()
yield prepare_database(db_conn, db_engine, config)
db_conn.commit()
db_conn.close()
else:
# We need to do cleanup on PostgreSQL
def cleanup():
# Close all the db pools
hs.get_db_pool().close()
# Drop the test database
db_conn = db_engine.module.connect(
database=POSTGRES_BASE_DB, user=POSTGRES_USER
)
db_conn.autocommit = True
cur = db_conn.cursor()
cur.execute("DROP DATABASE IF EXISTS %s;" % (test_db,))
db_conn.commit()
cur.close()
db_conn.close()
# Register the cleanup hook
cleanup_func(cleanup)
hs.setup()
else:
hs = HomeServer(

20
tox.ini
View File

@ -1,7 +1,7 @@
[tox]
envlist = packaging, py27, py36, pep8, check_isort
[testenv]
[base]
deps =
coverage
Twisted>=15.1
@ -15,6 +15,15 @@ deps =
setenv =
PYTHONDONTWRITEBYTECODE = no_byte_code
[testenv]
deps =
{[base]deps}
setenv =
{[base]setenv}
passenv = *
commands =
/usr/bin/find "{toxinidir}" -name '*.pyc' -delete
coverage run {env:COVERAGE_OPTS:} --source="{toxinidir}/synapse" \
@ -46,6 +55,15 @@ commands =
# )
usedevelop=true
[testenv:py27-postgres]
usedevelop=true
deps =
{[base]deps}
psycopg2
setenv =
{[base]setenv}
SYNAPSE_POSTGRES = 1
[testenv:py36]
usedevelop=true
commands =