Matrix-Mirrors/anonymousland-synapse
1
0
Fork 0
mirror of https://git.anonymousland.org/anonymousland/synapse.git synced 2025-12-11 00:11:10 -05:00
Code Issues Projects Releases Packages Wiki Activity

Performance improvements and refactor of Ratelimiter (#7595)

Browse source 
While working on https://github.com/matrix-org/synapse/issues/5665 I found myself digging into the `Ratelimiter` class and seeing that it was both:

* Rather undocumented, and
* causing a *lot* of config checks

This PR attempts to refactor and comment the `Ratelimiter` class, as well as encourage config file accesses to only be done at instantiation. 

Best to be reviewed commit-by-commit.
This commit is contained in:
Andrew Morgan 2020-06-05 10:47:20 +01:00 • committed by GitHub
parent c389bfb6ea
commit f4e6495b5d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 325 additions and 233 deletions
Download patch file Download diff file Expand all files Collapse all files

100
tests/api/test_ratelimiting.py
View file

@ -1,39 +1,97 @@
from synapse.api.ratelimiting import Ratelimiter
from synapse.api.ratelimiting import LimitExceededError, Ratelimiter
from tests import unittest
class TestRatelimiter(unittest.TestCase):
def test_allowed(self):
limiter = Ratelimiter()
allowed, time_allowed = limiter.can_do_action(
key="test_id", time_now_s=0, rate_hz=0.1, burst_count=1
)
def test_allowed_via_can_do_action(self):
limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
allowed, time_allowed = limiter.can_do_action(key="test_id", _time_now_s=0)
self.assertTrue(allowed)
self.assertEquals(10.0, time_allowed)
allowed, time_allowed = limiter.can_do_action(
key="test_id", time_now_s=5, rate_hz=0.1, burst_count=1
)
allowed, time_allowed = limiter.can_do_action(key="test_id", _time_now_s=5)
self.assertFalse(allowed)
self.assertEquals(10.0, time_allowed)
allowed, time_allowed = limiter.can_do_action(
key="test_id", time_now_s=10, rate_hz=0.1, burst_count=1
)
allowed, time_allowed = limiter.can_do_action(key="test_id", _time_now_s=10)
self.assertTrue(allowed)
self.assertEquals(20.0, time_allowed)
def test_allowed_via_ratelimit(self):
limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
# Shouldn't raise
limiter.ratelimit(key="test_id", _time_now_s=0)
# Should raise
with self.assertRaises(LimitExceededError) as context:
limiter.ratelimit(key="test_id", _time_now_s=5)
self.assertEqual(context.exception.retry_after_ms, 5000)
# Shouldn't raise
limiter.ratelimit(key="test_id", _time_now_s=10)
def test_allowed_via_can_do_action_and_overriding_parameters(self):
"""Test that we can override options of can_do_action that would otherwise fail
an action
"""
# Create a Ratelimiter with a very low allowed rate_hz and burst_count
limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
# First attempt should be allowed
allowed, time_allowed = limiter.can_do_action(("test_id",), _time_now_s=0,)
self.assertTrue(allowed)
self.assertEqual(10.0, time_allowed)
# Second attempt, 1s later, will fail
allowed, time_allowed = limiter.can_do_action(("test_id",), _time_now_s=1,)
self.assertFalse(allowed)
self.assertEqual(10.0, time_allowed)
# But, if we allow 10 actions/sec for this request, we should be allowed
# to continue.
allowed, time_allowed = limiter.can_do_action(
("test_id",), _time_now_s=1, rate_hz=10.0
)
self.assertTrue(allowed)
self.assertEqual(1.1, time_allowed)
# Similarly if we allow a burst of 10 actions
allowed, time_allowed = limiter.can_do_action(
("test_id",), _time_now_s=1, burst_count=10
)
self.assertTrue(allowed)
self.assertEqual(1.0, time_allowed)
def test_allowed_via_ratelimit_and_overriding_parameters(self):
"""Test that we can override options of the ratelimit method that would otherwise
fail an action
"""
# Create a Ratelimiter with a very low allowed rate_hz and burst_count
limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
# First attempt should be allowed
limiter.ratelimit(key=("test_id",), _time_now_s=0)
# Second attempt, 1s later, will fail
with self.assertRaises(LimitExceededError) as context:
limiter.ratelimit(key=("test_id",), _time_now_s=1)
self.assertEqual(context.exception.retry_after_ms, 9000)
# But, if we allow 10 actions/sec for this request, we should be allowed
# to continue.
limiter.ratelimit(key=("test_id",), _time_now_s=1, rate_hz=10.0)
# Similarly if we allow a burst of 10 actions
limiter.ratelimit(key=("test_id",), _time_now_s=1, burst_count=10)
def test_pruning(self):
limiter = Ratelimiter()
allowed, time_allowed = limiter.can_do_action(
key="test_id_1", time_now_s=0, rate_hz=0.1, burst_count=1
)
limiter = Ratelimiter(clock=None, rate_hz=0.1, burst_count=1)
limiter.can_do_action(key="test_id_1", _time_now_s=0)
self.assertIn("test_id_1", limiter.message_counts)
self.assertIn("test_id_1", limiter.actions)
allowed, time_allowed = limiter.can_do_action(
key="test_id_2", time_now_s=10, rate_hz=0.1, burst_count=1
)
limiter.can_do_action(key="test_id_2", _time_now_s=10)
self.assertNotIn("test_id_1", limiter.message_counts)
self.assertNotIn("test_id_1", limiter.actions)

6
tests/handlers/test_profile.py
View file

@ -14,7 +14,7 @@
# limitations under the License.
from mock import Mock, NonCallableMock
from mock import Mock
from twisted.internet import defer
@ -55,12 +55,8 @@ class ProfileTestCase(unittest.TestCase):
federation_client=self.mock_federation,
federation_server=Mock(),
federation_registry=self.mock_registry,
ratelimiter=NonCallableMock(spec_set=["can_do_action"]),
)
self.ratelimiter = hs.get_ratelimiter()
self.ratelimiter.can_do_action.return_value = (True, 0)
self.store = hs.get_datastore()
self.frank = UserID.from_string("@1234ABCD:test")

9
tests/replication/slave/storage/_base.py
View file

@ -13,7 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from mock import Mock, NonCallableMock
from mock import Mock
from tests.replication._base import BaseStreamTestCase
@ -21,12 +21,7 @@ from tests.replication._base import BaseStreamTestCase
class BaseSlavedStoreTestCase(BaseStreamTestCase):
def make_homeserver(self, reactor, clock):
hs = self.setup_test_homeserver(
federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["can_do_action"]),
)
hs.get_ratelimiter().can_do_action.return_value = (True, 0)
hs = self.setup_test_homeserver(federation_client=Mock())
return hs

8
tests/rest/client/v1/test_events.py
View file

@ -15,7 +15,7 @@
""" Tests REST events for /events paths."""
from mock import Mock, NonCallableMock
from mock import Mock
import synapse.rest.admin
from synapse.rest.client.v1 import events, login, room
@ -40,11 +40,7 @@ class EventStreamPermissionsTestCase(unittest.HomeserverTestCase):
config["enable_registration"] = True
config["auto_join_rooms"] = []
hs = self.setup_test_homeserver(
config=config, ratelimiter=NonCallableMock(spec_set=["can_do_action"])
)
self.ratelimiter = hs.get_ratelimiter()
self.ratelimiter.can_do_action.return_value = (True, 0)
hs = self.setup_test_homeserver(config=config)
hs.get_handlers().federation_handler = Mock()

49
tests/rest/client/v1/test_login.py
View file

@ -29,7 +29,6 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase):
]
def make_homeserver(self, reactor, clock):
self.hs = self.setup_test_homeserver()
self.hs.config.enable_registration = True
self.hs.config.registrations_require_3pid = []
@ -38,10 +37,20 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase):
return self.hs
@override_config(
{
"rc_login": {
"address": {"per_second": 0.17, "burst_count": 5},
# Prevent the account login ratelimiter from raising first
#
# This is normally covered by the default test homeserver config
# which sets these values to 10000, but as we're overriding the entire
# rc_login dict here, we need to set this manually as well
"account": {"per_second": 10000, "burst_count": 10000},
}
}
)
def test_POST_ratelimiting_per_address(self):
self.hs.config.rc_login_address.burst_count = 5
self.hs.config.rc_login_address.per_second = 0.17
# Create different users so we're sure not to be bothered by the per-user
# ratelimiter.
for i in range(0, 6):
@ -80,10 +89,20 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase):
self.assertEquals(channel.result["code"], b"200", channel.result)
@override_config(
{
"rc_login": {
"account": {"per_second": 0.17, "burst_count": 5},
# Prevent the address login ratelimiter from raising first
#
# This is normally covered by the default test homeserver config
# which sets these values to 10000, but as we're overriding the entire
# rc_login dict here, we need to set this manually as well
"address": {"per_second": 10000, "burst_count": 10000},
}
}
)
def test_POST_ratelimiting_per_account(self):
self.hs.config.rc_login_account.burst_count = 5
self.hs.config.rc_login_account.per_second = 0.17
self.register_user("kermit", "monkey")
for i in range(0, 6):
@ -119,10 +138,20 @@ class LoginRestServletTestCase(unittest.HomeserverTestCase):
self.assertEquals(channel.result["code"], b"200", channel.result)
@override_config(
{
"rc_login": {
# Prevent the address login ratelimiter from raising first
#
# This is normally covered by the default test homeserver config
# which sets these values to 10000, but as we're overriding the entire
# rc_login dict here, we need to set this manually as well
"address": {"per_second": 10000, "burst_count": 10000},
"failed_attempts": {"per_second": 0.17, "burst_count": 5},
}
}
)
def test_POST_ratelimiting_per_account_failed_attempts(self):
self.hs.config.rc_login_failed_attempts.burst_count = 5
self.hs.config.rc_login_failed_attempts.per_second = 0.17
self.register_user("kermit", "monkey")
for i in range(0, 6):

9
tests/rest/client/v1/test_rooms.py
View file

@ -20,7 +20,7 @@
import json
from mock import Mock, NonCallableMock
from mock import Mock
from six.moves.urllib import parse as urlparse
from twisted.internet import defer
@ -46,13 +46,8 @@ class RoomBase(unittest.HomeserverTestCase):
def make_homeserver(self, reactor, clock):
self.hs = self.setup_test_homeserver(
"red",
http_client=None,
federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["can_do_action"]),
"red", http_client=None, federation_client=Mock(),
)
self.ratelimiter = self.hs.get_ratelimiter()
self.ratelimiter.can_do_action.return_value = (True, 0)
self.hs.get_federation_handler = Mock(return_value=Mock())

10
tests/rest/client/v1/test_typing.py
View file

@ -16,7 +16,7 @@
"""Tests REST events for /rooms paths."""
from mock import Mock, NonCallableMock
from mock import Mock
from twisted.internet import defer
@ -39,17 +39,11 @@ class RoomTypingTestCase(unittest.HomeserverTestCase):
def make_homeserver(self, reactor, clock):
hs = self.setup_test_homeserver(
"red",
http_client=None,
federation_client=Mock(),
ratelimiter=NonCallableMock(spec_set=["can_do_action"]),
"red", http_client=None, federation_client=Mock(),
)
self.event_source = hs.get_event_sources().sources["typing"]
self.ratelimiter = hs.get_ratelimiter()
self.ratelimiter.can_do_action.return_value = (True, 0)
hs.get_handlers().federation_handler = Mock()
def get_user_by_access_token(token=None, allow_guest=False):

9
tests/rest/client/v2_alpha/test_register.py
View file

@ -29,6 +29,7 @@ from synapse.rest.client.v1 import login, logout
from synapse.rest.client.v2_alpha import account, account_validity, register, sync
from tests import unittest
from tests.unittest import override_config
class RegisterRestServletTestCase(unittest.HomeserverTestCase):
@ -146,10 +147,8 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase):
self.assertEquals(channel.result["code"], b"403", channel.result)
self.assertEquals(channel.json_body["error"], "Guest access is disabled")
@override_config({"rc_registration": {"per_second": 0.17, "burst_count": 5}})
def test_POST_ratelimiting_guest(self):
self.hs.config.rc_registration.burst_count = 5
self.hs.config.rc_registration.per_second = 0.17
for i in range(0, 6):
url = self.url + b"?kind=guest"
request, channel = self.make_request(b"POST", url, b"{}")
@ -168,10 +167,8 @@ class RegisterRestServletTestCase(unittest.HomeserverTestCase):
self.assertEquals(channel.result["code"], b"200", channel.result)
@override_config({"rc_registration": {"per_second": 0.17, "burst_count": 5}})
def test_POST_ratelimiting(self):
self.hs.config.rc_registration.burst_count = 5
self.hs.config.rc_registration.per_second = 0.17
for i in range(0, 6):
params = {
"username": "kermit" + str(i),