insertion into monthly_active_users

This commit is contained in:
Neil Johnson 2018-08-02 13:47:19 +01:00
parent 4a6725d9d1
commit 00f99f74b1
5 changed files with 99 additions and 11 deletions

View File

@ -213,7 +213,7 @@ class Auth(object):
default=[b""] default=[b""]
)[0] )[0]
if user and access_token and ip_addr: if user and access_token and ip_addr:
self.store.insert_client_ip( yield self.store.insert_client_ip(
user_id=user.to_string(), user_id=user.to_string(),
access_token=access_token, access_token=access_token,
ip=ip_addr, ip=ip_addr,

View File

@ -39,6 +39,7 @@ from .filtering import FilteringStore
from .group_server import GroupServerStore from .group_server import GroupServerStore
from .keys import KeyStore from .keys import KeyStore
from .media_repository import MediaRepositoryStore from .media_repository import MediaRepositoryStore
from .monthly_active_users import MonthlyActiveUsersStore
from .openid import OpenIdStore from .openid import OpenIdStore
from .presence import PresenceStore, UserPresenceState from .presence import PresenceStore, UserPresenceState
from .profile import ProfileStore from .profile import ProfileStore
@ -87,6 +88,7 @@ class DataStore(RoomMemberStore, RoomStore,
UserDirectoryStore, UserDirectoryStore,
GroupServerStore, GroupServerStore,
UserErasureStore, UserErasureStore,
MonthlyActiveUsersStore,
): ):
def __init__(self, db_conn, hs): def __init__(self, db_conn, hs):

View File

@ -35,6 +35,7 @@ LAST_SEEN_GRANULARITY = 120 * 1000
class ClientIpStore(background_updates.BackgroundUpdateStore): class ClientIpStore(background_updates.BackgroundUpdateStore):
def __init__(self, db_conn, hs): def __init__(self, db_conn, hs):
self.client_ip_last_seen = Cache( self.client_ip_last_seen = Cache(
name="client_ip_last_seen", name="client_ip_last_seen",
keylen=4, keylen=4,
@ -74,6 +75,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
"before", "shutdown", self._update_client_ips_batch "before", "shutdown", self._update_client_ips_batch
) )
@defer.inlineCallbacks
def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id, def insert_client_ip(self, user_id, access_token, ip, user_agent, device_id,
now=None): now=None):
if not now: if not now:
@ -84,7 +86,7 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
last_seen = self.client_ip_last_seen.get(key) last_seen = self.client_ip_last_seen.get(key)
except KeyError: except KeyError:
last_seen = None last_seen = None
yield self._populate_monthly_active_users(user_id)
# Rate-limited inserts # Rate-limited inserts
if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY: if last_seen is not None and (now - last_seen) < LAST_SEEN_GRANULARITY:
return return
@ -93,6 +95,24 @@ class ClientIpStore(background_updates.BackgroundUpdateStore):
self._batch_row_update[key] = (user_agent, device_id, now) self._batch_row_update[key] = (user_agent, device_id, now)
@defer.inlineCallbacks
def _populate_monthly_active_users(self, user_id):
store = self.hs.get_datastore()
print "entering _populate_monthly_active_users"
if self.hs.config.limit_usage_by_mau:
print "self.hs.config.limit_usage_by_mau is TRUE"
is_user_monthly_active = yield store.is_user_monthly_active(user_id)
print "is_user_monthly_active is %r" % is_user_monthly_active
if is_user_monthly_active:
yield store.upsert_monthly_active_user(user_id)
else:
count = yield store.get_monthly_active_count()
print "count is %d" % count
if count < self.hs.config.max_mau_value:
print "count is less than self.hs.config.max_mau_value "
res = yield store.upsert_monthly_active_user(user_id)
print "upsert response is %r" % res
def _update_client_ips_batch(self): def _update_client_ips_batch(self):
def update(): def update():
to_update = self._batch_row_update to_update = self._batch_row_update

View File

@ -4,7 +4,7 @@ from ._base import SQLBaseStore
class MonthlyActiveUsersStore(SQLBaseStore): class MonthlyActiveUsersStore(SQLBaseStore):
def __init__(self, hs): def __init__(self, dbconn, hs):
super(MonthlyActiveUsersStore, self).__init__(None, hs) super(MonthlyActiveUsersStore, self).__init__(None, hs)
self._clock = hs.get_clock() self._clock = hs.get_clock()
self.max_mau_value = hs.config.max_mau_value self.max_mau_value = hs.config.max_mau_value
@ -14,24 +14,28 @@ class MonthlyActiveUsersStore(SQLBaseStore):
Cleans out monthly active user table to ensure that no stale Cleans out monthly active user table to ensure that no stale
entries exist. entries exist.
Return: Return:
defered, no return type Defered()
""" """
def _reap_users(txn): def _reap_users(txn):
thirty_days_ago = ( thirty_days_ago = (
int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30) int(self._clock.time_msec()) - (1000 * 60 * 60 * 24 * 30)
) )
sql = "DELETE FROM monthly_active_users WHERE timestamp < ?" sql = "DELETE FROM monthly_active_users WHERE timestamp < ?"
txn.execute(sql, (thirty_days_ago,)) txn.execute(sql, (thirty_days_ago,))
sql = """
DELETE FROM monthly_active_users
ORDER BY timestamp desc
LIMIT -1 OFFSET ?
"""
txn.execute(sql, (self.max_mau_value,))
return self.runInteraction("reap_monthly_active_users", _reap_users) return self.runInteraction("reap_monthly_active_users", _reap_users)
def get_monthly_active_count(self): def get_monthly_active_count(self):
""" """
Generates current count of monthly active users.abs Generates current count of monthly active users.abs
return: Return:
defered resolves to int Defered(int): Number of current monthly active users
""" """
def _count_users(txn): def _count_users(txn):
sql = "SELECT COALESCE(count(*), 0) FROM monthly_active_users" sql = "SELECT COALESCE(count(*), 0) FROM monthly_active_users"
@ -46,6 +50,8 @@ class MonthlyActiveUsersStore(SQLBaseStore):
Updates or inserts monthly active user member Updates or inserts monthly active user member
Arguments: Arguments:
user_id (str): user to add/update user_id (str): user to add/update
Deferred(bool): True if a new entry was created, False if an
existing one was updated.
""" """
return self._simple_upsert( return self._simple_upsert(
desc="upsert_monthly_active_user", desc="upsert_monthly_active_user",

View File

@ -12,6 +12,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and # See the License for the specific language governing permissions and
# limitations under the License. # limitations under the License.
from mock import Mock
from twisted.internet import defer from twisted.internet import defer
@ -27,9 +28,9 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
@defer.inlineCallbacks @defer.inlineCallbacks
def setUp(self): def setUp(self):
hs = yield tests.utils.setup_test_homeserver() self.hs = yield tests.utils.setup_test_homeserver()
self.store = hs.get_datastore() self.store = self.hs.get_datastore()
self.clock = hs.get_clock() self.clock = self.hs.get_clock()
@defer.inlineCallbacks @defer.inlineCallbacks
def test_insert_new_client_ip(self): def test_insert_new_client_ip(self):
@ -54,3 +55,62 @@ class ClientIpStoreTestCase(tests.unittest.TestCase):
}, },
r r
) )
@defer.inlineCallbacks
def test_disabled_monthly_active_user(self):
self.hs.config.limit_usage_by_mau = False
self.hs.config.max_mau_value = 50
user_id = "@user:server"
yield self.store.insert_client_ip(
user_id, "access_token", "ip", "user_agent", "device_id",
)
active = yield self.store.is_user_monthly_active(user_id)
self.assertFalse(active)
@defer.inlineCallbacks
def test_adding_monthly_active_user_when_full(self):
self.hs.config.limit_usage_by_mau = True
self.hs.config.max_mau_value = 50
lots_of_users = 100
user_id = "@user:server"
self.store.get_monthly_active_count = Mock(
return_value=defer.succeed(lots_of_users)
)
yield self.store.insert_client_ip(
user_id, "access_token", "ip", "user_agent", "device_id",
)
active = yield self.store.is_user_monthly_active(user_id)
self.assertFalse(active)
@defer.inlineCallbacks
def test_adding_monthly_active_user_when_space(self):
self.hs.config.limit_usage_by_mau = True
self.hs.config.max_mau_value = 50
user_id = "@user:server"
active = yield self.store.is_user_monthly_active(user_id)
self.assertFalse(active)
yield self.store.insert_client_ip(
user_id, "access_token", "ip", "user_agent", "device_id",
)
active = yield self.store.is_user_monthly_active(user_id)
self.assertTrue(active)
@defer.inlineCallbacks
def test_updating_monthly_active_user_when_space(self):
self.hs.config.limit_usage_by_mau = True
self.hs.config.max_mau_value = 50
user_id = "@user:server"
active = yield self.store.is_user_monthly_active(user_id)
self.assertFalse(active)
yield self.store.insert_client_ip(
user_id, "access_token", "ip", "user_agent", "device_id",
)
yield self.store.insert_client_ip(
user_id, "access_token", "ip", "user_agent", "device_id",
)
active = yield self.store.is_user_monthly_active(user_id)
self.assertTrue(active)