Add option to track MAU stats (but not limit people) (#3830)

changelog.d/3830.feature

@@ -0,0 +1 @@
+Add option to track MAU stats (but not limit people)

synapse/app/homeserver.py

@@ -535,7 +535,7 @@ def run(hs):
         current_mau_count = 0
         reserved_count = 0
         store = hs.get_datastore()
-        if hs.config.limit_usage_by_mau:
+        if hs.config.limit_usage_by_mau or hs.config.mau_stats_only:
             current_mau_count = yield store.get_monthly_active_count()
             reserved_count = yield store.get_registered_reserved_users_count()
         current_mau_gauge.set(float(current_mau_count))

synapse/config/server.py

@@ -77,6 +77,7 @@ class ServerConfig(Config):
             self.max_mau_value = config.get(
                 "max_mau_value", 0,
             )
+        self.mau_stats_only = config.get("mau_stats_only", False)
 
         self.mau_limits_reserved_threepids = config.get(
             "mau_limit_reserved_threepids", []
@@ -372,6 +373,11 @@ class ServerConfig(Config):
           # max_mau_value: 50
           # mau_trial_days: 2
           #
+          # If enabled, the metrics for the number of monthly active users will
+          # be populated, however no one will be limited. If limit_usage_by_mau
+          # is true, this is implied to be true.
+          # mau_stats_only: False
+          #
           # Sometimes the server admin will want to ensure certain accounts are
           # never blocked by mau checking. These accounts are specified here.
           #

synapse/storage/monthly_active_users.py

@@ -96,37 +96,38 @@ class MonthlyActiveUsersStore(SQLBaseStore):
 
             txn.execute(sql, query_args)
 
-            # If MAU user count still exceeds the MAU threshold, then delete on
+            if self.hs.config.limit_usage_by_mau:
-            # a least recently active basis.
+                # If MAU user count still exceeds the MAU threshold, then delete on
-            # Note it is not possible to write this query using OFFSET due to
+                # a least recently active basis.
-            # incompatibilities in how sqlite and postgres support the feature.
+                # Note it is not possible to write this query using OFFSET due to
-            # sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
+                # incompatibilities in how sqlite and postgres support the feature.
-            # While Postgres does not require 'LIMIT', but also does not support
+                # sqlite requires 'LIMIT -1 OFFSET ?', the LIMIT must be present
-            # negative LIMIT values. So there is no way to write it that both can
+                # While Postgres does not require 'LIMIT', but also does not support
-            # support
+                # negative LIMIT values. So there is no way to write it that both can
-            safe_guard = self.hs.config.max_mau_value - len(self.reserved_users)
+                # support
-            # Must be greater than zero for postgres
+                safe_guard = self.hs.config.max_mau_value - len(self.reserved_users)
-            safe_guard = safe_guard if safe_guard > 0 else 0
+                # Must be greater than zero for postgres
-            query_args = [safe_guard]
+                safe_guard = safe_guard if safe_guard > 0 else 0
+                query_args = [safe_guard]
 
-            base_sql = """
+                base_sql = """
-                DELETE FROM monthly_active_users
+                    DELETE FROM monthly_active_users
-                WHERE user_id NOT IN (
+                    WHERE user_id NOT IN (
-                    SELECT user_id FROM monthly_active_users
+                        SELECT user_id FROM monthly_active_users
-                    ORDER BY timestamp DESC
+                        ORDER BY timestamp DESC
-                    LIMIT ?
+                        LIMIT ?
+                        )
+                    """
+                # Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
+                # when len(reserved_users) == 0. Works fine on sqlite.
+                if len(self.reserved_users) > 0:
+                    query_args.extend(self.reserved_users)
+                    sql = base_sql + """ AND user_id NOT IN ({})""".format(
+                        ','.join(questionmarks)
                     )
-                """
+                else:
-            # Need if/else since 'AND user_id NOT IN ({})' fails on Postgres
+                    sql = base_sql
-            # when len(reserved_users) == 0. Works fine on sqlite.
+                txn.execute(sql, query_args)
-            if len(self.reserved_users) > 0:
-                query_args.extend(self.reserved_users)
-                sql = base_sql + """ AND user_id NOT IN ({})""".format(
-                    ','.join(questionmarks)
-                )
-            else:
-                sql = base_sql
-            txn.execute(sql, query_args)
 
         yield self.runInteraction("reap_monthly_active_users", _reap_users)
         # It seems poor to invalidate the whole cache, Postgres supports
@@ -252,8 +253,7 @@ class MonthlyActiveUsersStore(SQLBaseStore):
         Args:
             user_id(str): the user_id to query
         """
-
+        if self.hs.config.limit_usage_by_mau or self.hs.config.mau_stats_only:
-        if self.hs.config.limit_usage_by_mau:
             # Trial users and guests should not be included as part of MAU group
             is_guest = yield self.is_guest(user_id)
             if is_guest:
@@ -271,8 +271,14 @@ class MonthlyActiveUsersStore(SQLBaseStore):
             # but only update if we have not previously seen the user for
             # LAST_SEEN_GRANULARITY ms
             if last_seen_timestamp is None:
-                count = yield self.get_monthly_active_count()
+                # In the case where mau_stats_only is True and limit_usage_by_mau is
-                if count < self.hs.config.max_mau_value:
+                # False, there is no point in checking get_monthly_active_count - it
+                # adds no value and will break the logic if max_mau_value is exceeded.
+                if not self.hs.config.limit_usage_by_mau:
                     yield self.upsert_monthly_active_user(user_id)
+                else:
+                    count = yield self.get_monthly_active_count()
+                    if count < self.hs.config.max_mau_value:
+                        yield self.upsert_monthly_active_user(user_id)
             elif now - last_seen_timestamp > LAST_SEEN_GRANULARITY:
                 yield self.upsert_monthly_active_user(user_id)

tests/storage/test_monthly_active_users.py

@@ -220,3 +220,28 @@ class MonthlyActiveUsersTestCase(HomeserverTestCase):
         self.store.user_add_threepid(user2, "email", user2_email, now, now)
         count = self.store.get_registered_reserved_users_count()
         self.assertEquals(self.get_success(count), len(threepids))
+
+    def test_track_monthly_users_without_cap(self):
+        self.hs.config.limit_usage_by_mau = False
+        self.hs.config.mau_stats_only = True
+        self.hs.config.max_mau_value = 1  # should not matter
+
+        count = self.store.get_monthly_active_count()
+        self.assertEqual(0, self.get_success(count))
+
+        self.store.upsert_monthly_active_user("@user1:server")
+        self.store.upsert_monthly_active_user("@user2:server")
+        self.pump()
+
+        count = self.store.get_monthly_active_count()
+        self.assertEqual(2, self.get_success(count))
+
+    def test_no_users_when_not_tracking(self):
+        self.hs.config.limit_usage_by_mau = False
+        self.hs.config.mau_stats_only = False
+        self.store.upsert_monthly_active_user = Mock()
+
+        self.store.populate_monthly_active_users("@user:sever")
+        self.pump()
+
+        self.store.upsert_monthly_active_user.assert_not_called()

tests/test_mau.py

@@ -171,6 +171,24 @@ class TestMauLimit(unittest.HomeserverTestCase):
         self.assertEqual(e.code, 403)
         self.assertEqual(e.errcode, Codes.RESOURCE_LIMIT_EXCEEDED)
 
+    def test_tracked_but_not_limited(self):
+        self.hs.config.max_mau_value = 1  # should not matter
+        self.hs.config.limit_usage_by_mau = False
+        self.hs.config.mau_stats_only = True
+
+        # Simply being able to create 2 users indicates that the
+        # limit was not reached.
+        token1 = self.create_user("kermit1")
+        self.do_sync_for_user(token1)
+        token2 = self.create_user("kermit2")
+        self.do_sync_for_user(token2)
+
+        # We do want to verify that the number of tracked users
+        # matches what we want though
+        count = self.store.get_monthly_active_count()
+        self.reactor.advance(100)
+        self.assertEqual(2, self.successResultOf(count))
+
     def create_user(self, localpart):
         request_data = json.dumps(
             {

tests/utils.py

@@ -134,6 +134,7 @@ def default_config(name):
     config.hs_disabled_limit_type = ""
     config.max_mau_value = 50
     config.mau_trial_days = 0
+    config.mau_stats_only = False
     config.mau_limits_reserved_threepids = []
     config.admin_contact = None
     config.rc_messages_per_second = 10000