Add sync state indicator and support for clearing cache

maubot/__meta__.py

@@ -1 +1 @@
-__version__ = "0.1.0.dev22"
+__version__ = "0.1.0.dev23"

maubot/client.py

@@ -13,7 +13,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, List, Optional, Set, TYPE_CHECKING
+from typing import Dict, List, Optional, Set, Callable, Any, Awaitable, TYPE_CHECKING
 import asyncio
 import logging
 
@@ -23,6 +23,7 @@ from aiohttp import ClientSession
 from mautrix.errors import MatrixInvalidToken, MatrixRequestError
 from mautrix.types import (UserID, SyncToken, FilterID, ContentURI, StrippedStateEvent, Membership,
                            EventType, Filter, RoomFilter, RoomEventFilter)
+from mautrix.client import InternalEventType
 
 from .db import DBClient
 from .matrix import MaubotMatrixClient
@@ -51,11 +52,22 @@ class Client:
         self.log = log.getChild(self.id)
         self.references = set()
         self.started = False
+        self.sync_ok = True
         self.client = MaubotMatrixClient(mxid=self.id, base_url=self.homeserver,
                                          token=self.access_token, client_session=self.http_client,
                                          log=self.log, loop=self.loop, store=self.db_instance)
+        self.client.ignore_initial_sync = True
+        self.client.ignore_first_sync = True
         if self.autojoin:
             self.client.add_event_handler(EventType.ROOM_MEMBER, self._handle_invite)
+        self.client.add_event_handler(InternalEventType.SYNC_ERRORED, self._set_sync_ok(False))
+        self.client.add_event_handler(InternalEventType.SYNC_SUCCESSFUL, self._set_sync_ok(True))
+
+    def _set_sync_ok(self, ok: bool) -> Callable[[Dict[str, Any]], Awaitable[None]]:
+        async def handler(data: Dict[str, Any]) -> None:
+            self.sync_ok = ok
+
+        return handler
 
     async def start(self, try_n: Optional[int] = 0) -> None:
         try:
@@ -128,6 +140,13 @@ class Client:
             await self.stop_plugins()
             self.stop_sync()
 
+    def clear_cache(self) -> None:
+        self.stop_sync()
+        self.db_instance.filter_id = ""
+        self.db_instance.next_batch = ""
+        self.db.commit()
+        self.start_sync()
+
     def delete(self) -> None:
         try:
             del self.cache[self.id]
@@ -144,6 +163,7 @@ class Client:
             "enabled": self.enabled,
             "started": self.started,
             "sync": self.sync,
+            "sync_ok": self.sync_ok,
             "autojoin": self.autojoin,
             "displayname": self.displayname,
             "avatar_url": self.avatar_url,

maubot/management/api/client.py

@@ -130,3 +130,13 @@ async def delete_client(request: web.Request) -> web.Response:
         await client.stop()
     client.delete()
     return resp.deleted
+
+
+@routes.post("/client/{id}/clearcache")
+async def clear_client_cache(request: web.Request) -> web.Response:
+    user_id = request.match_info.get("id", None)
+    client = Client.get(user_id, None)
+    if not client:
+        return resp.client_not_found
+    client.clear_cache()
+    return resp.ok

maubot/management/api/client_proxy.py

@@ -36,6 +36,7 @@ async def proxy(request: web.Request) -> web.StreamResponse:
     except KeyError:
         pass
     headers = request.headers.copy()
+    del headers["Host"]
     headers["Authorization"] = f"Bearer {client.access_token}"
     if "X-Forwarded-For" not in headers:
         peer = request.transport.get_extra_info("peername")

maubot/management/api/spec.yaml

@@ -399,6 +399,32 @@ paths:
             application/json:
               schema:
                 $ref: '#/components/schemas/Error'
+  '/client/{id}/clearcache':
+    parameters:
+    - name: id
+      in: path
+      description: The Matrix user ID of the client to change
+      required: true
+      schema:
+        type: string
+    put:
+      operationId: clear_client_cache
+      summary: Clear the sync/state cache of a Matrix client
+      tags: [Clients]
+      responses:
+        200:
+          description: Cache cleared
+          content:
+            application/json:
+              schema:
+                type: object
+                properties:
+                  success:
+                    type: boolean
+        401:
+          $ref: '#/components/responses/Unauthorized'
+        404:
+          $ref: '#/components/responses/ClientNotFound'
   /client/auth/servers:
     get:
       operationId: get_client_auth_servers
@@ -607,29 +633,42 @@ components:
           type: string
           example: '@putkiteippi:maunium.net'
           readOnly: true
+          description: The Matrix user ID of this client.
         homeserver:
           type: string
           example: 'https://maunium.net'
+          description: The homeserver URL for this client.
         access_token:
           type: string
+          description: The Matrix access token for this client.
         enabled:
           type: boolean
           example: true
+          description: Whether or not this client is enabled.
         started:
           type: boolean
           example: true
+          description: Whether or not this client and its instances have been started.
         sync:
           type: boolean
           example: true
+          description: Whether or not syncing is enabled on this client.
+        sync_ok:
+          type: boolean
+          example: true
+          description: Whether or not the previous sync was successful on this client.
         autojoin:
           type: boolean
           example: true
+          description: Whether or not this client should automatically join rooms when invited.
         displayname:
           type: string
           example: J. E. Saarinen
+          description: The display name for this client.
         avatar_url:
           type: string
           example: 'mxc://maunium.net/FsPQQTntCCqhJMFtwArmJdaU'
+          description: The content URI of the avatar for this client.
         instances:
           type: array
           readOnly: true

maubot/management/frontend/src/api.js

@@ -37,7 +37,7 @@ async function defaultDelete(type, id) {
 }
 
 async function defaultPut(type, entry, id = undefined, suffix = undefined) {
-    const resp = await fetch(`${BASE_PATH}/${type}/${id || entry.id}${suffix || ''}`, {
+    const resp = await fetch(`${BASE_PATH}/${type}/${id || entry.id}${suffix || ""}`, {
         headers: getHeaders(),
         body: JSON.stringify(entry),
         method: "PUT",
@@ -221,6 +221,14 @@ export function getAvatarURL({ id, avatar_url }) {
 export const putClient = client => defaultPut("client", client)
 export const deleteClient = id => defaultDelete("client", id)
 
+export async function clearClientCache(id) {
+    const resp = await fetch(`${BASE_PATH}/client/${id}/clearcache`, {
+        headers: getHeaders(),
+        method: "POST",
+    })
+    return await resp.json()
+}
+
 export const getClientAuthServers = () => defaultGet("/client/auth/servers")
 
 export async function doClientAuth(server, type, username, password) {
@@ -240,6 +248,6 @@ export default {
     getInstances, getInstance, putInstance, deleteInstance,
     getInstanceDatabase, queryInstanceDatabase,
     getPlugins, getPlugin, uploadPlugin, deletePlugin,
-    getClients, getClient, uploadAvatar, getAvatarURL, putClient, deleteClient,
-    getClientAuthServers, doClientAuth
+    getClients, getClient, uploadAvatar, getAvatarURL, putClient, deleteClient, clearClientCache,
+    getClientAuthServers, doClientAuth,
 }

maubot/management/frontend/src/pages/dashboard/Client.js

@@ -44,6 +44,7 @@ class Client extends BaseMainView {
     constructor(props) {
         super(props)
         this.deleteFunc = api.deleteClient
+        this.homeserverOptions = {}
     }
 
     get entryKeys() {
@@ -69,6 +70,7 @@ class Client extends BaseMainView {
             saving: false,
             deleting: false,
             startingOrStopping: false,
+            clearingCache: false,
             error: "",
         }
     }
@@ -79,6 +81,7 @@ class Client extends BaseMainView {
         delete client.saving
         delete client.deleting
         delete client.startingOrStopping
+        delete client.clearingCache
         delete client.error
         delete client.instances
         return client
@@ -88,7 +91,7 @@ class Client extends BaseMainView {
         return this.state.homeserver
             ? this.homeserverEntry([this.props.ctx.homeserversByURL[this.state.homeserver],
                 this.state.homeserver])
-            : {}
+            : this.homeserverOptions[0] || {}
     }
 
     homeserverEntry = ([serverName, serverURL]) => serverURL && {
@@ -156,8 +159,39 @@ class Client extends BaseMainView {
         }
     }
 
+    clearCache = async () => {
+        this.setState({ clearingCache: true })
+        const resp = await api.clearClientCache(this.props.entry.id)
+        if (resp.success) {
+            this.setState({ clearingCache: false, error: "" })
+        } else {
+            this.setState({ clearingCache: false, error: resp.error })
+        }
+    }
+
     get loading() {
-        return this.state.saving || this.state.startingOrStopping || this.state.deleting
+        return this.state.saving || this.state.startingOrStopping || this.clearingCache || this.state.deleting
+    }
+
+    renderStartedContainer = () => {
+        let text
+        if (this.props.entry.started) {
+            if (this.props.entry.sync_ok) {
+                text = "Started"
+            } else {
+                text = "Erroring"
+            }
+        } else if (this.props.entry.enabled) {
+            text = "Stopped"
+        } else {
+            text = "Disabled"
+        }
+        return <div className="started-container">
+            <span className={`started ${this.props.entry.started}
+                             ${this.props.entry.sync_ok ? "sync_ok" : "sync_error"}
+                             ${this.props.entry.enabled ? "" : "disabled"}`}/>
+            <span className="text">{text}</span>
+        </div>
     }
 
     renderSidebar = () => !this.isNew && (
@@ -172,20 +206,16 @@ class Client extends BaseMainView {
                        onDragLeave={evt => evt.target.parentElement.classList.remove("drag")}/>
                 {this.state.uploadingAvatar && <Spinner/>}
             </div>
-            <div className="started-container">
-                <span className={`started ${this.props.entry.started}
-                        ${this.props.entry.enabled ? "" : "disabled"}`}/>
-                <span className="text">
-                    {this.props.entry.started ? "Started" :
-                        (this.props.entry.enabled ? "Stopped" : "Disabled")}
-                </span>
-            </div>
-            {(this.props.entry.started || this.props.entry.enabled) && (
+            {this.renderStartedContainer()}
+            {(this.props.entry.started || this.props.entry.enabled) && <>
                 <button className="save" onClick={this.startOrStop} disabled={this.loading}>
                     {this.state.startingOrStopping ? <Spinner/>
                         : (this.props.entry.started ? "Stop" : "Start")}
                 </button>
-            )}
+                <button className="clearcache" onClick={this.clearCache} disabled={this.loading}>
+                    {this.state.clearingCache ? <Spinner/> : "Clear cache"}
+                </button>
+            </>}
         </div>
     )
 
@@ -195,10 +225,17 @@ class Client extends BaseMainView {
                        name={this.isNew ? "id" : ""} className="id"
                        value={this.state.id} origValue={this.props.entry.id}
                        placeholder="@fancybot:example.com" onChange={this.inputChange}/>
-            <PrefSelect rowName="Homeserver" options={this.homeserverOptions} isSearchable={true}
-                        value={this.selectedHomeserver} origValue={this.props.entry.homeserver}
-                        onChange={({ id }) => this.setState({ homeserver: id })}
-                        creatable={true} isValidNewOption={this.isValidHomeserver}/>
+            {api.getFeatures().client_auth ? (
+                <PrefSelect rowName="Homeserver" options={this.homeserverOptions}
+                            isSearchable={true} value={this.selectedHomeserver}
+                            origValue={this.props.entry.homeserver}
+                            onChange={({ value }) => this.setState({ homeserver: value })}
+                            creatable={true} isValidNewOption={this.isValidHomeserver}/>
+            ) : (
+                <PrefInput rowName="Homeserver" type="text" name="homeserver"
+                           value={this.state.homeserver} origValue={this.props.entry.homeserver}
+                           placeholder="https://example.com" onChange={this.inputChange}/>
+            )}
             <PrefInput rowName="Access token" type="text" name="access_token"
                        value={this.state.access_token} origValue={this.props.entry.access_token}
                        placeholder="MDAxYWxvY2F0aW9uIG1hdHJpeC5sb2NhbAowMDEzaWRlbnRpZmllc"

maubot/management/frontend/src/pages/dashboard/index.js

@@ -75,10 +75,11 @@ class Dashboard extends Component {
         }
         const homeserversByName = homeservers
         const homeserversByURL = {}
-        for (const [key, value] of Object.entries(homeservers)) {
-            homeserversByURL[value] = key
+        if (api.getFeatures().client_auth) {
+            for (const [key, value] of Object.entries(homeservers)) {
+                homeserversByURL[value] = key
+            }
         }
-        console.log(homeserversByName, homeserversByURL)
         this.setState({ instances, clients, plugins, homeserversByName, homeserversByURL })
 
         await this.enableLogs()

maubot/management/frontend/src/setupProxy.js

@@ -1,6 +1,6 @@
 const proxy = require("http-proxy-middleware")
 
 module.exports = function(app) {
-    app.use(proxy("/_matrix/maubot/v1", { target: "http://localhost:29316" }))
-    app.use(proxy("/_matrix/maubot/v1/logs", { target: "http://localhost:29316", ws: true }))
+    app.use(proxy("/_matrix/maubot/v1", { target: process.env.PROXY || "http://localhost:29316" }))
+    app.use(proxy("/_matrix/maubot/v1/logs", { target: process.env.PROXY || "http://localhost:29316", ws: true }))
 }

maubot/management/frontend/src/style/base/vars.sass

@@ -23,6 +23,7 @@ $secondary-light: #62EBFF
 $error: #B71C1C
 $error-dark: #7F0000
 $error-light: #F05545
+$warning: orange
 
 $border-color: #DDD
 $text-color: #212121

maubot/management/frontend/src/style/pages/client/index.sass

@@ -23,7 +23,7 @@
         width: 8rem
         margin-right: 1rem
 
-        > div
+        > *
             margin-bottom: 1rem
 
         @import avatar

maubot/management/frontend/src/style/pages/client/started.sass

@@ -25,8 +25,13 @@
         margin: .5rem
 
         &.true
-            background-color: $primary
-            box-shadow: 0 0 .75rem .75rem $primary
+            &.sync_ok
+                background-color: $primary
+                box-shadow: 0 0 .75rem .75rem $primary
+
+            &.sync_error
+                background-color: $warning
+                box-shadow: 0 0 .75rem .75rem $warning
 
         &.false
             background-color: $error-light

maubot/management/frontend/src/style/pages/dashboard.sass

@@ -116,7 +116,7 @@
             &:hover
                 background-color: $error !important
 
-        button.save, button.delete
+        button.save, button.clearcache, button.delete
             +button
             +main-color-button
             width: 100%

maubot/matrix.py

@@ -18,7 +18,7 @@ from markdown.extensions import Extension
 import markdown as md
 import attr
 
-from mautrix.client import Client as MatrixClient
+from mautrix.client import Client as MatrixClient, SyncStream
 from mautrix.util.formatter import parse_html
 from mautrix.types import (EventType, MessageEvent, Event, EventID, RoomID, MessageEventContent,
                            MessageType, TextMessageEventContent, Format, RelatesTo)
@@ -83,12 +83,12 @@ class MaubotMatrixClient(MatrixClient):
             content.relates_to = relates_to
         return self.send_message(room_id, content, **kwargs)
 
-    async def call_handlers(self, event: Event, source) -> None:
+    async def dispatch_event(self, event: Event, source: SyncStream = SyncStream.INTERNAL) -> None:
         if isinstance(event, MessageEvent):
             event = MaubotMessageEvent(event, self)
-        else:
+        elif source != SyncStream.INTERNAL:
             event.client = self
-        return await super().call_handlers(event, source)
+        return await super().dispatch_event(event, source)
 
     async def get_event(self, room_id: RoomID, event_id: EventID) -> Event:
         event = await super().get_event(room_id, event_id)

setup.py

@@ -21,7 +21,7 @@ setuptools.setup(
     packages=setuptools.find_packages(),
 
     install_requires=[
-        "mautrix>=0.4.dev46,<0.5",
+        "mautrix>=0.4.dev47,<0.5",
         "aiohttp>=3.0.1,<4",
         "SQLAlchemy>=1.2.3,<2",
         "alembic>=1.0.0,<2",