Support MSC1708 (and co.) and prepare for MSC1711

Fixes https://github.com/turt2live/matrix-dimension/issues/234

Later support for MSC1711 will be done in https://github.com/turt2live/matrix-dimension/issues/238

package-lock.json

@@ -4379,14 +4379,12 @@
         "balanced-match": {
           "version": "1.0.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "brace-expansion": {
           "version": "1.1.11",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "balanced-match": "^1.0.0",
             "concat-map": "0.0.1"
@@ -4401,20 +4399,17 @@
         "code-point-at": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "concat-map": {
           "version": "0.0.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "console-control-strings": {
           "version": "1.1.0",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "core-util-is": {
           "version": "1.0.2",
@@ -4531,8 +4526,7 @@
         "inherits": {
           "version": "2.0.3",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "ini": {
           "version": "1.3.5",
@@ -4544,7 +4538,6 @@
           "version": "1.0.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "number-is-nan": "^1.0.0"
           }
@@ -4559,7 +4552,6 @@
           "version": "3.0.4",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "brace-expansion": "^1.1.7"
           }
@@ -4567,14 +4559,12 @@
         "minimist": {
           "version": "0.0.8",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "minipass": {
           "version": "2.2.4",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "safe-buffer": "^5.1.1",
             "yallist": "^3.0.0"
@@ -4593,7 +4583,6 @@
           "version": "0.5.1",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "minimist": "0.0.8"
           }
@@ -4674,8 +4663,7 @@
         "number-is-nan": {
           "version": "1.0.1",
           "bundled": true,
-          "dev": true,
+          "dev": true
-          "optional": true
         },
         "object-assign": {
           "version": "4.1.1",
@@ -4687,7 +4675,6 @@
           "version": "1.4.0",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "wrappy": "1"
           }
@@ -4809,7 +4796,6 @@
           "version": "1.0.2",
           "bundled": true,
           "dev": true,
-          "optional": true,
           "requires": {
             "code-point-at": "^1.0.0",
             "is-fullwidth-code-point": "^1.0.0",
@@ -5672,8 +5658,7 @@
     "ip-regex": {
       "version": "2.1.0",
       "resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
-      "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=",
+      "integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk="
-      "dev": true
     },
     "ipaddr.js": {
       "version": "1.8.0",
@@ -5948,6 +5933,11 @@
       "integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
       "dev": true
     },
+    "isipaddress": {
+      "version": "0.0.2",
+      "resolved": "https://registry.npmjs.org/isipaddress/-/isipaddress-0.0.2.tgz",
+      "integrity": "sha1-qeRIRlEGrwHmCFHPI146wwEUUNM="
+    },
     "isnumeric": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/isnumeric/-/isnumeric-0.2.0.tgz",
@@ -11018,6 +11008,25 @@
         }
       }
     },
+    "request-promise": {
+      "version": "4.2.2",
+      "resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.2.tgz",
+      "integrity": "sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=",
+      "requires": {
+        "bluebird": "^3.5.0",
+        "request-promise-core": "1.1.1",
+        "stealthy-require": "^1.1.0",
+        "tough-cookie": ">=2.3.3"
+      }
+    },
+    "request-promise-core": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz",
+      "integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=",
+      "requires": {
+        "lodash": "^4.13.1"
+      }
+    },
     "require-dir-all": {
       "version": "0.4.15",
       "resolved": "https://registry.npmjs.org/require-dir-all/-/require-dir-all-0.4.15.tgz",
@@ -12003,6 +12012,11 @@
       "integrity": "sha1-kPn0ZqIOjjnvJNqVnB5hHCow3VQ=",
       "dev": true
     },
+    "split-host": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/split-host/-/split-host-0.1.1.tgz",
+      "integrity": "sha512-nrlaPJMHkr3hKx7aCyr+S0OgUvAm/xKzWWMHej0IsMamWjRC52Fv+NGZwuqRE1lyu1iNWCmcrpZ1S1qvk+Uiwg=="
+    },
     "split-string": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz",
@@ -12153,6 +12167,11 @@
         "readable-stream": "^2.0.1"
       }
     },
+    "stealthy-require": {
+      "version": "1.1.1",
+      "resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
+      "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
+    },
     "stream-browserify": {
       "version": "2.0.1",
       "resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz",
@@ -12758,6 +12777,23 @@
       "resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz",
       "integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg="
     },
+    "tough-cookie": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz",
+      "integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==",
+      "requires": {
+        "ip-regex": "^2.1.0",
+        "psl": "^1.1.28",
+        "punycode": "^2.1.1"
+      },
+      "dependencies": {
+        "punycode": {
+          "version": "2.1.1",
+          "resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
+          "integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
+        }
+      }
+    },
     "trim-newlines": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",

package.json

@@ -27,6 +27,7 @@
     "dns-then": "^0.1.0",
     "express": "^4.16.4",
     "git-rev-sync": "^1.12.0",
+    "isipaddress": "0.0.2",
     "js-yaml": "^3.12.0",
     "lodash": "^4.17.5",
     "matrix-js-snippets": "^0.2.8",
@@ -36,10 +37,12 @@
     "netmask": "^1.0.6",
     "random-string": "^0.2.0",
     "request": "^2.88.0",
+    "request-promise": "^4.2.2",
     "require-dir-all": "^0.4.15",
     "sequelize": "^4.39.1",
     "sequelize-typescript": "^0.6.6",
     "sharp": "^0.21.1",
+    "split-host": "^0.1.1",
     "spotify-uri": "^1.0.0",
     "sqlite3": "^4.0.4",
     "telegraf": "^3.25.5",

src/api/admin/AdminService.ts

@@ -4,7 +4,7 @@ import config from "../../config";
 import { ApiError } from "../ApiError";
 import { MatrixLiteClient } from "../../matrix/MatrixLiteClient";
 import { CURRENT_VERSION } from "../../version";
-import { getFederationUrl } from "../../matrix/helpers";
+import { getFederationConnInfo } from "../../matrix/helpers";
 
 interface DimensionVersionResponse {
     version: string;
@@ -17,6 +17,7 @@ interface DimensionConfigResponse {
         name: string;
         userId: string;
         federationUrl: string;
+        federationHostname: string;
         clientServerUrl: string;
     };
 }
@@ -70,15 +71,28 @@ export class AdminService {
         await AdminService.validateAndGetAdminTokenOwner(scalarToken);
 
         const client = new MatrixLiteClient(config.homeserver.accessToken);
+        const fedInfo = await getFederationConnInfo(config.homeserver.name);
         return {
             admins: config.admins,
             widgetBlacklist: config.widgetBlacklist,
             homeserver: {
                 name: config.homeserver.name,
                 userId: await client.whoAmI(),
-                federationUrl: await getFederationUrl(config.homeserver.name),
+                federationUrl: fedInfo.url,
+                federationHostname: fedInfo.hostname,
                 clientServerUrl: config.homeserver.clientServerUrl,
             },
         };
     }
+
+    @GET
+    @Path("test/federation")
+    public async testFederationRouting(@QueryParam("scalar_token") scalarToken: string, @QueryParam("server_name") serverName: string): Promise<any> {
+        await AdminService.validateAndGetAdminTokenOwner(scalarToken);
+
+        return {
+            inputServerName: serverName,
+            resolvedServer: await getFederationConnInfo(serverName),
+        };
+    }
 }

src/matrix/helpers.ts

@@ -3,14 +3,26 @@ import { LogService } from "matrix-js-snippets";
 import { Cache, CACHE_FEDERATION } from "../MemoryCache";
 import * as request from "request";
 import config from "../config";
+import splitHost from 'split-host';
+import * as isIP from "isipaddress";
+import * as requestPromise from "request-promise";
 
-export async function getFederationUrl(serverName: string): Promise<string> {
+export interface IFederationConnectionInfo {
+    hostname: string;
+    url: string;
+}
+
+export async function getFederationConnInfo(serverName: string): Promise<IFederationConnectionInfo> {
+    const expirationMs = 2 * 60 * 60 * 1000; // 2 hours
+
+    // Check to see if we've cached the hostname at all already
     const cachedUrl = Cache.for(CACHE_FEDERATION).get(serverName);
     if (cachedUrl) {
-        LogService.verbose("matrix", "Cached federation URL for " + serverName + " is " + cachedUrl);
+        LogService.verbose("matrix", "Cached federation URL for " + serverName + " is " + cachedUrl.url);
         return cachedUrl;
     }
 
+    // Rely on the configuration for a federation URL if we can
     if (serverName === config.homeserver.name && config.homeserver.federationUrl) {
         let url = config.homeserver.federationUrl;
         if (url.endsWith("/")) {
@@ -18,50 +30,137 @@ export async function getFederationUrl(serverName: string): Promise<string> {
         }
 
         LogService.info("matrix", "Using configured federation URL for " + serverName);
-        Cache.for(CACHE_FEDERATION).put(serverName, url);
+        const fedObj = {url, hostname: serverName};
-        return url;
+        Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+        return fedObj;
     }
 
-    let serverUrl = null;
+    // Dev note: The remainder of this is largely transcribed from matrix-media-repo
-    let expirationMs = 4 * 60 * 60 * 1000; // default is 4 hours
 
+    const hp = splitHost(serverName);
+    if (!hp.host) throw new Error("No hostname provided");
+    let defaultPort = false;
+    if (!hp.port) {
+        defaultPort = true;
+        hp.port = 8448;
+    }
+
+    // Step 1 of the discovery process: if the hostname is an IP, use that with explicit or default port
+    if (isIP.test(hp.host)) {
+        const fedUrl = `https://${hp.host}:${hp.port}`;
+        const fedObj = {url: fedUrl, hostname: serverName};
+        Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+        LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (IP address)`);
+        return fedObj;
+    }
+
+    // Step 2: if the hostname is not an IP address, and an explicit port is given, use that
+    if (!defaultPort) {
+        const fedUrl = `https://${hp.host}:${hp.port}`;
+        const fedObj = {url: fedUrl, hostname: hp.host};
+        Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+        LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (explicit port)`);
+        return fedObj;
+    }
+
+    // Step 3: if the hostname is not an IP address and no explicit port is given, do .well-known
     try {
-        const records = await dns.resolveSrv("_matrix._tcp." + serverName);
+        let result = await requestPromise(`https://${hp.host}/.well-known/matrix/server`);
-        if (records && records.length > 0) {
+        if (typeof (result) === 'string') result = JSON.parse(result);
-            serverUrl = "https://" + records[0].name + ":" + records[0].port;
+        const wkServerAddr = result['m.server'];
-            expirationMs = records[0].ttl * 1000;
+        if (wkServerAddr) {
+            const wkHp = splitHost(wkServerAddr);
+            if (!wkHp.host) {
+                // noinspection ExceptionCaughtLocallyJS
+                throw new Error("No hostname provided for m.server");
+            }
+            let wkDefaultPort = false;
+            if (!wkHp.port) {
+                wkDefaultPort = true;
+                wkHp.port = 8448;
+            }
+
+            // Step 3a: if the delegated host is an IP address, use that (regardless of port)
+            if (isIP.test(wkHp.host)) {
+                const fedUrl = `https://${wkHp.host}:${wkHp.port}`;
+                const fedObj = {url: fedUrl, hostname: wkServerAddr};
+                Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+                LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (WK; IP address)`);
+                return fedObj;
+            }
+
+            // Step 3b: if the delegated host is not an IP and an explicit port is given, use that
+            if (!wkDefaultPort) {
+                const fedUrl = `https://${wkHp.host}:${wkHp.port}`;
+                const fedObj = {url: fedUrl, hostname: wkHp.host};
+                Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+                LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (WK; explicit port)`);
+                return fedObj;
+            }
+
+            // Step 3c: if the delegated host is not an IP and doesn't have a port, start a SRV lookup and use that
+            try {
+                const records = await dns.resolveSrv("_matrix._tcp." + hp.host);
+                if (records && records.length > 0) {
+                    const fedUrl = `https://${records[0].name}:${records[0].port}`;
+                    const fedObj = {url: fedUrl, hostname: wkHp.host};
+                    Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+                    LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (WK; SRV)`);
+                    return fedObj;
+                }
+            } catch (e) {
+                LogService.warn("matrix", "Non-fatal error looking up .well-known SRV for " + serverName);
+                LogService.warn("matrix", e);
+            }
+
+            // Step 3d: use the delegated host as-is
+            const fedUrl = `https://${wkHp.host}:${wkHp.port}`;
+            const fedObj = {url: fedUrl, hostname: wkHp.host};
+            Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+            LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (WK; fallback)`);
+            return fedObj;
         }
-    } catch (err) {
+    } catch (e) {
-        // Not having the SRV record isn't bad, it just means that the server operator decided to not use SRV records.
+        LogService.warn("matrix", "Non-fatal error looking up .well-known for " + serverName);
-        // When there's no SRV record we default to port 8448 (as per the federation rules) in the lower .then()
+        LogService.warn("matrix", e);
-        // People tend to think that the lack of an SRV record is bad, but in reality it's only a problem if one was set and
-        // it's not being found. Most people don't set up the SRV record, but some do.
-        LogService.verbose("matrix", err);
-        LogService.warn("matrix", "Could not find _matrix._tcp." + serverName + " DNS record. This is normal for most servers.");
     }
 
-    if (!(expirationMs > 0)) { // This is weird so we can catch NaN easier
+    // Step 4: try resolving a hostname using SRV records and use that
-        expirationMs = 4 * 60 * 60 * 1000;
+    try {
+        const records = await dns.resolveSrv("_matrix._tcp." + hp.host);
+        if (records && records.length > 0) {
+            const fedUrl = `https://${records[0].name}:${records[0].port}`;
+            const fedObj = {url: fedUrl, hostname: hp.host};
+            Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+            LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (SRV)`);
+            return fedObj;
+        }
+    } catch (e) {
+        LogService.warn("matrix", "Non-fatal error looking up SRV for " + serverName);
+        LogService.warn("matrix", e);
     }
 
-    if (!serverUrl) serverUrl = "https://" + serverName + ":8448";
+    // Step 5: use the target host as-is
-    LogService.verbose("matrix", "Federation URL for " + serverName + " is " + serverUrl + " - caching for " + expirationMs + " ms");
+    const fedUrl = `https://${hp.host}:${hp.port}`;
-    Cache.for(CACHE_FEDERATION).put(serverName, serverUrl, expirationMs);
+    const fedObj = {url: fedUrl, hostname: hp.host};
-    return serverUrl;
+    Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
+    LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (SRV)`);
+    return fedObj;
 }
 
 export async function doFederatedApiCall(method: string, serverName: string, endpoint: string, query?: object, body?: object): Promise<any> {
-    const federationUrl = await getFederationUrl(serverName);
+    const federationInfo = await getFederationConnInfo(serverName);
-    LogService.info("matrix", "Doing federated API call: " + federationUrl + endpoint);
+    LogService.info("matrix", "Doing federated API call: " + federationInfo.url + endpoint);
     return new Promise((resolve, reject) => {
         request({
             method: method,
-            url: federationUrl + endpoint,
+            url: federationInfo.url + endpoint,
             qs: query,
             json: body,
+            // TODO: Remove this for MSC1711 support
             rejectUnauthorized: false, // allow self signed certs (for federation)
             headers: {
-                "Host": serverName,
+                "Host": federationInfo.hostname,
             },
         }, (err, res, _body) => {
             if (err) {
@@ -72,7 +171,7 @@ export async function doFederatedApiCall(method: string, serverName: string, end
                 LogService.error("matrix", "Got status code " + res.statusCode + " while calling federated endpoint " + endpoint);
                 reject(new Error("Error in request: invalid status code"));
             } else {
-                if (typeof(res.body) === "string") res.body = JSON.parse(res.body);
+                if (typeof (res.body) === "string") res.body = JSON.parse(res.body);
                 resolve(res.body);
             }
         });
@@ -109,7 +208,7 @@ export async function doClientApiCall(method: string, endpoint: string, query?:
                 LogService.error("matrix", res.body);
                 reject(new Error("Error in request: invalid status code"));
             } else {
-                if (typeof(res.body) === "string") res.body = JSON.parse(res.body);
+                if (typeof (res.body) === "string") res.body = JSON.parse(res.body);
                 resolve(res.body);
             }
         });

web/app/admin/home/home.component.html

@@ -23,6 +23,7 @@
                     <strong>Homeserver</strong><br />
                     Name: {{ config.homeserver.name }}<br />
                     Federation URL: {{ config.homeserver.federationUrl }}<br />
+                    Federation Hostname: {{ config.homeserver.federationHostname }}<br />
                     Client/Server URL: {{ config.homeserver.clientServerUrl }}<br />
                     Utility User ID: {{ config.homeserver.userId }}
                 </div>

web/app/shared/models/admin-responses.ts

@@ -7,6 +7,7 @@ export interface FE_DimensionConfig {
         name: string;
         userId: string;
         federationUrl: string;
+        federationHostname: string;
         clientServerUrl: string;
     };
 }