Support MSC1708 (and co.) and prepare for MSC1711

Fixes https://github.com/turt2live/matrix-dimension/issues/234

Later support for MSC1711 will be done in https://github.com/turt2live/matrix-dimension/issues/238
This commit is contained in:
Travis Ralston 2019-02-07 21:17:50 -07:00
parent 3db2896d76
commit 38ea8d30db
6 changed files with 209 additions and 55 deletions

82
package-lock.json generated
View File

@ -4379,14 +4379,12 @@
"balanced-match": {
"version": "1.0.0",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"brace-expansion": {
"version": "1.1.11",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"balanced-match": "^1.0.0",
"concat-map": "0.0.1"
@ -4401,20 +4399,17 @@
"code-point-at": {
"version": "1.1.0",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"concat-map": {
"version": "0.0.1",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"console-control-strings": {
"version": "1.1.0",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"core-util-is": {
"version": "1.0.2",
@ -4531,8 +4526,7 @@
"inherits": {
"version": "2.0.3",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"ini": {
"version": "1.3.5",
@ -4544,7 +4538,6 @@
"version": "1.0.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"number-is-nan": "^1.0.0"
}
@ -4559,7 +4552,6 @@
"version": "3.0.4",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"brace-expansion": "^1.1.7"
}
@ -4567,14 +4559,12 @@
"minimist": {
"version": "0.0.8",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"minipass": {
"version": "2.2.4",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"safe-buffer": "^5.1.1",
"yallist": "^3.0.0"
@ -4593,7 +4583,6 @@
"version": "0.5.1",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"minimist": "0.0.8"
}
@ -4674,8 +4663,7 @@
"number-is-nan": {
"version": "1.0.1",
"bundled": true,
"dev": true,
"optional": true
"dev": true
},
"object-assign": {
"version": "4.1.1",
@ -4687,7 +4675,6 @@
"version": "1.4.0",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"wrappy": "1"
}
@ -4809,7 +4796,6 @@
"version": "1.0.2",
"bundled": true,
"dev": true,
"optional": true,
"requires": {
"code-point-at": "^1.0.0",
"is-fullwidth-code-point": "^1.0.0",
@ -5672,8 +5658,7 @@
"ip-regex": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/ip-regex/-/ip-regex-2.1.0.tgz",
"integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk=",
"dev": true
"integrity": "sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk="
},
"ipaddr.js": {
"version": "1.8.0",
@ -5948,6 +5933,11 @@
"integrity": "sha1-6PvzdNxVb/iUehDcsFctYz8s+hA=",
"dev": true
},
"isipaddress": {
"version": "0.0.2",
"resolved": "https://registry.npmjs.org/isipaddress/-/isipaddress-0.0.2.tgz",
"integrity": "sha1-qeRIRlEGrwHmCFHPI146wwEUUNM="
},
"isnumeric": {
"version": "0.2.0",
"resolved": "https://registry.npmjs.org/isnumeric/-/isnumeric-0.2.0.tgz",
@ -11018,6 +11008,25 @@
}
}
},
"request-promise": {
"version": "4.2.2",
"resolved": "https://registry.npmjs.org/request-promise/-/request-promise-4.2.2.tgz",
"integrity": "sha1-0epG1lSm7k+O5qT+oQGMIpEZBLQ=",
"requires": {
"bluebird": "^3.5.0",
"request-promise-core": "1.1.1",
"stealthy-require": "^1.1.0",
"tough-cookie": ">=2.3.3"
}
},
"request-promise-core": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz",
"integrity": "sha1-Pu4AssWqgyOc+wTFcA2jb4HNCLY=",
"requires": {
"lodash": "^4.13.1"
}
},
"require-dir-all": {
"version": "0.4.15",
"resolved": "https://registry.npmjs.org/require-dir-all/-/require-dir-all-0.4.15.tgz",
@ -12003,6 +12012,11 @@
"integrity": "sha1-kPn0ZqIOjjnvJNqVnB5hHCow3VQ=",
"dev": true
},
"split-host": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/split-host/-/split-host-0.1.1.tgz",
"integrity": "sha512-nrlaPJMHkr3hKx7aCyr+S0OgUvAm/xKzWWMHej0IsMamWjRC52Fv+NGZwuqRE1lyu1iNWCmcrpZ1S1qvk+Uiwg=="
},
"split-string": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/split-string/-/split-string-3.1.0.tgz",
@ -12153,6 +12167,11 @@
"readable-stream": "^2.0.1"
}
},
"stealthy-require": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/stealthy-require/-/stealthy-require-1.1.1.tgz",
"integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks="
},
"stream-browserify": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/stream-browserify/-/stream-browserify-2.0.1.tgz",
@ -12758,6 +12777,23 @@
"resolved": "https://registry.npmjs.org/toposort-class/-/toposort-class-1.0.1.tgz",
"integrity": "sha1-f/0feMi+KMO6Rc1OGj9e4ZO9mYg="
},
"tough-cookie": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-3.0.1.tgz",
"integrity": "sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg==",
"requires": {
"ip-regex": "^2.1.0",
"psl": "^1.1.28",
"punycode": "^2.1.1"
},
"dependencies": {
"punycode": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-2.1.1.tgz",
"integrity": "sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A=="
}
}
},
"trim-newlines": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/trim-newlines/-/trim-newlines-1.0.0.tgz",

View File

@ -27,6 +27,7 @@
"dns-then": "^0.1.0",
"express": "^4.16.4",
"git-rev-sync": "^1.12.0",
"isipaddress": "0.0.2",
"js-yaml": "^3.12.0",
"lodash": "^4.17.5",
"matrix-js-snippets": "^0.2.8",
@ -36,10 +37,12 @@
"netmask": "^1.0.6",
"random-string": "^0.2.0",
"request": "^2.88.0",
"request-promise": "^4.2.2",
"require-dir-all": "^0.4.15",
"sequelize": "^4.39.1",
"sequelize-typescript": "^0.6.6",
"sharp": "^0.21.1",
"split-host": "^0.1.1",
"spotify-uri": "^1.0.0",
"sqlite3": "^4.0.4",
"telegraf": "^3.25.5",

View File

@ -4,7 +4,7 @@ import config from "../../config";
import { ApiError } from "../ApiError";
import { MatrixLiteClient } from "../../matrix/MatrixLiteClient";
import { CURRENT_VERSION } from "../../version";
import { getFederationUrl } from "../../matrix/helpers";
import { getFederationConnInfo } from "../../matrix/helpers";
interface DimensionVersionResponse {
version: string;
@ -17,6 +17,7 @@ interface DimensionConfigResponse {
name: string;
userId: string;
federationUrl: string;
federationHostname: string;
clientServerUrl: string;
};
}
@ -70,15 +71,28 @@ export class AdminService {
await AdminService.validateAndGetAdminTokenOwner(scalarToken);
const client = new MatrixLiteClient(config.homeserver.accessToken);
const fedInfo = await getFederationConnInfo(config.homeserver.name);
return {
admins: config.admins,
widgetBlacklist: config.widgetBlacklist,
homeserver: {
name: config.homeserver.name,
userId: await client.whoAmI(),
federationUrl: await getFederationUrl(config.homeserver.name),
federationUrl: fedInfo.url,
federationHostname: fedInfo.hostname,
clientServerUrl: config.homeserver.clientServerUrl,
},
};
}
@GET
@Path("test/federation")
public async testFederationRouting(@QueryParam("scalar_token") scalarToken: string, @QueryParam("server_name") serverName: string): Promise<any> {
await AdminService.validateAndGetAdminTokenOwner(scalarToken);
return {
inputServerName: serverName,
resolvedServer: await getFederationConnInfo(serverName),
};
}
}

View File

@ -3,14 +3,26 @@ import { LogService } from "matrix-js-snippets";
import { Cache, CACHE_FEDERATION } from "../MemoryCache";
import * as request from "request";
import config from "../config";
import splitHost from 'split-host';
import * as isIP from "isipaddress";
import * as requestPromise from "request-promise";
export async function getFederationUrl(serverName: string): Promise<string> {
export interface IFederationConnectionInfo {
hostname: string;
url: string;
}
export async function getFederationConnInfo(serverName: string): Promise<IFederationConnectionInfo> {
const expirationMs = 2 * 60 * 60 * 1000; // 2 hours
// Check to see if we've cached the hostname at all already
const cachedUrl = Cache.for(CACHE_FEDERATION).get(serverName);
if (cachedUrl) {
LogService.verbose("matrix", "Cached federation URL for " + serverName + " is " + cachedUrl);
LogService.verbose("matrix", "Cached federation URL for " + serverName + " is " + cachedUrl.url);
return cachedUrl;
}
// Rely on the configuration for a federation URL if we can
if (serverName === config.homeserver.name && config.homeserver.federationUrl) {
let url = config.homeserver.federationUrl;
if (url.endsWith("/")) {
@ -18,50 +30,137 @@ export async function getFederationUrl(serverName: string): Promise<string> {
}
LogService.info("matrix", "Using configured federation URL for " + serverName);
Cache.for(CACHE_FEDERATION).put(serverName, url);
return url;
const fedObj = {url, hostname: serverName};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
return fedObj;
}
let serverUrl = null;
let expirationMs = 4 * 60 * 60 * 1000; // default is 4 hours
// Dev note: The remainder of this is largely transcribed from matrix-media-repo
const hp = splitHost(serverName);
if (!hp.host) throw new Error("No hostname provided");
let defaultPort = false;
if (!hp.port) {
defaultPort = true;
hp.port = 8448;
}
// Step 1 of the discovery process: if the hostname is an IP, use that with explicit or default port
if (isIP.test(hp.host)) {
const fedUrl = `https://${hp.host}:${hp.port}`;
const fedObj = {url: fedUrl, hostname: serverName};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (IP address)`);
return fedObj;
}
// Step 2: if the hostname is not an IP address, and an explicit port is given, use that
if (!defaultPort) {
const fedUrl = `https://${hp.host}:${hp.port}`;
const fedObj = {url: fedUrl, hostname: hp.host};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (explicit port)`);
return fedObj;
}
// Step 3: if the hostname is not an IP address and no explicit port is given, do .well-known
try {
const records = await dns.resolveSrv("_matrix._tcp." + serverName);
if (records && records.length > 0) {
serverUrl = "https://" + records[0].name + ":" + records[0].port;
expirationMs = records[0].ttl * 1000;
let result = await requestPromise(`https://${hp.host}/.well-known/matrix/server`);
if (typeof (result) === 'string') result = JSON.parse(result);
const wkServerAddr = result['m.server'];
if (wkServerAddr) {
const wkHp = splitHost(wkServerAddr);
if (!wkHp.host) {
// noinspection ExceptionCaughtLocallyJS
throw new Error("No hostname provided for m.server");
}
let wkDefaultPort = false;
if (!wkHp.port) {
wkDefaultPort = true;
wkHp.port = 8448;
}
// Step 3a: if the delegated host is an IP address, use that (regardless of port)
if (isIP.test(wkHp.host)) {
const fedUrl = `https://${wkHp.host}:${wkHp.port}`;
const fedObj = {url: fedUrl, hostname: wkServerAddr};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (WK; IP address)`);
return fedObj;
}
// Step 3b: if the delegated host is not an IP and an explicit port is given, use that
if (!wkDefaultPort) {
const fedUrl = `https://${wkHp.host}:${wkHp.port}`;
const fedObj = {url: fedUrl, hostname: wkHp.host};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (WK; explicit port)`);
return fedObj;
}
// Step 3c: if the delegated host is not an IP and doesn't have a port, start a SRV lookup and use that
try {
const records = await dns.resolveSrv("_matrix._tcp." + hp.host);
if (records && records.length > 0) {
const fedUrl = `https://${records[0].name}:${records[0].port}`;
const fedObj = {url: fedUrl, hostname: wkHp.host};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (WK; SRV)`);
return fedObj;
}
} catch (e) {
LogService.warn("matrix", "Non-fatal error looking up .well-known SRV for " + serverName);
LogService.warn("matrix", e);
}
// Step 3d: use the delegated host as-is
const fedUrl = `https://${wkHp.host}:${wkHp.port}`;
const fedObj = {url: fedUrl, hostname: wkHp.host};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (WK; fallback)`);
return fedObj;
}
} catch (err) {
// Not having the SRV record isn't bad, it just means that the server operator decided to not use SRV records.
// When there's no SRV record we default to port 8448 (as per the federation rules) in the lower .then()
// People tend to think that the lack of an SRV record is bad, but in reality it's only a problem if one was set and
// it's not being found. Most people don't set up the SRV record, but some do.
LogService.verbose("matrix", err);
LogService.warn("matrix", "Could not find _matrix._tcp." + serverName + " DNS record. This is normal for most servers.");
} catch (e) {
LogService.warn("matrix", "Non-fatal error looking up .well-known for " + serverName);
LogService.warn("matrix", e);
}
if (!(expirationMs > 0)) { // This is weird so we can catch NaN easier
expirationMs = 4 * 60 * 60 * 1000;
// Step 4: try resolving a hostname using SRV records and use that
try {
const records = await dns.resolveSrv("_matrix._tcp." + hp.host);
if (records && records.length > 0) {
const fedUrl = `https://${records[0].name}:${records[0].port}`;
const fedObj = {url: fedUrl, hostname: hp.host};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (SRV)`);
return fedObj;
}
} catch (e) {
LogService.warn("matrix", "Non-fatal error looking up SRV for " + serverName);
LogService.warn("matrix", e);
}
if (!serverUrl) serverUrl = "https://" + serverName + ":8448";
LogService.verbose("matrix", "Federation URL for " + serverName + " is " + serverUrl + " - caching for " + expirationMs + " ms");
Cache.for(CACHE_FEDERATION).put(serverName, serverUrl, expirationMs);
return serverUrl;
// Step 5: use the target host as-is
const fedUrl = `https://${hp.host}:${hp.port}`;
const fedObj = {url: fedUrl, hostname: hp.host};
Cache.for(CACHE_FEDERATION).put(serverName, fedObj, expirationMs);
LogService.info("matrix", `Federation URL for ${serverName} is ${fedUrl} (SRV)`);
return fedObj;
}
export async function doFederatedApiCall(method: string, serverName: string, endpoint: string, query?: object, body?: object): Promise<any> {
const federationUrl = await getFederationUrl(serverName);
LogService.info("matrix", "Doing federated API call: " + federationUrl + endpoint);
const federationInfo = await getFederationConnInfo(serverName);
LogService.info("matrix", "Doing federated API call: " + federationInfo.url + endpoint);
return new Promise((resolve, reject) => {
request({
method: method,
url: federationUrl + endpoint,
url: federationInfo.url + endpoint,
qs: query,
json: body,
// TODO: Remove this for MSC1711 support
rejectUnauthorized: false, // allow self signed certs (for federation)
headers: {
"Host": serverName,
"Host": federationInfo.hostname,
},
}, (err, res, _body) => {
if (err) {
@ -72,7 +171,7 @@ export async function doFederatedApiCall(method: string, serverName: string, end
LogService.error("matrix", "Got status code " + res.statusCode + " while calling federated endpoint " + endpoint);
reject(new Error("Error in request: invalid status code"));
} else {
if (typeof(res.body) === "string") res.body = JSON.parse(res.body);
if (typeof (res.body) === "string") res.body = JSON.parse(res.body);
resolve(res.body);
}
});
@ -109,9 +208,9 @@ export async function doClientApiCall(method: string, endpoint: string, query?:
LogService.error("matrix", res.body);
reject(new Error("Error in request: invalid status code"));
} else {
if (typeof(res.body) === "string") res.body = JSON.parse(res.body);
if (typeof (res.body) === "string") res.body = JSON.parse(res.body);
resolve(res.body);
}
});
});
}
}

View File

@ -23,6 +23,7 @@
<strong>Homeserver</strong><br />
Name: {{ config.homeserver.name }}<br />
Federation URL: {{ config.homeserver.federationUrl }}<br />
Federation Hostname: {{ config.homeserver.federationHostname }}<br />
Client/Server URL: {{ config.homeserver.clientServerUrl }}<br />
Utility User ID: {{ config.homeserver.userId }}
</div>

View File

@ -7,6 +7,7 @@ export interface FE_DimensionConfig {
name: string;
userId: string;
federationUrl: string;
federationHostname: string;
clientServerUrl: string;
};
}