From ff088e3e490443848f5d98fdf5cabb8ca4a455dd Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Fri, 30 Mar 2018 16:34:39 -0600 Subject: [PATCH] Add a whole lot more logging to the backend --- src/MemoryCache.ts | 2 ++ src/api/Webserver.ts | 2 +- src/api/admin/AdminAppserviceService.ts | 3 ++- src/api/admin/AdminIntegrationsService.ts | 7 +++++-- src/api/admin/AdminNebService.ts | 18 ++++++++++++++---- src/api/admin/AdminUpstreamService.ts | 4 +++- src/api/dimension/DimensionHealthService.ts | 2 ++ src/api/scalar/ScalarService.ts | 1 + src/db/AppserviceStore.ts | 14 +++++++++----- src/matrix/helpers.ts | 2 ++ src/neb/NebClient.ts | 1 + src/neb/NebProxy.ts | 2 +- src/scalar/ScalarClient.ts | 1 + src/version.ts | 14 +++++++++++--- 14 files changed, 55 insertions(+), 18 deletions(-) diff --git a/src/MemoryCache.ts b/src/MemoryCache.ts index 6453a26..a0b81f7 100644 --- a/src/MemoryCache.ts +++ b/src/MemoryCache.ts @@ -1,4 +1,5 @@ import * as memoryCache from "memory-cache"; +import { LogService } from "matrix-js-snippets"; export class MemoryCache { @@ -30,6 +31,7 @@ class _CacheManager { public for(namespace: string): MemoryCache { let cache = this.caches[namespace]; if (!cache) { + LogService.info("MemoryCache", "Creating a new cache for namespace: " + namespace); cache = this.caches[namespace] = new MemoryCache(); } diff --git a/src/api/Webserver.ts b/src/api/Webserver.ts index 2394706..669b12b 100644 --- a/src/api/Webserver.ts +++ b/src/api/Webserver.ts @@ -62,7 +62,7 @@ export default class Webserver { parsedUrl.query["scalar_token"] = "redacted"; parsedUrl.search = undefined; // to force URL.format to use `query` } - LogService.verbose("Webserver", "Incoming request: " + req.method + " " + URL.format(parsedUrl)); + LogService.info("Webserver", "Incoming request: " + req.method + " " + URL.format(parsedUrl)); next(); }); this.app.use((_req, res, next) => { diff --git a/src/api/admin/AdminAppserviceService.ts b/src/api/admin/AdminAppserviceService.ts index ad4ca05..d3a2dcc 100644 --- a/src/api/admin/AdminAppserviceService.ts +++ b/src/api/admin/AdminAppserviceService.ts @@ -47,7 +47,7 @@ export class AdminAppserviceService { @POST @Path("new") public async createAppservice(@QueryParam("scalar_token") scalarToken: string, request: AppserviceCreateRequest): Promise { - await AdminService.validateAndGetAdminTokenOwner(scalarToken); + const userId = await AdminService.validateAndGetAdminTokenOwner(scalarToken); // Trim off the @ sign if it's on the prefix if (request.userPrefix[0] === "@") { @@ -60,6 +60,7 @@ export class AdminAppserviceService { } const appservice = await AppserviceStore.create(AppserviceStore.getSafeUserId(request.userPrefix)); + LogService.info("AdminAppserviceService", userId + " created an application service"); return this.mapAppservice(appservice); } diff --git a/src/api/admin/AdminIntegrationsService.ts b/src/api/admin/AdminIntegrationsService.ts index 402172e..9c9174d 100644 --- a/src/api/admin/AdminIntegrationsService.ts +++ b/src/api/admin/AdminIntegrationsService.ts @@ -5,6 +5,7 @@ import { DimensionIntegrationsService } from "../dimension/DimensionIntegrations import { WidgetStore } from "../../db/WidgetStore"; import { Cache, CACHE_INTEGRATIONS } from "../../MemoryCache"; import { Integration } from "../../integrations/Integration"; +import { LogService } from "matrix-js-snippets"; interface SetEnabledRequest { enabled: boolean; @@ -24,11 +25,12 @@ export class AdminIntegrationsService { @POST @Path(":category/:type/options") public async setOptions(@QueryParam("scalar_token") scalarToken: string, @PathParam("category") category: string, @PathParam("type") type: string, body: SetOptionsRequest): Promise { - await AdminService.validateAndGetAdminTokenOwner(scalarToken); + const userId = await AdminService.validateAndGetAdminTokenOwner(scalarToken); if (category === "widget") await WidgetStore.setOptions(type, body.options); else throw new ApiError(400, "Unrecognized category"); + LogService.info("AdminIntegrationsService", userId + " updated the integration options for " + category + "/" + type); Cache.for(CACHE_INTEGRATIONS).clear(); return {}; // 200 OK } @@ -37,11 +39,12 @@ export class AdminIntegrationsService { @POST @Path(":category/:type/enabled") public async setEnabled(@QueryParam("scalar_token") scalarToken: string, @PathParam("category") category: string, @PathParam("type") type: string, body: SetEnabledRequest): Promise { - await AdminService.validateAndGetAdminTokenOwner(scalarToken); + const userId = await AdminService.validateAndGetAdminTokenOwner(scalarToken); if (category === "widget") await WidgetStore.setEnabled(type, body.enabled); else throw new ApiError(400, "Unrecognized category"); + LogService.info("AdminIntegrationsService", userId + " set " + category + "/" + type + " to " + (body.enabled ? "enabled" : "disabled")); Cache.for(CACHE_INTEGRATIONS).clear(); return {}; // 200 OK } diff --git a/src/api/admin/AdminNebService.ts b/src/api/admin/AdminNebService.ts index 81e8bb5..55c795a 100644 --- a/src/api/admin/AdminNebService.ts +++ b/src/api/admin/AdminNebService.ts @@ -51,8 +51,11 @@ export class AdminNebService { @POST @Path(":id/integration/:type/enabled") public async setIntegrationEnabled(@QueryParam("scalar_token") scalarToken: string, @PathParam("id") nebId: number, @PathParam("type") integrationType: string, request: SetEnabledRequest): Promise { - await AdminService.validateAndGetAdminTokenOwner(scalarToken); + const userId = await AdminService.validateAndGetAdminTokenOwner(scalarToken); + await NebStore.setIntegrationEnabled(nebId, integrationType, request.enabled); + LogService.info("AdminNebService", userId + " set the " + integrationType + " on NEB " + nebId + " to " + (request.enabled ? "enabled" : "disabled")); + Cache.for(CACHE_NEB).clear(); Cache.for(CACHE_INTEGRATIONS).clear(); return {}; // 200 OK @@ -61,8 +64,11 @@ export class AdminNebService { @POST @Path(":id/integration/:type/config") public async setIntegrationConfig(@QueryParam("scalar_token") scalarToken: string, @PathParam("id") nebId: number, @PathParam("type") integrationType: string, newConfig: any): Promise { - await AdminService.validateAndGetAdminTokenOwner(scalarToken); + const userId = await AdminService.validateAndGetAdminTokenOwner(scalarToken); + await NebStore.setIntegrationConfig(nebId, integrationType, newConfig); + LogService.info("AdminNebService", userId + " updated the configuration for " + integrationType + " on NEB " + nebId); + Cache.for(CACHE_NEB).clear(); Cache.for(CACHE_INTEGRATIONS).clear(); return {}; // 200 OK @@ -78,10 +84,12 @@ export class AdminNebService { @POST @Path("new/upstream") public async newConfigForUpstream(@QueryParam("scalar_token") scalarToken: string, request: CreateWithUpstream): Promise { - await AdminService.validateAndGetAdminTokenOwner(scalarToken); + const userId = await AdminService.validateAndGetAdminTokenOwner(scalarToken); try { const neb = await NebStore.createForUpstream(request.upstreamId); + LogService.info("AdminNebService", userId + " created a new NEB instance from upstream " + request.upstreamId); + Cache.for(CACHE_NEB).clear(); Cache.for(CACHE_INTEGRATIONS).clear(); return neb; @@ -94,10 +102,12 @@ export class AdminNebService { @POST @Path("new/appservice") public async newConfigForAppservice(@QueryParam("scalar_token") scalarToken: string, request: CreateWithAppservice): Promise { - await AdminService.validateAndGetAdminTokenOwner(scalarToken); + const userId = await AdminService.validateAndGetAdminTokenOwner(scalarToken); try { const neb = await NebStore.createForAppservice(request.appserviceId, request.adminUrl); + LogService.info("AdminNebService", userId + " created a new NEB instance from appservice " + request.appserviceId); + Cache.for(CACHE_NEB).clear(); Cache.for(CACHE_INTEGRATIONS).clear(); return neb; diff --git a/src/api/admin/AdminUpstreamService.ts b/src/api/admin/AdminUpstreamService.ts index 36ab43d..5c9d994 100644 --- a/src/api/admin/AdminUpstreamService.ts +++ b/src/api/admin/AdminUpstreamService.ts @@ -2,6 +2,7 @@ import { GET, Path, POST, QueryParam } from "typescript-rest"; import { AdminService } from "./AdminService"; import { Cache, CACHE_UPSTREAM } from "../../MemoryCache"; import Upstream from "../../db/models/Upstream"; +import { LogService } from "matrix-js-snippets"; interface UpstreamRepsonse { id: number; @@ -42,7 +43,7 @@ export class AdminUpstreamService { @POST @Path("new") public async createUpstream(@QueryParam("scalar_token") scalarToken: string, request: NewUpstreamRequest): Promise { - await AdminService.validateAndGetAdminTokenOwner(scalarToken); + const userId = await AdminService.validateAndGetAdminTokenOwner(scalarToken); const upstream = await Upstream.create({ name: request.name, @@ -51,6 +52,7 @@ export class AdminUpstreamService { apiUrl: request.apiUrl, }); + LogService.info("AdminUpstreamService", userId + " created a new upstream"); Cache.for(CACHE_UPSTREAM).clear(); return this.mapUpstream(upstream); } diff --git a/src/api/dimension/DimensionHealthService.ts b/src/api/dimension/DimensionHealthService.ts index 00cf8c7..7cda207 100644 --- a/src/api/dimension/DimensionHealthService.ts +++ b/src/api/dimension/DimensionHealthService.ts @@ -1,4 +1,5 @@ import { GET, Path } from "typescript-rest"; +import { LogService } from "matrix-js-snippets"; /** * API for the health of Dimension @@ -9,6 +10,7 @@ export class DimensionHealthService { @GET @Path("heartbeat") public async heartbeat(): Promise { + LogService.info("DimensionHealthService", "Heartbeat called"); return {}; // 200 OK } } \ No newline at end of file diff --git a/src/api/scalar/ScalarService.ts b/src/api/scalar/ScalarService.ts index ff6be08..2bfc524 100644 --- a/src/api/scalar/ScalarService.ts +++ b/src/api/scalar/ScalarService.ts @@ -84,6 +84,7 @@ export class ScalarService { isDimensionToken: true, }); + LogService.info("ScalarService", mxUserId + " has registered for a scalar token successfully"); return {scalar_token: dimensionScalarToken.scalarToken}; } diff --git a/src/db/AppserviceStore.ts b/src/db/AppserviceStore.ts index c4fe326..2a16d60 100644 --- a/src/db/AppserviceStore.ts +++ b/src/db/AppserviceStore.ts @@ -2,6 +2,7 @@ import AppService from "./models/AppService"; import AppServiceUser from "./models/AppServiceUser"; import * as randomString from "random-string"; import { MatrixAppserviceClient } from "../matrix/MatrixAppserviceClient"; +import { LogService } from "matrix-js-snippets"; export class AppserviceStore { @@ -37,22 +38,25 @@ export class AppserviceStore { public static getSafeUserId(userIdOrPrefix: string): string { // Force user IDs to be lowercase and strip out characters that aren't permitted // https://matrix.org/docs/spec/appendices.html#user-identifiers + if (userIdOrPrefix.startsWith('@')) { + // we only change out the parts we care about. The rest will be crushed down. + userIdOrPrefix = userIdOrPrefix.replace(/@/g, '=40'); + userIdOrPrefix = userIdOrPrefix.replace(/:/g, '=3A'); + } return userIdOrPrefix.toLowerCase().replace(/[^a-z0-9._\-=]/g, '.'); } - public static async getUsers(appserviceId: string): Promise { - return AppServiceUser.findAll({where: {appserviceId: appserviceId}}); - } - public static async registerUser(appserviceId: string, userId: string): Promise { const appservice = await AppService.findOne({where: {id: appserviceId}}); if (!appservice) throw new Error("Appservice not found"); + LogService.info("AppserviceStore", "Registering to own " + userId + " in appservice " + appserviceId); const client = new MatrixAppserviceClient(appservice); const localpart = AppserviceStore.getSafeUserId(userId.substring(1).split(":")[0]); const response = await client.registerUser(localpart); + LogService.info("AppserviceStore", "Successfully registered " + userId); - return AppServiceUser.create({ + return await AppServiceUser.create({ id: userId, appserviceId: appserviceId, accessToken: response.access_token, diff --git a/src/matrix/helpers.ts b/src/matrix/helpers.ts index 8a5fe4c..38c9663 100644 --- a/src/matrix/helpers.ts +++ b/src/matrix/helpers.ts @@ -48,6 +48,7 @@ export async function getFederationUrl(serverName: string): Promise { export async function doFederatedApiCall(method: string, serverName: string, endpoint: string, query?: object, body?: object): Promise { const federationUrl = await getFederationUrl(serverName); + LogService.info("matrix", "Doing federated API call: " + federationUrl + endpoint); return new Promise((resolve, reject) => { request({ method: method, @@ -74,6 +75,7 @@ export async function doFederatedApiCall(method: string, serverName: string, end export async function doClientApiCall(method: string, endpoint: string, query?: object, body?: object): Promise { let url = config.homeserver.clientServerUrl; if (url.endsWith("/")) url = url.substring(0, url.length - 1); + LogService.info("matrix", "Doing client API call: " + url + endpoint); return new Promise((resolve, reject) => { request({ diff --git a/src/neb/NebClient.ts b/src/neb/NebClient.ts index af8ae51..3621141 100644 --- a/src/neb/NebClient.ts +++ b/src/neb/NebClient.ts @@ -60,6 +60,7 @@ export class NebClient { private doRequest(endpoint: string, body: any): Promise { const adminUrl = (this.neb.adminUrl.endsWith("/") ? this.neb.adminUrl.substring(0, this.neb.adminUrl.length - 1) : this.neb.adminUrl); if (!endpoint.startsWith("/")) endpoint = "/" + endpoint; + LogService.info("NebClient", "Doing NEB call: " + adminUrl + endpoint); return new Promise((resolve, reject) => { request({ diff --git a/src/neb/NebProxy.ts b/src/neb/NebProxy.ts index 9a8f8b7..02ead02 100644 --- a/src/neb/NebProxy.ts +++ b/src/neb/NebProxy.ts @@ -27,7 +27,6 @@ interface InternalTravisCiConfig { export class NebProxy { constructor(private neb: NebConfig, private requestingUserId: string) { - } public async getBotUserId(integration: NebIntegration): Promise { @@ -324,6 +323,7 @@ export class NebProxy { const apiUrl = upstream.apiUrl.endsWith("/") ? upstream.apiUrl.substring(0, upstream.apiUrl.length - 1) : upstream.apiUrl; const url = apiUrl + (endpoint.startsWith("/") ? endpoint : "/" + endpoint); + LogService.info("NebProxy", "Doing upstream NEB request: " + url); return new Promise((resolve, reject) => { request({ diff --git a/src/scalar/ScalarClient.ts b/src/scalar/ScalarClient.ts index 4cc86b6..c042a8c 100644 --- a/src/scalar/ScalarClient.ts +++ b/src/scalar/ScalarClient.ts @@ -9,6 +9,7 @@ export class ScalarClient { } public register(openId: OpenId): Promise { + LogService.info("ScalarClient", "Doing upstream scalar request: " + this.upstream.scalarUrl + "/register"); return new Promise((resolve, reject) => { request({ method: "POST", diff --git a/src/version.ts b/src/version.ts index ec868fb..3a9a0d7 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,12 +1,20 @@ -import { LogService } from "matrix-js-snippets"; import * as git from "git-rev-sync"; let version = "Unknown"; +let gitHash = null; try { version = "v" + require("../../package.json").version; } catch (error) { - LogService.error("version", error); + // The log service isn't set up by the time we require this file + console.error("version", error); } -export const CURRENT_VERSION = version + "-" + git.short(); +try { + gitHash = git.short(); +} catch (error) { + // The log service isn't set up by the time we require this file + console.error("version", error); +} + +export const CURRENT_VERSION = version + (gitHash ? "-" + gitHash : "");