diff --git a/config/default.yaml b/config/default.yaml index 7a4287a..bd55489 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -79,6 +79,15 @@ stickers: # Whether or not to allow people to add custom sticker packs enabled: true + # Whether to resize images that are not 512x512 during import. This might slow down the import of animated Stickers. + resize: true + + # Whether to allow animated stickers. If set to false animations will be stripped from images. + allowAnimated: true + + # Whether to skip over images that are not 512x512. Ignored when resize is true. + verifyImageSize: true + # The sticker manager bot to promote stickerBot: "@stickers:t2bot.io" diff --git a/package.json b/package.json index c68f36c..e4911b5 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "semver": "^7.3.5", "sequelize": "^6.12.0-alpha.1", "sequelize-typescript": "^2.1.1", - "sharp": "^0.29.0", + "sharp": "^0.30.7", "split-host": "^0.1.1", "spotify-uri": "^2.2.0", "sqlite3": "^5.0.2", diff --git a/src/config.ts b/src/config.ts index fad170c..39bc6d9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,6 +36,9 @@ export interface DimensionConfig { }; stickers: { enabled: boolean; + resize: boolean; + allowAnimated: boolean; + verifyImageSize: boolean; stickerBot: string; managerUrl: string; }; diff --git a/src/matrix/MatrixLiteClient.ts b/src/matrix/MatrixLiteClient.ts index 939dd95..a85597d 100644 --- a/src/matrix/MatrixLiteClient.ts +++ b/src/matrix/MatrixLiteClient.ts @@ -36,6 +36,15 @@ export class MatrixLiteClient { return baseUrl + `/_matrix/media/r0/thumbnail/${serverName}/${contentId}?width=${width}&height=${height}&method=${method}&animated=${isAnimated}`; } + public async getMediaUrl(serverName: string, contentId: string): Promise { + let baseUrl = config.homeserver.mediaUrl; + if (!baseUrl) baseUrl = config.homeserver.clientServerUrl; + if (baseUrl.endsWith("/")) baseUrl = baseUrl.substring(0, baseUrl.length - 1); + + // DO NOT RETURN THE ACCESS TOKEN. + return baseUrl + `/_matrix/media/r0/download/${serverName}/${contentId}`; + } + public async whoAmI(): Promise { const response = await doClientApiCall( "GET", @@ -106,6 +115,7 @@ export class MatrixLiteClient { } public async upload(content: Buffer, contentType: string): Promise { + LogService.info("MatrixLiteClient", "Uploading file (type:" + contentType + ")"); return doClientApiCall( "POST", "/_matrix/media/r0/upload", @@ -126,6 +136,7 @@ export class MatrixLiteClient { method: "GET", url: url, encoding: null, + headers: {}, }, (err, res, _body) => { if (err) { LogService.error("MatrixLiteClient", "Error downloading file from " + url); @@ -140,4 +151,50 @@ export class MatrixLiteClient { }); }); } + + public async parseMediaMIME(url: string): Promise { + return new Promise((resolve, reject) => { + request({ + method: "GET", + url: url, + encoding: null, + headers: { + 'Range': 'bytes=0-32' + }, + }, (err, res, _body) => { + if (err) { + LogService.error("MatrixLiteClient", "Error downloading file from " + url); + LogService.error("MatrixLiteClient", err); + reject(err); + } else if (res.statusCode !== 200) { + if (res.statusCode !== 206) { + LogService.error("MatrixLiteClient", "Got status code " + res.statusCode + " while calling url " + url); + reject(new Error("Error in request: invalid status code")); + } + } else { + return this.parseFileHeaderMIME(res.body); + } + }); + }); + } + + public parseFileHeaderMIME(data: Buffer): string { + const s = data.slice(0,32); + if (s.slice(0,8).includes(Buffer.from("89504E470D0A1A0A", "hex"))) { + return("image/png"); + } else if (s.slice(0,3).includes(Buffer.from("474946", "hex"))) { + return("image/gif"); + } else if (s.slice(0,3).includes(Buffer.from("FFD8FF", "hex"))) { + return("image/jpeg"); + } else if (s.slice(0,3).includes(Buffer.from("000000", "hex")) && s.slice(4,8).includes(Buffer.from("66747970", "hex"))) { + if (s.slice(16,28).includes(Buffer.from("61766973", "hex"))) { + return("image/avif-sequence"); + } else if (s.slice(16,28).includes(Buffer.from("61766966", "hex"))) { + return("image/avif"); + } + } else if (s.slice(0,4).includes(Buffer.from("52494646", "hex")) && s.slice(8,12).includes(Buffer.from("57454250", "hex"))) { + return("image/webp"); + } + return; + } } diff --git a/src/matrix/MatrixStickerBot.ts b/src/matrix/MatrixStickerBot.ts index 74cca66..0d06183 100644 --- a/src/matrix/MatrixStickerBot.ts +++ b/src/matrix/MatrixStickerBot.ts @@ -12,6 +12,7 @@ import { MatrixLiteClient } from "./MatrixLiteClient"; import { Cache, CACHE_STICKERS } from "../MemoryCache"; import { LicenseMap } from "../utils/LicenseMap"; import { OpenId } from "../models/OpenId"; +import * as sharp from "sharp"; class _MatrixStickerBot { @@ -113,7 +114,119 @@ class _MatrixStickerBot { const serverName = mxc.substring("mxc://".length).split("/")[0]; const contentId = mxc.substring("mxc://".length).split("/")[1]; - stickerEvent.thumbMxc = await mx.uploadFromUrl(await mx.getThumbnailUrl(serverName, contentId, 512, 512, "scale", false), "image/png"); + + const url = await mx.getMediaUrl(serverName, contentId); + const downImage = await mx.downloadFromUrl(url); + + var mime = mx.parseFileHeaderMIME(downImage); + if (!mime) continue; + + const origImage = await sharp(downImage, {animated: config.stickers.allowAnimated}); + var resizedImage:any; + var size; + if (config.stickers.resize) { + const metadata = await origImage.metadata(); + size = metadata.height; + if (metadata.width > metadata.height) { + metadata.width; + } + if (size > 512) size = 512; + resizedImage = await origImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }); + } else { + if (config.stickers.verifyImageSize) { + const metadata = await origImage.metadata(); + if (metadata.width !== metadata.height || metadata.width !== 512) { + LogService.info("MatrixStickerBot", `Sticker ${stickerId} has an invalid size. Skipping...`); + continue; + } + } + resizedImage = origImage; + } + var imageUpload; + var thumbUpload; + size = 512; + if (mime === "image/png") { + if (config.stickers.resize) { + imageUpload = await resizedImage.png().toBuffer(); + thumbUpload = imageUpload; + } else { + thumbUpload = await resizedImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }).png().toBuffer(); + } + } + if (mime === "image/gif" || mime === "image/webp" || mime === "image/avif-sequence") { + if (config.stickers.allowAnimated) { + if (config.stickers.resize){ + imageUpload = await resizedImage.webp({quality: 60, effort: 3}).toBuffer(); + mime = "image/webp"; + thumbUpload = await sharp(downImage, {animated: false}).webp({quality: 50}).toBuffer(); + } else { + resizedImage = await sharp(downImage, {animated: false}).resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }); + if (mime === "image/gif") { + thumbUpload = await resizedImage.gif().toBuffer(); + } else if (mime === "image/avif-sequence") { + thumbUpload = await resizedImage.avif().toBuffer(); + } else { + thumbUpload = await resizedImage.webp({quality: 50}).toBuffer(); + } + } + + } else { + imageUpload = await resizedImage.clone().webp({quality: 60, effort: 3}).toBuffer(); + thumbUpload = await resizedImage.webp({quality: 50}).toBuffer(); + mime = "image/webp"; + } + + } + if (mime === "image/avif") { + if (config.stickers.resize) { + imageUpload = await resizedImage.clone().avif({quality: 70}).toBuffer(); + thumbUpload = await resizedImage.avif({quality: 50, chromaSubsampling: '4:2:0'}).toBuffer(); + } else { + thumbUpload = await resizedImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }).avif({quality: 50, chromaSubsampling: '4:2:0'}).toBuffer(); + } + } + if (mime === "image/jpeg") { + if (config.stickers.resize) { + imageUpload = await resizedImage.clone().jpeg({quality: 80, chromaSubsampling: '4:4:4'}).toBuffer(); + thumbUpload = await resizedImage.jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer(); + } else { + thumbUpload = await resizedImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }).jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer(); + } + } + if (imageUpload) { + stickerEvent.contentUri = await mx.upload(imageUpload, mime); + } + stickerEvent.mimetype = mime; + if (thumbUpload) { + stickerEvent.thumbMxc = await mx.upload(thumbUpload, mime); + } else { + continue; + } stickerEvents.push(stickerEvent); } @@ -142,8 +255,13 @@ class _MatrixStickerBot { pack.description = "Matrix sticker pack created by " + authorDisplayName; pack.license = license.name; pack.licensePath = license.url; - if (stickerEvents.length > 0) pack.avatarUrl = stickerEvents[0].contentUri; - await pack.save(); + if (stickerEvents.length > 0) { + pack.avatarUrl = stickerEvents[0].thumbMxc; + await pack.save(); + } else { + LogService.error("MatrixStickerBot", `No stickers in pack ${pack.name}. Removing...`); + pack.destroy(); + } const existingStickers = await Sticker.findAll({where: {packId: pack.id}}); for (const sticker of existingStickers) await sticker.destroy(); @@ -157,7 +275,7 @@ class _MatrixStickerBot { thumbnailMxc: stickerEvent.thumbMxc, thumbnailWidth: 512, thumbnailHeight: 512, - mimetype: "image/png", + mimetype: stickerEvent.mimetype, }); } } diff --git a/web/app/shared/services/scalar/scalar-widget.api.ts b/web/app/shared/services/scalar/scalar-widget.api.ts index e3c0e51..9e50a6b 100644 --- a/web/app/shared/services/scalar/scalar-widget.api.ts +++ b/web/app/shared/services/scalar/scalar-widget.api.ts @@ -45,7 +45,7 @@ export class ScalarWidgetApi { // Element Android requires content.body to contain the sticker description, otherwise // you will not be able to send any stickers body: sticker.description, - url: sticker.thumbnail.mxc, + url: sticker.image.mxc, info: { mimetype: sticker.image.mimetype, w: Math.round(sticker.thumbnail.width / 2),