From df1df4d110786a4fb027fa9ad22d62fb3cbd2fbb Mon Sep 17 00:00:00 2001 From: SG-O Date: Fri, 8 Jul 2022 04:08:26 +0200 Subject: [PATCH 1/8] Updated sharp to improve webp support --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c68f36c..e4911b5 100644 --- a/package.json +++ b/package.json @@ -71,7 +71,7 @@ "semver": "^7.3.5", "sequelize": "^6.12.0-alpha.1", "sequelize-typescript": "^2.1.1", - "sharp": "^0.29.0", + "sharp": "^0.30.7", "split-host": "^0.1.1", "spotify-uri": "^2.2.0", "sqlite3": "^5.0.2", From 49da8a60f7612dc8e31e09f489df60f3f4181e2f Mon Sep 17 00:00:00 2001 From: SG-O Date: Fri, 8 Jul 2022 04:13:17 +0200 Subject: [PATCH 2/8] Added support for several image formats This includes: - Gif - jpeg - avif - webp --- src/matrix/MatrixLiteClient.ts | 55 ++++++++++++++++++++++++++++++++++ src/matrix/MatrixStickerBot.ts | 53 ++++++++++++++++++++++++++++++-- 2 files changed, 105 insertions(+), 3 deletions(-) diff --git a/src/matrix/MatrixLiteClient.ts b/src/matrix/MatrixLiteClient.ts index 939dd95..a3365cb 100644 --- a/src/matrix/MatrixLiteClient.ts +++ b/src/matrix/MatrixLiteClient.ts @@ -36,6 +36,15 @@ export class MatrixLiteClient { return baseUrl + `/_matrix/media/r0/thumbnail/${serverName}/${contentId}?width=${width}&height=${height}&method=${method}&animated=${isAnimated}`; } + public async getMediaUrl(serverName: string, contentId: string): Promise { + let baseUrl = config.homeserver.mediaUrl; + if (!baseUrl) baseUrl = config.homeserver.clientServerUrl; + if (baseUrl.endsWith("/")) baseUrl = baseUrl.substring(0, baseUrl.length - 1); + + // DO NOT RETURN THE ACCESS TOKEN. + return baseUrl + `/_matrix/media/r0/download/${serverName}/${contentId}`; + } + public async whoAmI(): Promise { const response = await doClientApiCall( "GET", @@ -106,6 +115,7 @@ export class MatrixLiteClient { } public async upload(content: Buffer, contentType: string): Promise { + LogService.info("MatrixLiteClient", "Uploading file (type:" + contentType + ")"); return doClientApiCall( "POST", "/_matrix/media/r0/upload", @@ -126,6 +136,7 @@ export class MatrixLiteClient { method: "GET", url: url, encoding: null, + headers: {}, }, (err, res, _body) => { if (err) { LogService.error("MatrixLiteClient", "Error downloading file from " + url); @@ -140,4 +151,48 @@ export class MatrixLiteClient { }); }); } + + public async parseMediaMIME(url: string): Promise { + return new Promise((resolve, reject) => { + request({ + method: "GET", + url: url, + encoding: null, + headers: { + 'Range': 'bytes=0-16' + }, + }, (err, res, _body) => { + if (err) { + LogService.error("MatrixLiteClient", "Error downloading file from " + url); + LogService.error("MatrixLiteClient", err); + reject(err); + } else if (res.statusCode !== 200) { + if (res.statusCode !== 206) { + LogService.error("MatrixLiteClient", "Got status code " + res.statusCode + " while calling url " + url); + reject(new Error("Error in request: invalid status code")); + } + } else { + return this.parseFileHeaderMIME(res.body); + } + }); + }); + } + + public parseFileHeaderMIME(data: Buffer): string { + const s = data.slice(0,12); + if (s.slice(0,8).includes(Buffer.from("89504E470D0A1A0A", "hex"))) { + return("image/png"); + } else if (s.slice(0,3).includes(Buffer.from("474946", "hex"))) { + return("image/gif"); + } else if (s.slice(0,3).includes(Buffer.from("FFD8FF", "hex"))) { + return("image/jpeg"); + } else if (s.slice(0,12).includes(Buffer.from("000000206674797061766966", "hex"))) { + return("image/avif"); + } else if (s.slice(0,12).includes(Buffer.from("000000206674797061766973", "hex"))) { + return("image/avif-sequence"); + } else if (s.slice(0,4).includes(Buffer.from("52494646", "hex")) && s.slice(8,12).includes(Buffer.from("57454250", "hex"))) { + return("image/webp"); + } + return; + } } diff --git a/src/matrix/MatrixStickerBot.ts b/src/matrix/MatrixStickerBot.ts index 74cca66..385a6c2 100644 --- a/src/matrix/MatrixStickerBot.ts +++ b/src/matrix/MatrixStickerBot.ts @@ -12,6 +12,7 @@ import { MatrixLiteClient } from "./MatrixLiteClient"; import { Cache, CACHE_STICKERS } from "../MemoryCache"; import { LicenseMap } from "../utils/LicenseMap"; import { OpenId } from "../models/OpenId"; +import * as sharp from "sharp"; class _MatrixStickerBot { @@ -113,7 +114,53 @@ class _MatrixStickerBot { const serverName = mxc.substring("mxc://".length).split("/")[0]; const contentId = mxc.substring("mxc://".length).split("/")[1]; - stickerEvent.thumbMxc = await mx.uploadFromUrl(await mx.getThumbnailUrl(serverName, contentId, 512, 512, "scale", false), "image/png"); + + const url = await mx.getMediaUrl(serverName, contentId); + const downImage = await mx.downloadFromUrl(url); + + var mime = mx.parseFileHeaderMIME(downImage); + if (!mime) continue; + + const origImage = await sharp(downImage, {animated: true}); + const metadata = await origImage.metadata(); + var size = metadata.height; + if (metadata.width > metadata.height) { + metadata.width; + } + if (size > 512) size = 512; + const resizedImage = await origImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }); + var imageUpload; + var thumbUpload; + if (mime === "image/png") { + imageUpload = await resizedImage.png().toBuffer(); + thumbUpload = imageUpload; + } + if (mime === "image/gif" || mime === "image/webp" || mime === "image/avif-sequence") { + imageUpload = await resizedImage.webp({nearLossless: true, quality: 60}).toBuffer(); + thumbUpload = await sharp(downImage, {animated: false}).resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }).webp({quality: 60}).toBuffer(); + mime = "image/webp"; + } + if (mime === "image/avif") { + imageUpload = await resizedImage.clone().avif({quality: 70}).toBuffer(); + thumbUpload = await resizedImage.avif({quality: 50, chromaSubsampling: '4:2:0'}).toBuffer();; + } + if (mime === "image/jpeg") { + imageUpload = await resizedImage.clone().jpeg({quality: 80, chromaSubsampling: '4:4:4'}).toBuffer(); + thumbUpload = await resizedImage.avif({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer();; + } + stickerEvent.contentUri = await mx.upload(imageUpload, mime); + stickerEvent.mimetype = mime; + stickerEvent.thumbMxc = await mx.upload(thumbUpload, mime); stickerEvents.push(stickerEvent); } @@ -142,7 +189,7 @@ class _MatrixStickerBot { pack.description = "Matrix sticker pack created by " + authorDisplayName; pack.license = license.name; pack.licensePath = license.url; - if (stickerEvents.length > 0) pack.avatarUrl = stickerEvents[0].contentUri; + if (stickerEvents.length > 0) pack.avatarUrl = stickerEvents[0].thumbMxc; await pack.save(); const existingStickers = await Sticker.findAll({where: {packId: pack.id}}); @@ -157,7 +204,7 @@ class _MatrixStickerBot { thumbnailMxc: stickerEvent.thumbMxc, thumbnailWidth: 512, thumbnailHeight: 512, - mimetype: "image/png", + mimetype: stickerEvent.mimetype, }); } } From d89bfd2f224c737dcbea7acf7ff465ae48e67964 Mon Sep 17 00:00:00 2001 From: SG-O Date: Fri, 8 Jul 2022 05:07:04 +0200 Subject: [PATCH 3/8] Bugfix Animated stickers were not animating --- web/app/shared/services/scalar/scalar-widget.api.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web/app/shared/services/scalar/scalar-widget.api.ts b/web/app/shared/services/scalar/scalar-widget.api.ts index e3c0e51..9e50a6b 100644 --- a/web/app/shared/services/scalar/scalar-widget.api.ts +++ b/web/app/shared/services/scalar/scalar-widget.api.ts @@ -45,7 +45,7 @@ export class ScalarWidgetApi { // Element Android requires content.body to contain the sticker description, otherwise // you will not be able to send any stickers body: sticker.description, - url: sticker.thumbnail.mxc, + url: sticker.image.mxc, info: { mimetype: sticker.image.mimetype, w: Math.round(sticker.thumbnail.width / 2), From 87cecc9540bf5a185f2b8549bdc08f3c93020e6e Mon Sep 17 00:00:00 2001 From: SG-O Date: Fri, 8 Jul 2022 05:07:36 +0200 Subject: [PATCH 4/8] Improved size of animated images --- src/matrix/MatrixStickerBot.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/matrix/MatrixStickerBot.ts b/src/matrix/MatrixStickerBot.ts index 385a6c2..ec8e7ef 100644 --- a/src/matrix/MatrixStickerBot.ts +++ b/src/matrix/MatrixStickerBot.ts @@ -141,13 +141,13 @@ class _MatrixStickerBot { thumbUpload = imageUpload; } if (mime === "image/gif" || mime === "image/webp" || mime === "image/avif-sequence") { - imageUpload = await resizedImage.webp({nearLossless: true, quality: 60}).toBuffer(); + imageUpload = await resizedImage.webp({quality: 60, effort: 6}).toBuffer(); thumbUpload = await sharp(downImage, {animated: false}).resize({ width: size, height: size, fit: 'contain', background: 'rgba(0,0,0,0)', - }).webp({quality: 60}).toBuffer(); + }).webp({quality: 50}).toBuffer(); mime = "image/webp"; } if (mime === "image/avif") { From 0a9d11d8d37db4077a1b492dbed497ea672f96be Mon Sep 17 00:00:00 2001 From: SG-O Date: Fri, 8 Jul 2022 20:33:08 +0200 Subject: [PATCH 5/8] Improved detection of avif images --- src/matrix/MatrixLiteClient.ts | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/matrix/MatrixLiteClient.ts b/src/matrix/MatrixLiteClient.ts index a3365cb..a85597d 100644 --- a/src/matrix/MatrixLiteClient.ts +++ b/src/matrix/MatrixLiteClient.ts @@ -159,7 +159,7 @@ export class MatrixLiteClient { url: url, encoding: null, headers: { - 'Range': 'bytes=0-16' + 'Range': 'bytes=0-32' }, }, (err, res, _body) => { if (err) { @@ -179,17 +179,19 @@ export class MatrixLiteClient { } public parseFileHeaderMIME(data: Buffer): string { - const s = data.slice(0,12); + const s = data.slice(0,32); if (s.slice(0,8).includes(Buffer.from("89504E470D0A1A0A", "hex"))) { return("image/png"); } else if (s.slice(0,3).includes(Buffer.from("474946", "hex"))) { return("image/gif"); } else if (s.slice(0,3).includes(Buffer.from("FFD8FF", "hex"))) { return("image/jpeg"); - } else if (s.slice(0,12).includes(Buffer.from("000000206674797061766966", "hex"))) { - return("image/avif"); - } else if (s.slice(0,12).includes(Buffer.from("000000206674797061766973", "hex"))) { - return("image/avif-sequence"); + } else if (s.slice(0,3).includes(Buffer.from("000000", "hex")) && s.slice(4,8).includes(Buffer.from("66747970", "hex"))) { + if (s.slice(16,28).includes(Buffer.from("61766973", "hex"))) { + return("image/avif-sequence"); + } else if (s.slice(16,28).includes(Buffer.from("61766966", "hex"))) { + return("image/avif"); + } } else if (s.slice(0,4).includes(Buffer.from("52494646", "hex")) && s.slice(8,12).includes(Buffer.from("57454250", "hex"))) { return("image/webp"); } From 3fdd87a7271e8e6d0fbbe2fd64c4c09557f75d62 Mon Sep 17 00:00:00 2001 From: SG-O Date: Fri, 8 Jul 2022 20:45:09 +0200 Subject: [PATCH 6/8] Fixed bug in the jpeg thumbnail creation --- src/matrix/MatrixStickerBot.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/matrix/MatrixStickerBot.ts b/src/matrix/MatrixStickerBot.ts index ec8e7ef..ca38fec 100644 --- a/src/matrix/MatrixStickerBot.ts +++ b/src/matrix/MatrixStickerBot.ts @@ -156,7 +156,7 @@ class _MatrixStickerBot { } if (mime === "image/jpeg") { imageUpload = await resizedImage.clone().jpeg({quality: 80, chromaSubsampling: '4:4:4'}).toBuffer(); - thumbUpload = await resizedImage.avif({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer();; + thumbUpload = await resizedImage.jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer();; } stickerEvent.contentUri = await mx.upload(imageUpload, mime); stickerEvent.mimetype = mime; From d35e389882e94124a668f7825fb9e27b4cf04d48 Mon Sep 17 00:00:00 2001 From: SG-O Date: Sun, 10 Jul 2022 01:31:37 +0200 Subject: [PATCH 7/8] Added several config options This includes the ability to enable or disable the resizing of oversized images. The ability to allow the upload of animated stickers. The possibillity to disallow the upload of stickers that are not 512x512 when resizing is enabled --- config/default.yaml | 9 ++++ src/config.ts | 3 ++ src/matrix/MatrixStickerBot.ts | 85 ++++++++++++++++++++++++---------- 3 files changed, 73 insertions(+), 24 deletions(-) diff --git a/config/default.yaml b/config/default.yaml index 7a4287a..bd55489 100644 --- a/config/default.yaml +++ b/config/default.yaml @@ -79,6 +79,15 @@ stickers: # Whether or not to allow people to add custom sticker packs enabled: true + # Whether to resize images that are not 512x512 during import. This might slow down the import of animated Stickers. + resize: true + + # Whether to allow animated stickers. If set to false animations will be stripped from images. + allowAnimated: true + + # Whether to skip over images that are not 512x512. Ignored when resize is true. + verifyImageSize: true + # The sticker manager bot to promote stickerBot: "@stickers:t2bot.io" diff --git a/src/config.ts b/src/config.ts index fad170c..39bc6d9 100644 --- a/src/config.ts +++ b/src/config.ts @@ -36,6 +36,9 @@ export interface DimensionConfig { }; stickers: { enabled: boolean; + resize: boolean; + allowAnimated: boolean; + verifyImageSize: boolean; stickerBot: string; managerUrl: string; }; diff --git a/src/matrix/MatrixStickerBot.ts b/src/matrix/MatrixStickerBot.ts index ca38fec..0b679a5 100644 --- a/src/matrix/MatrixStickerBot.ts +++ b/src/matrix/MatrixStickerBot.ts @@ -121,19 +121,31 @@ class _MatrixStickerBot { var mime = mx.parseFileHeaderMIME(downImage); if (!mime) continue; - const origImage = await sharp(downImage, {animated: true}); - const metadata = await origImage.metadata(); - var size = metadata.height; - if (metadata.width > metadata.height) { - metadata.width; + const origImage = await sharp(downImage, {animated: config.stickers.allowAnimated}); + var resizedImage:any; + if (config.stickers.resize) { + const metadata = await origImage.metadata(); + var size = metadata.height; + if (metadata.width > metadata.height) { + metadata.width; + } + if (size > 512) size = 512; + resizedImage = await origImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }); + } else { + if (config.stickers.verifyImageSize) { + const metadata = await origImage.metadata(); + if (metadata.width !== metadata.height || metadata.width !== 512) { + LogService.info("MatrixStickerBot", `Sticker ${stickerId} has an invalid size. Skipping...`); + continue; + } + } + resizedImage = origImage; } - if (size > 512) size = 512; - const resizedImage = await origImage.resize({ - width: size, - height: size, - fit: 'contain', - background: 'rgba(0,0,0,0)', - }); var imageUpload; var thumbUpload; if (mime === "image/png") { @@ -141,14 +153,28 @@ class _MatrixStickerBot { thumbUpload = imageUpload; } if (mime === "image/gif" || mime === "image/webp" || mime === "image/avif-sequence") { - imageUpload = await resizedImage.webp({quality: 60, effort: 6}).toBuffer(); - thumbUpload = await sharp(downImage, {animated: false}).resize({ - width: size, - height: size, - fit: 'contain', - background: 'rgba(0,0,0,0)', - }).webp({quality: 50}).toBuffer(); - mime = "image/webp"; + if (config.stickers.allowAnimated) { + if (config.stickers.resize){ + imageUpload = await resizedImage.webp({quality: 60, effort: 6}).toBuffer(); + mime = "image/webp"; + thumbUpload = await sharp(downImage, {animated: false}).webp({quality: 50}).toBuffer(); + } else { + imageUpload = null; + if (mime === "image/gif") { + thumbUpload = await sharp(downImage, {animated: false}).toBuffer(); + } else if (mime === "image/avif-sequence") { + thumbUpload = await sharp(downImage, {animated: false}).toBuffer(); + } else { + thumbUpload = await sharp(downImage, {animated: false}).webp({quality: 50}).toBuffer(); + } + } + + } else { + imageUpload = await resizedImage.clone().webp({quality: 60, effort: 6}).toBuffer(); + thumbUpload = await resizedImage.webp({quality: 50}).toBuffer(); + mime = "image/webp"; + } + } if (mime === "image/avif") { imageUpload = await resizedImage.clone().avif({quality: 70}).toBuffer(); @@ -158,9 +184,15 @@ class _MatrixStickerBot { imageUpload = await resizedImage.clone().jpeg({quality: 80, chromaSubsampling: '4:4:4'}).toBuffer(); thumbUpload = await resizedImage.jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer();; } - stickerEvent.contentUri = await mx.upload(imageUpload, mime); + if (imageUpload) { + stickerEvent.contentUri = await mx.upload(imageUpload, mime); + } stickerEvent.mimetype = mime; - stickerEvent.thumbMxc = await mx.upload(thumbUpload, mime); + if (thumbUpload) { + stickerEvent.thumbMxc = await mx.upload(thumbUpload, mime); + } else { + continue; + } stickerEvents.push(stickerEvent); } @@ -189,8 +221,13 @@ class _MatrixStickerBot { pack.description = "Matrix sticker pack created by " + authorDisplayName; pack.license = license.name; pack.licensePath = license.url; - if (stickerEvents.length > 0) pack.avatarUrl = stickerEvents[0].thumbMxc; - await pack.save(); + if (stickerEvents.length > 0) { + pack.avatarUrl = stickerEvents[0].thumbMxc; + await pack.save(); + } else { + LogService.error("MatrixStickerBot", `No stickers in pack ${pack.name}. Removing...`); + pack.destroy(); + } const existingStickers = await Sticker.findAll({where: {packId: pack.id}}); for (const sticker of existingStickers) await sticker.destroy(); From cc0079ea5603fb867887d21d77bf8545fea94f52 Mon Sep 17 00:00:00 2001 From: SG-O Date: Sun, 10 Jul 2022 02:53:21 +0200 Subject: [PATCH 8/8] Improved speed and a small bugfix where thumbnails were not resized correctly. --- src/matrix/MatrixStickerBot.ts | 60 ++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/src/matrix/MatrixStickerBot.ts b/src/matrix/MatrixStickerBot.ts index 0b679a5..0d06183 100644 --- a/src/matrix/MatrixStickerBot.ts +++ b/src/matrix/MatrixStickerBot.ts @@ -123,9 +123,10 @@ class _MatrixStickerBot { const origImage = await sharp(downImage, {animated: config.stickers.allowAnimated}); var resizedImage:any; + var size; if (config.stickers.resize) { const metadata = await origImage.metadata(); - var size = metadata.height; + size = metadata.height; if (metadata.width > metadata.height) { metadata.width; } @@ -148,41 +149,74 @@ class _MatrixStickerBot { } var imageUpload; var thumbUpload; + size = 512; if (mime === "image/png") { - imageUpload = await resizedImage.png().toBuffer(); - thumbUpload = imageUpload; + if (config.stickers.resize) { + imageUpload = await resizedImage.png().toBuffer(); + thumbUpload = imageUpload; + } else { + thumbUpload = await resizedImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }).png().toBuffer(); + } } if (mime === "image/gif" || mime === "image/webp" || mime === "image/avif-sequence") { if (config.stickers.allowAnimated) { if (config.stickers.resize){ - imageUpload = await resizedImage.webp({quality: 60, effort: 6}).toBuffer(); + imageUpload = await resizedImage.webp({quality: 60, effort: 3}).toBuffer(); mime = "image/webp"; thumbUpload = await sharp(downImage, {animated: false}).webp({quality: 50}).toBuffer(); } else { - imageUpload = null; + resizedImage = await sharp(downImage, {animated: false}).resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }); if (mime === "image/gif") { - thumbUpload = await sharp(downImage, {animated: false}).toBuffer(); + thumbUpload = await resizedImage.gif().toBuffer(); } else if (mime === "image/avif-sequence") { - thumbUpload = await sharp(downImage, {animated: false}).toBuffer(); + thumbUpload = await resizedImage.avif().toBuffer(); } else { - thumbUpload = await sharp(downImage, {animated: false}).webp({quality: 50}).toBuffer(); + thumbUpload = await resizedImage.webp({quality: 50}).toBuffer(); } } } else { - imageUpload = await resizedImage.clone().webp({quality: 60, effort: 6}).toBuffer(); + imageUpload = await resizedImage.clone().webp({quality: 60, effort: 3}).toBuffer(); thumbUpload = await resizedImage.webp({quality: 50}).toBuffer(); mime = "image/webp"; } } if (mime === "image/avif") { - imageUpload = await resizedImage.clone().avif({quality: 70}).toBuffer(); - thumbUpload = await resizedImage.avif({quality: 50, chromaSubsampling: '4:2:0'}).toBuffer();; + if (config.stickers.resize) { + imageUpload = await resizedImage.clone().avif({quality: 70}).toBuffer(); + thumbUpload = await resizedImage.avif({quality: 50, chromaSubsampling: '4:2:0'}).toBuffer(); + } else { + thumbUpload = await resizedImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }).avif({quality: 50, chromaSubsampling: '4:2:0'}).toBuffer(); + } } if (mime === "image/jpeg") { - imageUpload = await resizedImage.clone().jpeg({quality: 80, chromaSubsampling: '4:4:4'}).toBuffer(); - thumbUpload = await resizedImage.jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer();; + if (config.stickers.resize) { + imageUpload = await resizedImage.clone().jpeg({quality: 80, chromaSubsampling: '4:4:4'}).toBuffer(); + thumbUpload = await resizedImage.jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer(); + } else { + thumbUpload = await resizedImage.resize({ + width: size, + height: size, + fit: 'contain', + background: 'rgba(0,0,0,0)', + }).jpeg({quality: 60, chromaSubsampling: '4:2:0'}).toBuffer(); + } } if (imageUpload) { stickerEvent.contentUri = await mx.upload(imageUpload, mime);