From 8a6fadf72974c80804ddc361f68f7c5f0e6c6908 Mon Sep 17 00:00:00 2001 From: bertybuttface <110790513+bertybuttface@users.noreply.github.com> Date: Tue, 31 Oct 2023 21:11:24 +0000 Subject: [PATCH] Begin dropping keyv --- src/env.ts | 81 ++++++++++++++++++------------ src/index.ts | 19 ++------ src/storage.ts | 130 ------------------------------------------------- 3 files changed, 54 insertions(+), 176 deletions(-) delete mode 100644 src/storage.ts diff --git a/src/env.ts b/src/env.ts index 503d4b5..122cf40 100644 --- a/src/env.ts +++ b/src/env.ts @@ -5,9 +5,6 @@ dotenv.config(); export const { DATA_PATH, - KEYV_BACKEND, - KEYV_URL, - KEYV_BOT_STORAGE, /** Matrix Bot Settings */ MATRIX_HOMESERVER_URL, MATRIX_ACCESS_TOKEN, @@ -39,38 +36,60 @@ export const { CHATGPT_REVERSE_PROXY, CHATGPT_TEMPERATURE, } = parseEnv(process.env, { - DATA_PATH: { schema: z.string().default("./storage"), description: "Set to /storage/ if using docker, ./storage if running without" }, - KEYV_BACKEND: { schema: z.enum(["file", "other"]).default("file"),description: "Set the Keyv backend to 'file' or 'other' if other set KEYV_URL" }, - KEYV_URL: { schema: z.string().default(""), description: "Set Keyv backend for storage, in-memory if blank, ignored if KEYV_BACKEND set to `file`"}, - KEYV_BOT_STORAGE: { schema: z.boolean().default(false), description: "Set to true to use a Keyv backend to store bot data. Uses a file if false."}, + DATA_PATH: { schema: z.string().default("./storage"), + description: "Set to /storage/ if using docker, ./storage if running without" }, /** Matrix Bot Settings */ - MATRIX_HOMESERVER_URL: { schema: z.string().default("https://matrix.org"), description: "Set matrix homeserver with 'https://' prefix" }, - MATRIX_ACCESS_TOKEN: { schema: z.string().optional(), description: "Set MATRIX_BOT_USERNAME & MATRIX_BOT_PASSWORD to print MATRIX_ACCESS_TOKEN or follow https://webapps.stackexchange.com/questions/131056/how-to-get-an-access-token-for-element-riot-matrix" }, - MATRIX_BOT_USERNAME: { schema: z.string().optional(), description: "Set full username: eg @bot:server.com (superseded by MATRIX_ACCESS_TOKEN if set)" }, - MATRIX_BOT_PASSWORD: { schema: z.string().optional(), description: "Set password (superseded by MATRIX_ACCESS_TOKEN if set)" }, + MATRIX_HOMESERVER_URL: { schema: z.string().default("https://matrix.org"), + description: "Set matrix homeserver with 'https://' prefix" }, + MATRIX_ACCESS_TOKEN: { schema: z.string().optional(), + description: "Set MATRIX_BOT_USERNAME & MATRIX_BOT_PASSWORD to print MATRIX_ACCESS_TOKEN or follow https://webapps.stackexchange.com/questions/131056/how-to-get-an-access-token-for-element-riot-matrix" }, + MATRIX_BOT_USERNAME: { schema: z.string().optional(), + description: "Set full username: eg @bot:server.com (superseded by MATRIX_ACCESS_TOKEN if set)" }, + MATRIX_BOT_PASSWORD: { schema: z.string().optional(), + description: "Set password (superseded by MATRIX_ACCESS_TOKEN if set)" }, /** Matrix Bot Features */ - MATRIX_AUTOJOIN: { schema: z.boolean().default(true), description: "Set to true if you want the bot to autojoin when invited" }, - MATRIX_ENCRYPTION: { schema: z.boolean().default(true), description: "Set to true if you want the bot to support encrypted channels" }, - MATRIX_THREADS: { schema: z.boolean().default(true), description: "Set to true if you want the bot to answer always in a new thread/conversation" }, - MATRIX_PREFIX_DM: { schema: z.boolean().default(false), description: "Set to false if you want the bot to answer to all messages in a one-to-one room" }, - MATRIX_RICH_TEXT: { schema: z.boolean().default(true), description: "Set to true if you want the bot to answer with enriched text" }, - MATRIX_WELCOME: { schema: z.boolean().default(true), description: "Set to true if you want the bot to post a message when it joins a new chat." }, + MATRIX_AUTOJOIN: { schema: z.boolean().default(true), + description: "Set to true if you want the bot to autojoin when invited" }, + MATRIX_ENCRYPTION: { schema: z.boolean().default(true), + description: "Set to true if you want the bot to support encrypted channels" }, + MATRIX_THREADS: { schema: z.boolean().default(true), + description: "Set to true if you want the bot to answer always in a new thread/conversation" }, + MATRIX_PREFIX_DM: { schema: z.boolean().default(false), + description: "Set to false if you want the bot to answer to all messages in a one-to-one room" }, + MATRIX_RICH_TEXT: { schema: z.boolean().default(true), + description: "Set to true if you want the bot to answer with enriched text" }, + MATRIX_WELCOME: { schema: z.boolean().default(true), + description: "Set to true if you want the bot to post a message when it joins a new chat." }, /** Matrix Access Control */ - MATRIX_BLACKLIST: { schema: z.string().optional(), description: "Set to a spaces separated string of 'user:homeserver' or a wildcard like ':anotherhomeserver.example' to blacklist users or domains" }, - MATRIX_WHITELIST: { schema: z.string().optional(), description: "Set to a spaces separated string of 'user:homeserver' or a wildcard like ':anotherhomeserver.example' to whitelist users or domains" }, - MATRIX_ROOM_BLACKLIST: { schema: z.string().optional(), description: "Set to a spaces separated string of 'user:homeserver' or a wildcard like ':anotherhomeserver.example' to blacklist rooms or domains" }, - MATRIX_ROOM_WHITELIST: { schema: z.string().optional(), description: "Set to a spaces separated string of 'user:homeserver' or a wildcard like ':anotherhomeserver.example' to whitelist rooms or domains" }, + MATRIX_BLACKLIST: { schema: z.string().optional(), + description: "Set to a spaces separated string of 'user:homeserver' or a wildcard like ':anotherhomeserver.example' to blacklist users or domains" }, + MATRIX_WHITELIST: { schema: z.string().optional(), + description: "Set to a spaces separated string of 'user:homeserver' or a wildcard like ':anotherhomeserver.example' to whitelist users or domains" }, + MATRIX_ROOM_BLACKLIST: { schema: z.string().optional(), + description: "Set to a spaces separated string of 'user:homeserver' or a wildcard like ':anotherhomeserver.example' to blacklist rooms or domains" }, + MATRIX_ROOM_WHITELIST: { schema: z.string().optional(), + description: "Set to a spaces separated string of 'user:homeserver' or a wildcard like ':anotherhomeserver.example' to whitelist rooms or domains" }, /** Matrix Bot Runtime Config */ - MATRIX_DEFAULT_PREFIX: { schema: z.string().default(""), description: "Set to a string if you want the bot to respond only when messages start with this prefix. Trailing space matters. Empty for no prefix." }, - MATRIX_DEFAULT_PREFIX_REPLY: { schema: z.boolean().default(false), description: "Set to false if you want the bot to answer to all messages in a thread/conversation" }, + MATRIX_DEFAULT_PREFIX: { schema: z.string().default(""), + description: "Set to a string if you want the bot to respond only when messages start with this prefix. Trailing space matters. Empty for no prefix." }, + MATRIX_DEFAULT_PREFIX_REPLY: { schema: z.boolean().default(false), + description: "Set to false if you want the bot to answer to all messages in a thread/conversation" }, /** ChatGPT Settings */ - OPENAI_AZURE: { schema: z.boolean().default(false), description: "Wether or not to use Azure OPENAI"}, - OPENAI_API_KEY: { schema: z.string().default(""), description: "Set to the API key from https://platform.openai.com/account/api-keys"}, - CHATGPT_TIMEOUT: { schema: z.number().default(2 * 60 * 1000), description: "Set number of milliseconds to wait for ChatGPT responses" }, - CHATGPT_CONTEXT: { schema: z.enum(["thread", "room", "both"]).default("thread"), description: "Set the ChatGPT conversation context to 'thread', 'room' or 'both'" }, - CHATGPT_API_MODEL: { schema: z.string().default(""), description: "The model for the ChatGPT-API to use. Keep in mind that these models will charge your OpenAI account depending on their pricing." }, + OPENAI_AZURE: { schema: z.boolean().default(false), + description: "Wether or not to use Azure OPENAI"}, + OPENAI_API_KEY: { schema: z.string().default(""), + description: "Set to the API key from https://platform.openai.com/account/api-keys"}, + CHATGPT_TIMEOUT: { schema: z.number().default(2 * 60 * 1000), + description: "Set number of milliseconds to wait for ChatGPT responses" }, + CHATGPT_CONTEXT: { schema: z.enum(["thread", "room", "both"]).default("thread"), + description: "Set the ChatGPT conversation context to 'thread', 'room' or 'both'" }, + CHATGPT_API_MODEL: { schema: z.string().default(""), + description: "The model for the ChatGPT-API to use. Keep in mind that these models will charge your OpenAI account depending on their pricing." }, CHATGPT_PROMPT_PREFIX: { schema: z.string().default('Instructions:\nYou are ChatGPT, a large language model trained by OpenAI.'), description: "Instructions to feed to ChatGPT on startup"}, - CHATGPT_IGNORE_MEDIA: { schema: z.boolean().default(false), description: "Wether or not the bot should react to non-text messages"}, - CHATGPT_REVERSE_PROXY: { schema: z.string().default(""), description: "Change the api url to use another (OpenAI-compatible) API endpoint" }, - CHATGPT_TEMPERATURE: { schema: z.number().default(0.8), description: "Set the temperature for the model" } + CHATGPT_IGNORE_MEDIA: { schema: z.boolean().default(false), + description: "Wether or not the bot should react to non-text messages"}, + CHATGPT_REVERSE_PROXY: { schema: z.string().default(""), + description: "Change the api url to use another (OpenAI-compatible) API endpoint" }, + CHATGPT_TEMPERATURE: { schema: z.number().default(0.8), + description: "Set the temperature for the model" } }); diff --git a/src/index.ts b/src/index.ts index 3a618a6..01f2476 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,5 +1,4 @@ import ChatGPTClient from '@waylaidwanderer/chatgpt-api'; -import Keyv from 'keyv' import { KeyvFile } from 'keyv-file'; import { MatrixAuth, MatrixClient, AutojoinRoomsMixin, LogService, LogLevel, RichConsoleLogger, @@ -8,13 +7,10 @@ import { import * as path from "path"; import { - DATA_PATH, KEYV_URL, OPENAI_AZURE, OPENAI_API_KEY, MATRIX_HOMESERVER_URL, MATRIX_ACCESS_TOKEN, MATRIX_AUTOJOIN, - MATRIX_BOT_PASSWORD, MATRIX_BOT_USERNAME, MATRIX_ENCRYPTION, MATRIX_THREADS, CHATGPT_CONTEXT, - CHATGPT_API_MODEL, KEYV_BOT_STORAGE, KEYV_BACKEND, CHATGPT_PROMPT_PREFIX, MATRIX_WELCOME, - CHATGPT_REVERSE_PROXY, CHATGPT_TEMPERATURE + DATA_PATH, MATRIX_HOMESERVER_URL, MATRIX_ACCESS_TOKEN, MATRIX_AUTOJOIN, MATRIX_WELCOME, MATRIX_BOT_PASSWORD, MATRIX_BOT_USERNAME, MATRIX_ENCRYPTION, MATRIX_THREADS, + OPENAI_AZURE, OPENAI_API_KEY, CHATGPT_CONTEXT, CHATGPT_API_MODEL, CHATGPT_PROMPT_PREFIX, CHATGPT_REVERSE_PROXY, CHATGPT_TEMPERATURE } from './env.js' import CommandHandler from "./handlers.js" -import { KeyvStorageProvider } from './storage.js' import { parseMatrixUsernamePretty, wrapPrompt } from './utils.js'; LogService.setLogger(new RichConsoleLogger()); @@ -22,22 +18,15 @@ LogService.setLogger(new RichConsoleLogger()); LogService.setLevel(LogLevel.INFO); // LogService.muteModule("Metrics"); LogService.trace = LogService.debug; -if (KEYV_URL && KEYV_BACKEND === 'file') LogService.warn('config', 'KEYV_URL is ignored when KEYV_BACKEND is set to `file`') let storage: IStorageProvider -if (KEYV_BOT_STORAGE) { - storage = new KeyvStorageProvider('chatgpt-bot-storage'); -} else { - storage = new SimpleFsStorageProvider(path.join(DATA_PATH, "bot.json")); // /storage/bot.json -} +storage = new SimpleFsStorageProvider(path.join(DATA_PATH, "bot.json")); // /storage/bot.json let cryptoStore: ICryptoStorageProvider; if (MATRIX_ENCRYPTION) cryptoStore = new RustSdkCryptoStorageProvider(path.join(DATA_PATH, "encrypted")); // /storage/encrypted let cacheOptions // Options for the Keyv cache, see https://www.npmjs.com/package/keyv -if (KEYV_BACKEND === 'file'){ - cacheOptions = { store: new KeyvFile({ filename: path.join(DATA_PATH, `chatgpt-bot-api.json`) }) }; -} else { cacheOptions = { uri: KEYV_URL } } +cacheOptions = { store: new KeyvFile({ filename: path.join(DATA_PATH, `chatgpt-bot-api.json`) }) }; async function main() { if (!MATRIX_ACCESS_TOKEN){ diff --git a/src/storage.ts b/src/storage.ts deleted file mode 100644 index 7287742..0000000 --- a/src/storage.ts +++ /dev/null @@ -1,130 +0,0 @@ -import Keyv from 'keyv' -import { KeyvFile } from 'keyv-file'; -import * as sha512 from "hash.js/lib/hash/sha/512.js"; -import * as path from "path"; -import { IAppserviceStorageProvider, IFilterInfo, IStorageProvider } from "matrix-bot-sdk"; -import { DATA_PATH, KEYV_BACKEND, KEYV_URL } from './env.js'; - -/** - * A storage provider that uses the disk to store information. - * @category Storage providers - */ -export class KeyvStorageProvider implements IStorageProvider, IAppserviceStorageProvider { - - private completedTransactions = []; - private db: Keyv; - - /** - * Creates a new simple file system storage provider. - * @param {Keyv} keyvStore A Keyv instance for storing data. - */ - constructor(namespace: string, private trackTransactionsInMemory = true, private maxInMemoryTransactions = 20) { - if (KEYV_BACKEND === 'file'){ - this.db = new Keyv({store: new KeyvFile({ filename: path.join(DATA_PATH, `${namespace}.json`),})}) - } else { - this.db = new Keyv(KEYV_URL, { namespace: namespace }); - } - this.db.set('syncToken', null) - this.db.set('filter', null) - this.db.set('appserviceUsers', {}) // userIdHash => { data } - this.db.set('appserviceTransactions', {}) // txnIdHash => { data } - this.db.set('kvStore', {}) // key => value (str) - } - - setSyncToken(token: string | null): Promise | void { - this.db.set('syncToken', token); - } - - getSyncToken(): string | Promise | null{ - return this.db.get('syncToken'); - } - - setFilter(filter: IFilterInfo): void { - this.db.set('filter', filter); - } - - getFilter(): IFilterInfo | Promise { - return this.db.get('filter'); - } - - addRegisteredUser(userId: string): Promise | void { - const key = sha512().update(userId).digest('hex'); - this.db.set(`appserviceUsers.${key}.userId`, userId) - this.db.set(`appserviceUsers.${key}.registered`, true) - } - - isUserRegistered(userId: string): boolean | Promise { - const key = sha512().update(userId).digest('hex'); - return this.db.get(`appserviceUsers.${key}.registered`); - } - - isTransactionCompleted(transactionId: string): boolean | Promise { - if (this.trackTransactionsInMemory) { - return this.completedTransactions.indexOf(transactionId) !== -1; - } - - const key = sha512().update(transactionId).digest('hex'); - return this.db.get(`appserviceTransactions.${key}.completed`); - } - - setTransactionCompleted(transactionId: string): Promise | void { - if (this.trackTransactionsInMemory) { - if (this.completedTransactions.indexOf(transactionId) === -1) { - this.completedTransactions.push(transactionId); - } - if (this.completedTransactions.length > this.maxInMemoryTransactions) { - this.completedTransactions = this.completedTransactions.reverse().slice(0, this.maxInMemoryTransactions).reverse(); - } - return; - } - - const key = sha512().update(transactionId).digest('hex'); - this.db.set(`appserviceTransactions.${key}.txnId`, transactionId) - this.db.set(`appserviceTransactions.${key}.completed`, true) - } - - readValue(key: string): string|Promise|null|undefined{ - return this.db.get(key) - } - - storeValue(key: string, value: string): Promise | void { - this.db.set(key, value); - } - - storageForUser(userId: string): IStorageProvider { - return new NamespacedKeyvProvider(userId, this); - } -} - -/** - * A namespaced storage provider that uses the disk to store information. - * @category Storage providers - */ -class NamespacedKeyvProvider implements IStorageProvider { - constructor(private prefix: string, private parent: KeyvStorageProvider) { - } - - setFilter(filter: IFilterInfo): Promise | void { - return this.parent.storeValue(`${this.prefix}_int_filter`, JSON.stringify(filter)); - } - - getFilter(): IFilterInfo | Promise { - return Promise.resolve(this.parent.readValue(`${this.prefix}_int_filter`)).then(r => r ? JSON.parse(r) : r); - } - - setSyncToken(token: string | null): Promise | void { - return this.parent.storeValue(`${this.prefix}_int_syncToken`, token || ""); - } - - getSyncToken(): string | Promise | null { - return Promise.resolve(this.parent.readValue(`${this.prefix}_int_syncToken`)).then(r => r ?? null); - } - - readValue(key: string): string | Promise | null | undefined { - return this.parent.readValue(`${this.prefix}_kv_${key}`); - } - - storeValue(key: string, value: string): Promise | void { - return this.parent.storeValue(`${this.prefix}_kv_${key}`, value); - } -}