From d8df41d101e5be0437e9b0c4d12d1fb784b5d887 Mon Sep 17 00:00:00 2001 From: Max Kammler Date: Tue, 7 Nov 2023 10:08:54 +0100 Subject: [PATCH 1/3] add option to increase token size; closes #246 --- src/env.ts | 6 +++++- src/index.ts | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/env.ts b/src/env.ts index 503d4b5..7db4f40 100644 --- a/src/env.ts +++ b/src/env.ts @@ -38,6 +38,8 @@ export const { CHATGPT_IGNORE_MEDIA, CHATGPT_REVERSE_PROXY, CHATGPT_TEMPERATURE, + CHATGPT_MAX_CONTEXT_TOKENS, + CHATGPT_MAX_PROMPT_TOKENS, } = parseEnv(process.env, { DATA_PATH: { schema: z.string().default("./storage"), description: "Set to /storage/ if using docker, ./storage if running without" }, KEYV_BACKEND: { schema: z.enum(["file", "other"]).default("file"),description: "Set the Keyv backend to 'file' or 'other' if other set KEYV_URL" }, @@ -72,5 +74,7 @@ export const { CHATGPT_PROMPT_PREFIX: { schema: z.string().default('Instructions:\nYou are ChatGPT, a large language model trained by OpenAI.'), description: "Instructions to feed to ChatGPT on startup"}, CHATGPT_IGNORE_MEDIA: { schema: z.boolean().default(false), description: "Wether or not the bot should react to non-text messages"}, CHATGPT_REVERSE_PROXY: { schema: z.string().default(""), description: "Change the api url to use another (OpenAI-compatible) API endpoint" }, - CHATGPT_TEMPERATURE: { schema: z.number().default(0.8), description: "Set the temperature for the model" } + CHATGPT_TEMPERATURE: { schema: z.number().default(0.8), description: "Set the temperature for the model" }, + CHATGPT_MAX_CONTEXT_TOKENS: { schema: z.number().default(4097), description: "Davinci models have a max context length of 4097 tokens, but you may need to change this for other models." } + CHATGPT_MAX_PROMPT_TOKENS { schema: z.number().default(3097), description: "You might want to lower this to save money if using a paid model. Earlier messages will be dropped until the prompt is within the limit." } }); diff --git a/src/index.ts b/src/index.ts index 3a618a6..54ce3f0 100644 --- a/src/index.ts +++ b/src/index.ts @@ -11,7 +11,7 @@ import { DATA_PATH, KEYV_URL, OPENAI_AZURE, OPENAI_API_KEY, MATRIX_HOMESERVER_URL, MATRIX_ACCESS_TOKEN, MATRIX_AUTOJOIN, MATRIX_BOT_PASSWORD, MATRIX_BOT_USERNAME, MATRIX_ENCRYPTION, MATRIX_THREADS, CHATGPT_CONTEXT, CHATGPT_API_MODEL, KEYV_BOT_STORAGE, KEYV_BACKEND, CHATGPT_PROMPT_PREFIX, MATRIX_WELCOME, - CHATGPT_REVERSE_PROXY, CHATGPT_TEMPERATURE + CHATGPT_REVERSE_PROXY, CHATGPT_TEMPERATURE, CHATGPT_MAX_CONTEXT_TOKENS, CHATGPT_MAX_PROMPT_TOKENS } from './env.js' import CommandHandler from "./handlers.js" import { KeyvStorageProvider } from './storage.js' @@ -66,6 +66,8 @@ async function main() { debug: false, azure: OPENAI_AZURE, reverseProxyUrl: CHATGPT_REVERSE_PROXY, + maxContextTokens: CHATGPT_MAX_CONTEXT_TOKENS, + maxPromptTokens: CHATGPT_MAX_PROMPT_TOKENS }; const chatgpt = new ChatGPTClient(OPENAI_API_KEY, clientOptions, cacheOptions); From b07ef97e33ef06f244452d58fdc624232c2880d6 Mon Sep 17 00:00:00 2001 From: Max Kammler Date: Tue, 7 Nov 2023 10:18:30 +0100 Subject: [PATCH 2/3] add information about token size parameters --- .env.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.env.example b/.env.example index 2d7824c..9c0e1ba 100644 --- a/.env.example +++ b/.env.example @@ -17,6 +17,10 @@ CHATGPT_API_MODEL=gpt-3.5-turbo #CHATGPT_REVERSE_PROXY=https://api.openai.com/v1/chat/completions # (Optional) Set the temperature of the model. 0.0 is deterministic, 1.0 is very creative. # CHATGPT_TEMPERATURE=0.8 +# (Optional) (Optional) Davinci models have a max context length of 4097 tokens, but you may need to change this for other models. +# CHATGPT_MAX_CONTEXT_TOKENS=4097 +# You might want to lower this to save money if using a paid model. Earlier messages will be dropped until the prompt is within the limit. +# CHATGPT_MAX_PROMPT_TOKENS=3097 # Set data store settings KEYV_BACKEND=file From 2d437334a6881a129fea4fe8bb4599ec3791afd0 Mon Sep 17 00:00:00 2001 From: Max Kammler Date: Tue, 7 Nov 2023 10:22:06 +0100 Subject: [PATCH 3/3] fix typo --- src/env.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/env.ts b/src/env.ts index 7db4f40..08059e7 100644 --- a/src/env.ts +++ b/src/env.ts @@ -75,6 +75,6 @@ export const { CHATGPT_IGNORE_MEDIA: { schema: z.boolean().default(false), description: "Wether or not the bot should react to non-text messages"}, CHATGPT_REVERSE_PROXY: { schema: z.string().default(""), description: "Change the api url to use another (OpenAI-compatible) API endpoint" }, CHATGPT_TEMPERATURE: { schema: z.number().default(0.8), description: "Set the temperature for the model" }, - CHATGPT_MAX_CONTEXT_TOKENS: { schema: z.number().default(4097), description: "Davinci models have a max context length of 4097 tokens, but you may need to change this for other models." } - CHATGPT_MAX_PROMPT_TOKENS { schema: z.number().default(3097), description: "You might want to lower this to save money if using a paid model. Earlier messages will be dropped until the prompt is within the limit." } + CHATGPT_MAX_CONTEXT_TOKENS: { schema: z.number().default(4097), description: "Davinci models have a max context length of 4097 tokens, but you may need to change this for other models." }, + CHATGPT_MAX_PROMPT_TOKENS: { schema: z.number().default(3097), description: "You might want to lower this to save money if using a paid model. Earlier messages will be dropped until the prompt is within the limit." }, });