AI/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2024-09-19 23:35:41 +00:00
Code Issues Packages Projects Releases Wiki Activity

When recalculating context we can't erase the BOS.

Browse Source
This commit is contained in:
Adam Treat 2023-06-12 09:21:28 -04:00 committed by AT
parent 0ae026e197
commit b906fb4057
1 changed files with 10 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
gpt4all-backend/llamamodel.cpp
View File

@ -193,7 +193,16 @@ LLModel::Token LLamaModel::sampleToken(PromptContext &promptCtx) const
bool LLamaModel::evalTokens(PromptContext &ctx, const std::vector<int32_t> &tokens) const
{
return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
// When we recalculate context we could have erased the original BOS token... we need to replace it
const bool useBOS = ctx.n_past == 0 && (ctx.tokens.empty() || ctx.tokens.front() != llama_token_bos());
if (useBOS) {
std::vector<int32_t> myTokens;
myTokens.push_back(llama_token_bos());
myTokens.insert(myTokens.end(), tokens.begin(), tokens.end());
ctx.n_past += 1;
return llama_eval(d_ptr->ctx, myTokens.data(), myTokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
} else
return llama_eval(d_ptr->ctx, tokens.data(), tokens.size(), ctx.n_past, d_ptr->n_threads) == 0;
}
int32_t LLamaModel::contextLength() const