AI/gpt4all
1
0
Fork 0
mirror of https://github.com/nomic-ai/gpt4all.git synced 2024-10-01 01:06:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

Update replit.cpp

Browse Source 
replit_tokenizer_detokenize returnins std::string now

Signed-off-by: niansa/tuxifan <tuxifan@posteo.de>
This commit is contained in:
niansa/tuxifan 2023-06-22 09:27:28 +02:00 committed by AT
parent 0855c0df1d
commit 47323f8591
1 changed files with 2 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
gpt4all-backend/replit.cpp
View File

@ -146,14 +146,12 @@ std::vector<LLModel::Token> replit_tokenizer_tokenize(replit_tokenizer & tokeniz
return tokenized.first;
}
std::string_view replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std::vector<LLModel::Token> & tokens) {
std::string replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std::vector<LLModel::Token> & tokens) {
std::string text;
for (auto token : tokens) {
text += tokenizer.raw_vocab.id_to_token[token];
}
static std::string denormalized_text;
denormalized_text = replace_all(text, ws_symbol, " ");
return denormalized_text;
return replace_all(text, ws_symbol, " ");
}
// no defaults for now