Update replit.cpp

replit_tokenizer_detokenize returnins std::string now

Signed-off-by: niansa/tuxifan <tuxifan@posteo.de>
This commit is contained in:
niansa/tuxifan 2023-06-22 09:27:28 +02:00 committed by AT
parent 0855c0df1d
commit 47323f8591

View File

@ -146,14 +146,12 @@ std::vector<LLModel::Token> replit_tokenizer_tokenize(replit_tokenizer & tokeniz
return tokenized.first; return tokenized.first;
} }
std::string_view replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std::vector<LLModel::Token> & tokens) { std::string replit_tokenizer_detokenize(replit_tokenizer & tokenizer, const std::vector<LLModel::Token> & tokens) {
std::string text; std::string text;
for (auto token : tokens) { for (auto token : tokens) {
text += tokenizer.raw_vocab.id_to_token[token]; text += tokenizer.raw_vocab.id_to_token[token];
} }
static std::string denormalized_text; return replace_all(text, ws_symbol, " ");
denormalized_text = replace_all(text, ws_symbol, " ");
return denormalized_text;
} }
// no defaults for now // no defaults for now