From b14953e13689fba6ad719fb1c7952d7ed4018ef8 Mon Sep 17 00:00:00 2001 From: Aaron Miller Date: Thu, 8 Jun 2023 11:08:10 -0700 Subject: [PATCH] sampling: remove incorrect offset for n_vocab (#900) no effect, but avoids a *potential* bug later if we use actualVocabSize - which is for when a model has a larger embedding tensor/# of output logits than actually trained token to allow room for adding extras in finetuning - presently all of our models have had "placeholder" tokens in the vocab so this hasn't broken anything, but if the sizes did differ we want the equivalent of `logits[actualVocabSize:]` (the start point is unchanged), not `logits[-actualVocabSize:]` (this.) --- gpt4all-backend/utils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpt4all-backend/utils.cpp b/gpt4all-backend/utils.cpp index 8769315e..46827980 100644 --- a/gpt4all-backend/utils.cpp +++ b/gpt4all-backend/utils.cpp @@ -230,7 +230,7 @@ gpt_vocab::id gpt_sample_top_k_top_p( int n_logits = actualVocabSize; const auto last_n_tokens = std::vector(last_n_tokens_data, last_n_tokens_data + last_n_tokens_size); - const auto * plogits = logits.data() + logits.size() - n_logits; + const auto * plogits = logits.data(); std::vector> logits_id; logits_id.reserve(n_logits);