AI/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-10-01 01:26:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

This is the correct way of sampling 1 token at a time

Browse Source
This commit is contained in:
oobabooga 2023-01-18 21:37:21 -03:00
parent 0f01a3b1fa
commit 022960a087
1 changed files with 5 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
server.py
View File

@ -139,11 +139,11 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
preset = infile.read()
loaded_preset = inference_settings
for i in range(tokens):
input_ids = encode(question, 1)
preset = preset.replace('max_new_tokens=tokens', 'max_new_tokens=1')
cuda = ".cuda()" if args.cpu else ""
for i in range(tokens):
if eos_token is None:
output = eval(f"model.generate(input_ids, {preset}){cuda}")
else:
@ -152,7 +152,6 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
reply = tokenizer.decode(output[0], skip_special_tokens=True)
reply = reply.replace(r'<|endoftext|>', '')
question = reply
if model_name.lower().startswith('galactica'):
reply = fix_galactica(reply)
yield reply, reply, generate_basic_html(reply)
@ -162,6 +161,8 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
else:
yield reply, 'Only applicable for GALACTICA models.', generate_basic_html(reply)
input_ids = output
# Choosing the default model
if args.model is not None:
model_name = args.model