AI/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-20 00:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Encode the input string correctly

Browse Source
This commit is contained in:
oobabooga 2023-01-20 00:45:02 -03:00
parent 83584ae2d7
commit 70ff685736
1 changed files with 1 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server.py
View File

@ -170,10 +170,10 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
cuda = "" if args.cpu else ".cuda()"
n = None if eos_token is None else tokenizer.encode(eos_token, return_tensors='pt')[0][-1]
input_ids = encode(question, tokens)
# Generate the entire reply at once
if args.no_stream:
input_ids = encode(question, tokens)
output = eval(f"model.generate(input_ids, eos_token_id={n}, {preset}){cuda}")
reply = decode(output[0])
yield formatted_outputs(reply, model_name)
@ -181,7 +181,6 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
# Generate the reply 1 token at a time
else:
yield formatted_outputs(question, model_name)
input_ids = encode(question, 1)
preset = preset.replace('max_new_tokens=tokens', 'max_new_tokens=1')
for i in tqdm(range(tokens)):
output = eval(f"model.generate(input_ids, {preset}){cuda}")