AI/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-10-01 01:26:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

Mention time and it/s in terminal with streaming off

Browse Source
This commit is contained in:
oobabooga 2023-01-22 20:07:19 -03:00
parent d87310ad61
commit ebf720585b
1 changed files with 3 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
server.py
View File

@ -182,8 +182,11 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
# Generate the entire reply at once # Generate the entire reply at once
if args.no_stream: if args.no_stream:
t0 = time.time()
output = eval(f"model.generate(input_ids, eos_token_id={n}, {preset}){cuda}") output = eval(f"model.generate(input_ids, eos_token_id={n}, {preset}){cuda}")
reply = decode(output[0]) reply = decode(output[0])
t1 = time.time()
print(f"Output generated in {(t1-t0):.2f} seconds ({(len(output[0])-len(input_ids[0]))/(t1-t0):.2f} it/s)")
yield formatted_outputs(reply, model_name) yield formatted_outputs(reply, model_name)
# Generate the reply 1 token at a time # Generate the reply 1 token at a time