mirror of
https://github.com/oobabooga/text-generation-webui.git
synced 2024-10-01 01:26:03 -04:00
Fix issue with generating on multiple GPUs
This commit is contained in:
parent
a97afa6965
commit
0a0d289537
@ -314,7 +314,7 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
|
||||
t0 = time.time()
|
||||
if args.deepspeed:
|
||||
with torch.no_grad():
|
||||
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})")
|
||||
output = eval(f"model.generate(input_ids, synced_gpus=True, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})")
|
||||
else:
|
||||
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset}){cuda}")
|
||||
reply = decode(output[0])
|
||||
@ -331,7 +331,7 @@ def generate_reply(question, tokens, inference_settings, selected_model, eos_tok
|
||||
for i in tqdm(range(tokens//8+1)):
|
||||
if args.deepspeed:
|
||||
with torch.no_grad():
|
||||
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})")
|
||||
output = eval(f"model.generate(input_ids, synced_gpus=True, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset})")
|
||||
else:
|
||||
output = eval(f"model.generate(input_ids, eos_token_id={n}, stopping_criteria=stopping_criteria_list, {preset}){cuda}")
|
||||
reply = decode(output[0])
|
||||
|
Loading…
Reference in New Issue
Block a user