AI/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-20 00:07:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make universal tokenizer, xformers, sdp-attention apply to monkey patch

Browse Source
This commit is contained in:
oobabooga 2023-04-25 23:18:11 -03:00
parent f39c99fa14
commit f642135517
2 changed files with 2 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
modules/models.py
View File

@ -149,8 +149,7 @@ def load_model(model_name):
print("Warning: applying the monkey patch for using LoRAs in 4-bit mode.\nIt may cause undefined behavior outside its intended scope.")
from modules.monkey_patch_gptq_lora import load_model_llama
model, tokenizer = load_model_llama(model_name)
return model, tokenizer
model, _ = load_model_llama(model_name)
# No monkey patch
else:

10
modules/monkey_patch_gptq_lora.py
View File

@ -18,17 +18,16 @@ from modules.GPTQ_loader import find_quantized_model_file
replace_peft_model_with_gptq_lora_model()
def load_model_llama(model_name):
config_path = str(Path(f'{shared.args.model_dir}/{model_name}'))
model_path = str(find_quantized_model_file(model_name))
model, tokenizer = load_llama_model_4bit_low_ram(config_path, model_path, groupsize=shared.args.groupsize, is_v1_model=False)
for n, m in model.named_modules():
if isinstance(m, Autograd4bitQuantLinear) or isinstance(m, Linear4bitLt):
if m.is_v1_model:
m.zeros = m.zeros.half()
m.scales = m.scales.half()
m.bias = m.bias.half()
autograd_4bit.use_new = True
autograd_4bit.auto_switch = True
@ -36,11 +35,4 @@ def load_model_llama(model_name):
wrapper = AMPWrapper(model)
wrapper.apply_generate()
try:
tokenizer.eos_token_id = 2
tokenizer.bos_token_id = 1
tokenizer.pad_token_id = 0
except:
pass
return model, tokenizer