AI/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-10-01 01:26:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

Make llava/minigpt-4 work with AutoGPTQ

Browse Source
This commit is contained in:
oobabooga 2023-06-11 17:52:23 -03:00
parent f4defde752
commit e471919e6d
1 changed files with 11 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
modules/AutoGPTQ_loader.py
View File

@ -51,4 +51,15 @@ def load_quantized(model_name):
logger.info(f"The AutoGPTQ params are: {params}") logger.info(f"The AutoGPTQ params are: {params}")
model = AutoGPTQForCausalLM.from_quantized(path_to_model, **params) model = AutoGPTQForCausalLM.from_quantized(path_to_model, **params)
# These lines fix the multimodal extension when used with AutoGPTQ
if not hasattr(model, 'dtype'):
model.dtype = model.model.dtype
if not hasattr(model, 'embed_tokens'):
model.embed_tokens = model.model.model.embed_tokens
if not hasattr(model.model, 'embed_tokens'):
model.model.embed_tokens = model.model.model.embed_tokens
return model return model