mirror of
https://github.com/oobabooga/text-generation-webui.git
synced 2024-10-01 01:26:03 -04:00
ExLlamaV2: fix loading when autosplit is not set
This commit is contained in:
parent
af0bbf5b13
commit
d6bd71db7f
@ -51,20 +51,21 @@ class Exllamav2Model:
|
||||
|
||||
model = ExLlamaV2(config)
|
||||
|
||||
if shared.args.cache_8bit:
|
||||
cache = ExLlamaV2Cache_8bit(model, lazy=True)
|
||||
else:
|
||||
cache = ExLlamaV2Cache(model, lazy=True)
|
||||
|
||||
if shared.args.autosplit:
|
||||
model.load_autosplit(cache)
|
||||
else:
|
||||
if not shared.args.autosplit:
|
||||
split = None
|
||||
if shared.args.gpu_split:
|
||||
split = [float(alloc) for alloc in shared.args.gpu_split.split(",")]
|
||||
|
||||
model.load(split)
|
||||
|
||||
if shared.args.cache_8bit:
|
||||
cache = ExLlamaV2Cache_8bit(model, lazy=shared.args.autosplit)
|
||||
else:
|
||||
cache = ExLlamaV2Cache(model, lazy=shared.args.autosplit)
|
||||
|
||||
if shared.args.autosplit:
|
||||
model.load_autosplit(cache)
|
||||
|
||||
tokenizer = ExLlamaV2Tokenizer(config)
|
||||
generator = ExLlamaV2StreamingGenerator(model, cache, tokenizer)
|
||||
|
||||
|
@ -36,24 +36,26 @@ class Exllamav2HF(PreTrainedModel):
|
||||
def __init__(self, config: ExLlamaV2Config):
|
||||
super().__init__(PretrainedConfig())
|
||||
self.ex_config = config
|
||||
self.ex_model = ExLlamaV2(config)
|
||||
self.loras = None
|
||||
self.generation_config = GenerationConfig()
|
||||
|
||||
if shared.args.cache_8bit:
|
||||
self.ex_cache = ExLlamaV2Cache_8bit(self.ex_model, lazy=True)
|
||||
else:
|
||||
self.ex_cache = ExLlamaV2Cache(self.ex_model, lazy=True)
|
||||
self.ex_model = ExLlamaV2(config)
|
||||
|
||||
if shared.args.autosplit:
|
||||
self.ex_model.load_autosplit(self.ex_cache)
|
||||
else:
|
||||
if not shared.args.autosplit:
|
||||
split = None
|
||||
if shared.args.gpu_split:
|
||||
split = [float(alloc) for alloc in shared.args.gpu_split.split(",")]
|
||||
|
||||
self.ex_model.load(split)
|
||||
|
||||
if shared.args.cache_8bit:
|
||||
self.ex_cache = ExLlamaV2Cache_8bit(self.ex_model, lazy=shared.args.autosplit)
|
||||
else:
|
||||
self.ex_cache = ExLlamaV2Cache(self.ex_model, lazy=shared.args.autosplit)
|
||||
|
||||
if shared.args.autosplit:
|
||||
self.ex_model.load_autosplit(self.ex_cache)
|
||||
|
||||
self.past_seq = None
|
||||
if shared.args.cfg_cache:
|
||||
if shared.args.cache_8bit:
|
||||
|
Loading…
Reference in New Issue
Block a user