AI/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-09-19 15:56:56 +00:00
Code Issues Packages Projects Releases Wiki Activity

Autodetect rms_norm_eps and n_gqa for llama-2-70b

Browse Source
This commit is contained in:
oobabooga 2023-07-24 15:26:29 -07:00
parent a07d070b6c
commit 08c622df2e
2 changed files with 5 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
models/config.yaml
View File

@ -277,3 +277,6 @@ TheBloke_WizardLM-30B-GPTQ:
.*llama-(2|v2).*chat:
mode: 'instruct'
instruction_template: 'Llama-v2'
.*llama.*70b.*ggml.*\.bin:
n_gqa: 8
rms_norm_eps: 1.0e-5

2
server.py
View File

@ -1104,6 +1104,8 @@ if __name__ == "__main__":
'skip_special_tokens': shared.settings['skip_special_tokens'],
'custom_stopping_strings': shared.settings['custom_stopping_strings'],
'truncation_length': shared.settings['truncation_length'],
'n_gqa': 0,
'rms_norm_eps': 0,
}
shared.model_config.move_to_end('.*', last=False) # Move to the beginning