Autodetect rms_norm_eps and n_gqa for llama-2-70b

This commit is contained in:
oobabooga 2023-07-24 15:26:29 -07:00
parent a07d070b6c
commit 08c622df2e
2 changed files with 5 additions and 0 deletions

View File

@ -277,3 +277,6 @@ TheBloke_WizardLM-30B-GPTQ:
.*llama-(2|v2).*chat:
mode: 'instruct'
instruction_template: 'Llama-v2'
.*llama.*70b.*ggml.*\.bin:
n_gqa: 8
rms_norm_eps: 1.0e-5

View File

@ -1104,6 +1104,8 @@ if __name__ == "__main__":
'skip_special_tokens': shared.settings['skip_special_tokens'],
'custom_stopping_strings': shared.settings['custom_stopping_strings'],
'truncation_length': shared.settings['truncation_length'],
'n_gqa': 0,
'rms_norm_eps': 0,
}
shared.model_config.move_to_end('.*', last=False) # Move to the beginning