AI/text-generation-webui
1
0
Fork 0
mirror of https://github.com/oobabooga/text-generation-webui.git synced 2024-10-01 01:26:03 -04:00
Code Issues Packages Projects Releases Wiki Activity

Minor improvements to evaluation logs

Browse Source
This commit is contained in:
oobabooga 2023-10-15 20:51:43 -07:00
parent b88b2b74a6
commit e14bde4946
2 changed files with 6 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
modules/evaluate.py
View File

@ -59,13 +59,13 @@ def calculate_perplexity(models, input_dataset, stride, _max_length):
for model in models: for model in models:
if is_in_past_evaluations(model, input_dataset, stride, _max_length): if is_in_past_evaluations(model, input_dataset, stride, _max_length):
cumulative_log += f"{model} has already been tested. Ignoring.\n\n" cumulative_log += f"`{model}` has already been tested. Ignoring.\n\n"
yield cumulative_log yield cumulative_log
continue continue
if model != 'current model': if model != 'current model':
try: try:
yield cumulative_log + f"Loading {model}...\n\n" yield cumulative_log + f"Loading `{model}`...\n\n"
model_settings = get_model_metadata(model) model_settings = get_model_metadata(model)
shared.settings.update({k: v for k, v in model_settings.items() if k in shared.settings}) # hijacking the interface defaults shared.settings.update({k: v for k, v in model_settings.items() if k in shared.settings}) # hijacking the interface defaults
update_model_parameters(model_settings) # hijacking the command-line arguments update_model_parameters(model_settings) # hijacking the command-line arguments
@ -73,11 +73,11 @@ def calculate_perplexity(models, input_dataset, stride, _max_length):
unload_model() unload_model()
shared.model, shared.tokenizer = load_model(shared.model_name) shared.model, shared.tokenizer = load_model(shared.model_name)
except: except:
cumulative_log += f"Failed to load {model}. Moving on.\n\n" cumulative_log += f"Failed to load `{model}`. Moving on.\n\n"
yield cumulative_log yield cumulative_log
continue continue
cumulative_log += f"Processing {shared.model_name}...\n\n" cumulative_log += f"Processing `{shared.model_name}`...\n\n"
yield cumulative_log + "Tokenizing the input dataset...\n\n" yield cumulative_log + "Tokenizing the input dataset...\n\n"
encodings = encode(text, add_special_tokens=False) encodings = encode(text, add_special_tokens=False)
seq_len = encodings.shape[1] seq_len = encodings.shape[1]
@ -115,7 +115,7 @@ def calculate_perplexity(models, input_dataset, stride, _max_length):
ppl = torch.exp(torch.stack(nlls).mean()) ppl = torch.exp(torch.stack(nlls).mean())
add_entry_to_past_evaluations(float(ppl), shared.model_name, input_dataset, stride, _max_length) add_entry_to_past_evaluations(float(ppl), shared.model_name, input_dataset, stride, _max_length)
save_past_evaluations(past_evaluations) save_past_evaluations(past_evaluations)
cumulative_log += f"The perplexity for {shared.model_name} is: {float(ppl)}\n\n" cumulative_log += f"The perplexity for `{shared.model_name}` is: {float(ppl)}\n\n"
yield cumulative_log yield cumulative_log

2
modules/models_settings.py
View File

@ -220,4 +220,4 @@ def save_model_settings(model, state):
with open(p, 'w') as f: with open(p, 'w') as f:
f.write(output) f.write(output)
yield (f"Settings for {model} saved to {p}") yield (f"Settings for `{model}` saved to `{p}`.")