From c3e4c9fdc261899dce3330cec0e851d8599bf805 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Fri, 15 Sep 2023 18:51:55 -0700 Subject: [PATCH] Add a simple tokenizer to the UI --- css/main.css | 5 ++--- modules/ui_default.py | 7 +++++++ modules/ui_notebook.py | 7 +++++++ modules/ui_parameters.py | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/css/main.css b/css/main.css index 06569f7f..f7641f11 100644 --- a/css/main.css +++ b/css/main.css @@ -120,7 +120,7 @@ div.svelte-15lo0d8 > *, div.svelte-15lo0d8 > .form > * { } .textbox_logits textarea { - height: calc(100dvh - 241px); + height: calc(100dvh - 232px); } .textbox_logits_notebook textarea { @@ -131,8 +131,7 @@ div.svelte-15lo0d8 > *, div.svelte-15lo0d8 > .form > * { .textbox_default_output textarea, .textbox_logits textarea, .textbox_logits_notebook textarea, -.textbox textarea -{ +.textbox textarea { font-size: 16px !important; color: #46464A !important; } diff --git a/modules/ui_default.py b/modules/ui_default.py index 54d1a220..38056701 100644 --- a/modules/ui_default.py +++ b/modules/ui_default.py @@ -3,6 +3,7 @@ import gradio as gr from modules import logits, shared, ui, utils from modules.prompts import count_tokens, load_prompt from modules.text_generation import ( + encode, generate_reply_wrapper, stop_everything_event ) @@ -54,6 +55,10 @@ def create_ui(): shared.gradio['logits-default'] = gr.Textbox(lines=23, label='Output', elem_classes=['textbox_logits', 'add_scrollbar']) shared.gradio['logits-default-previous'] = gr.Textbox(lines=23, label='Previous output', elem_classes=['textbox_logits', 'add_scrollbar']) + with gr.Tab('Tokens'): + shared.gradio['get_tokens-default'] = gr.Button('Get token IDs for the input') + shared.gradio['tokens-default'] = gr.Textbox(lines=23, label='Tokens', elem_classes=['textbox_logits', 'add_scrollbar']) + def create_event_handlers(): shared.gradio['Generate-default'].click( @@ -94,3 +99,5 @@ def create_event_handlers(): shared.gradio['get_logits-default'].click( ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then( logits.get_next_logits, gradio('textbox-default', 'interface_state', 'use_samplers-default', 'logits-default'), gradio('logits-default', 'logits-default-previous'), show_progress=False) + + shared.gradio['get_tokens-default'].click(lambda x : str(encode(x)[0].tolist()), gradio('textbox-default'), gradio('tokens-default'), show_progress=False) diff --git a/modules/ui_notebook.py b/modules/ui_notebook.py index ba2775d5..269de3ec 100644 --- a/modules/ui_notebook.py +++ b/modules/ui_notebook.py @@ -3,6 +3,7 @@ import gradio as gr from modules import logits, shared, ui, utils from modules.prompts import count_tokens, load_prompt from modules.text_generation import ( + encode, generate_reply_wrapper, stop_everything_event ) @@ -40,6 +41,10 @@ def create_ui(): shared.gradio['logits-notebook'] = gr.Textbox(lines=23, label='Output', elem_classes=['textbox_logits_notebook', 'add_scrollbar']) shared.gradio['logits-notebook-previous'] = gr.Textbox(lines=23, label='Previous output', elem_classes=['textbox_logits_notebook', 'add_scrollbar']) + with gr.Tab('Tokens'): + shared.gradio['get_tokens-notebook'] = gr.Button('Get token IDs for the input') + shared.gradio['tokens-notebook'] = gr.Textbox(lines=23, label='Tokens', elem_classes=['textbox_logits_notebook', 'add_scrollbar']) + with gr.Row(): shared.gradio['Generate-notebook'] = gr.Button('Generate', variant='primary', elem_classes='small-button') shared.gradio['Stop-notebook'] = gr.Button('Stop', elem_classes='small-button', elem_id='stop') @@ -96,3 +101,5 @@ def create_event_handlers(): shared.gradio['get_logits-notebook'].click( ui.gather_interface_values, gradio(shared.input_elements), gradio('interface_state')).then( logits.get_next_logits, gradio('textbox-notebook', 'interface_state', 'use_samplers-notebook', 'logits-notebook'), gradio('logits-notebook', 'logits-notebook-previous'), show_progress=False) + + shared.gradio['get_tokens-notebook'].click(lambda x : str(encode(x)[0].tolist()), gradio('textbox-notebook'), gradio('tokens-notebook'), show_progress=False) diff --git a/modules/ui_parameters.py b/modules/ui_parameters.py index 32fb1c02..9fbe6456 100644 --- a/modules/ui_parameters.py +++ b/modules/ui_parameters.py @@ -118,7 +118,7 @@ def create_ui(default_preset): with gr.Column(): shared.gradio['auto_max_new_tokens'] = gr.Checkbox(value=shared.settings['auto_max_new_tokens'], label='auto_max_new_tokens', info='Expand max_new_tokens to the available context length.') shared.gradio['ban_eos_token'] = gr.Checkbox(value=shared.settings['ban_eos_token'], label='Ban the eos_token', info='Forces the model to never end the generation prematurely.') - shared.gradio['custom_token_bans'] = gr.Textbox(value=shared.settings['custom_token_bans'] or None, label='Custom token bans', info='Specific token IDs to ban from generating, comma-separated. The IDs can be found in a tokenizer.json file.') + shared.gradio['custom_token_bans'] = gr.Textbox(value=shared.settings['custom_token_bans'] or None, label='Custom token bans', info='Specific token IDs to ban from generating, comma-separated. The IDs can be found in the Default or Notebook tab.') shared.gradio['add_bos_token'] = gr.Checkbox(value=shared.settings['add_bos_token'], label='Add the bos_token to the beginning of prompts', info='Disabling this can make the replies more creative.') shared.gradio['skip_special_tokens'] = gr.Checkbox(value=shared.settings['skip_special_tokens'], label='Skip special tokens', info='Some specific models need this unset.') shared.gradio['stream'] = gr.Checkbox(value=shared.settings['stream'], label='Activate text streaming')