mirror of
https://github.com/oobabooga/text-generation-webui.git
synced 2024-09-20 00:07:06 +00:00
37 lines
705 B
Python
37 lines
705 B
Python
from modules.text_generation import decode, encode
|
|
|
|
|
|
def token_count(prompt):
|
|
tokens = encode(prompt)[0]
|
|
|
|
return {
|
|
'results': [{
|
|
'tokens': len(tokens)
|
|
}]
|
|
}
|
|
|
|
|
|
def token_encode(input, encoding_format):
|
|
# if isinstance(input, list):
|
|
tokens = encode(input)[0]
|
|
|
|
return {
|
|
'results': [{
|
|
'tokens': tokens,
|
|
'length': len(tokens),
|
|
}]
|
|
}
|
|
|
|
|
|
def token_decode(tokens, encoding_format):
|
|
# if isinstance(input, list):
|
|
# if encoding_format == "base64":
|
|
# tokens = base64_to_float_list(tokens)
|
|
output = decode(tokens)[0]
|
|
|
|
return {
|
|
'results': [{
|
|
'text': output
|
|
}]
|
|
}
|