2023-05-20 17:42:17 -04:00
|
|
|
import json
|
|
|
|
|
|
|
|
|
|
import requests
|
|
|
|
|
|
|
|
|
|
# For local streaming, the websockets are hosted without ssl - http://
|
|
|
|
|
HOST = 'localhost:5000'
|
|
|
|
|
URI = f'http://{HOST}/api/v1/chat'
|
|
|
|
|
|
|
|
|
|
# For reverse-proxied streaming, the remote will likely host with ssl - https://
|
|
|
|
|
# URI = 'https://your-uri-here.trycloudflare.com/api/v1/generate'
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
def run(user_input, history):
|
|
|
|
|
request = {
|
|
|
|
|
'user_input': user_input,
|
|
|
|
|
'history': history,
|
|
|
|
|
'mode': 'instruct', # Valid options: 'chat', 'chat-instruct', 'instruct'
|
|
|
|
|
'character': 'Example',
|
|
|
|
|
'instruction_template': 'Vicuna-v1.1',
|
|
|
|
|
|
|
|
|
|
'regenerate': False,
|
|
|
|
|
'_continue': False,
|
|
|
|
|
'stop_at_newline': False,
|
|
|
|
|
'chat_prompt_size': 2048,
|
|
|
|
|
'chat_generation_attempts': 1,
|
|
|
|
|
'chat-instruct_command': 'Continue the chat dialogue below. Write a single reply for the character "<|character|>".\n\n<|prompt|>',
|
|
|
|
|
|
|
|
|
|
'max_new_tokens': 250,
|
|
|
|
|
'do_sample': True,
|
|
|
|
|
'temperature': 0.7,
|
|
|
|
|
'top_p': 0.1,
|
|
|
|
|
'typical_p': 1,
|
2023-05-21 14:11:57 -04:00
|
|
|
'epsilon_cutoff': 0, # In units of 1e-4
|
|
|
|
|
'eta_cutoff': 0, # In units of 1e-4
|
2023-05-20 17:42:17 -04:00
|
|
|
'repetition_penalty': 1.18,
|
|
|
|
|
'top_k': 40,
|
|
|
|
|
'min_length': 0,
|
|
|
|
|
'no_repeat_ngram_size': 0,
|
|
|
|
|
'num_beams': 1,
|
|
|
|
|
'penalty_alpha': 0,
|
|
|
|
|
'length_penalty': 1,
|
|
|
|
|
'early_stopping': False,
|
2023-05-22 18:37:24 -04:00
|
|
|
'mirostat_mode': 0,
|
|
|
|
|
'mirostat_tau': 5,
|
|
|
|
|
'mirostat_eta': 0.1,
|
2023-05-20 17:42:17 -04:00
|
|
|
'seed': -1,
|
|
|
|
|
'add_bos_token': True,
|
|
|
|
|
'truncation_length': 2048,
|
|
|
|
|
'ban_eos_token': False,
|
|
|
|
|
'skip_special_tokens': True,
|
|
|
|
|
'stopping_strings': []
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
response = requests.post(URI, json=request)
|
|
|
|
|
|
|
|
|
|
if response.status_code == 200:
|
|
|
|
|
result = response.json()['results'][0]['history']
|
|
|
|
|
print(json.dumps(result, indent=4))
|
|
|
|
|
print()
|
|
|
|
|
print(result['visible'][-1][1])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
|
|
|
|
user_input = "Please give me a step-by-step guide on how to plant a tree in my backyard."
|
|
|
|
|
|
|
|
|
|
# Basic example
|
|
|
|
|
history = {'internal': [], 'visible': []}
|
|
|
|
|
|
|
|
|
|
# "Continue" example. Make sure to set '_continue' to True above
|
|
|
|
|
# arr = [user_input, 'Surely, here is']
|
|
|
|
|
# history = {'internal': [arr], 'visible': [arr]}
|
|
|
|
|
|
|
|
|
|
run(user_input, history)
|
