AI/turbopilot
1
0
Fork 0
mirror of https://github.com/ravenscroftj/turbopilot.git synced 2024-06-28 23:32:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

expose batch size flag to cli

Browse Source
This commit is contained in:
James Ravenscroft 2023-08-24 11:40:19 +01:00
parent e0adf0519b
commit 2d617b458e
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main.cpp
View File

@ -60,6 +60,11 @@ int main(int argc, char **argv)
.default_value(0.1f)
.scan<'g', float>();
program.add_argument("-b", "--batch-size")
.help("set batch size for model completion")
.default_value(512)
.scan<'i',int>();
program.add_argument("prompt").remaining();
@ -96,6 +101,7 @@ int main(int argc, char **argv)
config.n_threads = program.get<int>("--threads");
config.temp = program.get<float>("--temperature");
config.top_p = program.get<float>("--top-p");
config.n_batch = program.get<int>("--batch-size");
if(model_type.compare("codegen") == 0) {
spdlog::info("Initializing GPT-J type model for '{}' model", model_type);
@ -131,6 +137,7 @@ int main(int argc, char **argv)
return "Hello world";
});
CROW_ROUTE(app, "/copilot_internal/v2/token")([](){
//return "Hello world";