AI/turbopilot
1
0
Fork 0
mirror of https://github.com/ravenscroftj/turbopilot.git synced 2024-06-28 23:32:20 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #59 from ravenscroftj/feature/batch-flag

Browse Source 
expose batch size flag to cli
This commit is contained in:
James Ravenscroft 2023-08-24 14:13:40 +01:00 committed by GitHub
commit 6b0a25cb71
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/main.cpp
View File

@ -60,6 +60,11 @@ int main(int argc, char **argv)
.default_value(0.1f)
.scan<'g', float>();
program.add_argument("-b", "--batch-size")
.help("set batch size for model completion")
.default_value(512)
.scan<'i',int>();
program.add_argument("prompt").remaining();
@ -96,6 +101,7 @@ int main(int argc, char **argv)
config.n_threads = program.get<int>("--threads");
config.temp = program.get<float>("--temperature");
config.top_p = program.get<float>("--top-p");
config.n_batch = program.get<int>("--batch-size");
if(model_type.compare("codegen") == 0) {
spdlog::info("Initializing GPT-J type model for '{}' model", model_type);
@ -131,6 +137,7 @@ int main(int argc, char **argv)
return "Hello world";
});
CROW_ROUTE(app, "/copilot_internal/v2/token")([](){
//return "Hello world";