diff --git a/MODELS.md b/MODELS.md index 1d2c455..0ca2228 100644 --- a/MODELS.md +++ b/MODELS.md @@ -6,9 +6,9 @@ | Model Name | RAM Requirement | Direct Download | HF Project Link | |---------------------|-----------------|-----------------|-----------------| -| StarCoder | ~3GiB | [:arrow_down:](https://huggingface.co/TheBloke/stablecode-instruct-alpha-3b-GGML/blob/main/stablecode-instruct-alpha-3b.ggmlv1.q4_0.bin) | [:hugs:](https://huggingface.co/TheBloke/stablecode-instruct-alpha-3b-GGML/) | - +| StableCode | ~3GiB | [:arrow_down:](https://huggingface.co/TheBloke/stablecode-instruct-alpha-3b-GGML/blob/main/stablecode-instruct-alpha-3b.ggmlv1.q4_0.bin) | [:hugs:](https://huggingface.co/TheBloke/stablecode-instruct-alpha-3b-GGML/) | +To run in Turbopilot set model type `-m stablecode` ## "Coder" family models @@ -23,7 +23,7 @@ This model is primarily trained on Python, Java and Javscript. | Model Name | RAM Requirement | Direct Download | HF Project Link | |---------------------|-----------------|-----------------|-----------------| -| StarCoder | ~2GiB | [:arrow_down:](https://huggingface.co/mike-ravkine/gpt_bigcode-santacoder-GGML/resolve/main/santacoder-q4_0.bin) | [:hugs:](https://huggingface.co/mike-ravkine/gpt_bigcode-santacoder-GGML/) | +| SantaCoder | ~2GiB | [:arrow_down:](https://huggingface.co/mike-ravkine/gpt_bigcode-santacoder-GGML/resolve/main/santacoder-q4_0.bin) | [:hugs:](https://huggingface.co/mike-ravkine/gpt_bigcode-santacoder-GGML/) | To run in Turbopilot set model type `-m starcoder` @@ -39,7 +39,7 @@ Even when quantized, WizardCoder is a large model that takes up a significant am |---------------------|-----------------|-----------------|-----------------| | WizardCoder | ~12GiB | [:arrow_down:](https://huggingface.co/TheBloke/WizardCoder-15B-1.0-GGML/resolve/main/WizardCoder-15B-1.0.ggmlv3.q4_0.bin) | [:hugs:](https://huggingface.co/TheBloke/WizardCoder-15B-1.0-GGML/) | -To run in Turbopilot set model type `-m starcoder` +To run in Turbopilot set model type `-m wizardcoder` ### StarCoder (Released 4/5/2023) diff --git a/src/main.cpp b/src/main.cpp index 04c1fb3..df78e97 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -60,6 +60,11 @@ int main(int argc, char **argv) .default_value(0.1f) .scan<'g', float>(); + program.add_argument("-b", "--batch-size") + .help("set batch size for model completion") + .default_value(512) + .scan<'i',int>(); + program.add_argument("prompt").remaining(); @@ -96,6 +101,7 @@ int main(int argc, char **argv) config.n_threads = program.get("--threads"); config.temp = program.get("--temperature"); config.top_p = program.get("--top-p"); + config.n_batch = program.get("--batch-size"); if(model_type.compare("codegen") == 0) { spdlog::info("Initializing GPT-J type model for '{}' model", model_type); @@ -136,6 +142,7 @@ int main(int argc, char **argv) return "Hello world"; }); + CROW_ROUTE(app, "/copilot_internal/v2/token")([](){ //return "Hello world";