mirror of
https://github.com/ravenscroftj/turbopilot.git
synced 2024-10-01 05:06:01 +00:00
Compare commits
4 Commits
e840e5f293
...
29033d1d71
Author | SHA1 | Date | |
---|---|---|---|
|
29033d1d71 | ||
|
eaeb52fcb0 | ||
|
83cb7c042f | ||
|
eacb0d59c2 |
61
.github/workflows/codacy.yml
vendored
Normal file
61
.github/workflows/codacy.yml
vendored
Normal file
@ -0,0 +1,61 @@
|
|||||||
|
# This workflow uses actions that are not certified by GitHub.
|
||||||
|
# They are provided by a third-party and are governed by
|
||||||
|
# separate terms of service, privacy policy, and support
|
||||||
|
# documentation.
|
||||||
|
|
||||||
|
# This workflow checks out code, performs a Codacy security scan
|
||||||
|
# and integrates the results with the
|
||||||
|
# GitHub Advanced Security code scanning feature. For more information on
|
||||||
|
# the Codacy security scan action usage and parameters, see
|
||||||
|
# https://github.com/codacy/codacy-analysis-cli-action.
|
||||||
|
# For more information on Codacy Analysis CLI in general, see
|
||||||
|
# https://github.com/codacy/codacy-analysis-cli.
|
||||||
|
|
||||||
|
name: Codacy Security Scan
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches: [ "main" ]
|
||||||
|
pull_request:
|
||||||
|
# The branches below must be a subset of the branches above
|
||||||
|
branches: [ "main" ]
|
||||||
|
schedule:
|
||||||
|
- cron: '36 12 * * 3'
|
||||||
|
|
||||||
|
permissions:
|
||||||
|
contents: read
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
codacy-security-scan:
|
||||||
|
permissions:
|
||||||
|
contents: read # for actions/checkout to fetch code
|
||||||
|
security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
|
||||||
|
actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
|
||||||
|
name: Codacy Security Scan
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
# Checkout the repository to the GitHub Actions runner
|
||||||
|
- name: Checkout code
|
||||||
|
uses: actions/checkout@v3
|
||||||
|
|
||||||
|
# Execute Codacy Analysis CLI and generate a SARIF output with the security issues identified during the analysis
|
||||||
|
- name: Run Codacy Analysis CLI
|
||||||
|
uses: codacy/codacy-analysis-cli-action@d840f886c4bd4edc059706d09c6a1586111c540b
|
||||||
|
with:
|
||||||
|
# Check https://github.com/codacy/codacy-analysis-cli#project-token to get your project token from your Codacy repository
|
||||||
|
# You can also omit the token and run the tools that support default configurations
|
||||||
|
project-token: ${{ secrets.CODACY_PROJECT_TOKEN }}
|
||||||
|
verbose: true
|
||||||
|
output: results.sarif
|
||||||
|
format: sarif
|
||||||
|
# Adjust severity of non-security issues
|
||||||
|
gh-code-scanning-compat: true
|
||||||
|
# Force 0 exit code to allow SARIF file generation
|
||||||
|
# This will handover control about PR rejection to the GitHub side
|
||||||
|
max-allowed-issues: 2147483647
|
||||||
|
|
||||||
|
# Upload the SARIF file generated in the previous step
|
||||||
|
- name: Upload SARIF results file
|
||||||
|
uses: github/codeql-action/upload-sarif@v2
|
||||||
|
with:
|
||||||
|
sarif_file: results.sarif
|
@ -84,20 +84,21 @@ bool gpt_neox_eval(
|
|||||||
const int n_vocab = hparams.n_vocab;
|
const int n_vocab = hparams.n_vocab;
|
||||||
const int n_rot = hparams.n_rot;
|
const int n_rot = hparams.n_rot;
|
||||||
|
|
||||||
static size_t buf_size = 256u*1024*1024;
|
static size_t buf_size = 512u*1024*1024;
|
||||||
static void * buf = malloc(buf_size);
|
static void * buf = malloc(buf_size);
|
||||||
|
|
||||||
// use 2 scratch buffers
|
// use 2 scratch buffers
|
||||||
// TODO: very hacky solution - reimplement in a more elegant way
|
// TODO: very hacky solution - reimplement in a more elegant way
|
||||||
static size_t scr0_size = 256u*1024*1024;
|
static size_t scr0_size = 512*1024*1024;
|
||||||
static void * scr0 = malloc(scr0_size);
|
static void * scr0 = malloc(scr0_size);
|
||||||
|
|
||||||
static size_t scr1_size = 256u*1024*1024;
|
static size_t scr1_size = 512*1024*1024;
|
||||||
static void * scr1 = malloc(scr1_size);
|
static void * scr1 = malloc(scr1_size);
|
||||||
|
|
||||||
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
|
|
||||||
|
if (mem_per_token > 0 && (mem_per_token*N) > buf_size) {
|
||||||
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
|
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||||
//printf("\n%s: reallocating buffer from %zu to %zu bytes\n", __func__, buf_size, buf_size_new);
|
spdlog::debug("{}: reallocating buffer from {} to {} bytes\n", __func__, buf_size, buf_size_new);
|
||||||
|
|
||||||
|
|
||||||
// reallocate
|
// reallocate
|
||||||
@ -107,8 +108,6 @@ bool gpt_neox_eval(
|
|||||||
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
|
fprintf(stderr, "%s: failed to allocate %zu bytes\n", __func__, buf_size);
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
spdlog::debug("{}: reallocating context buffer {} -> now {} bytes of tokens in prompt = {}", __func__, buf_size, buf_size_new);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
struct ggml_init_params params = {
|
struct ggml_init_params params = {
|
||||||
@ -303,12 +302,14 @@ bool gpt_neox_eval(
|
|||||||
embd_w.resize(n_vocab);
|
embd_w.resize(n_vocab);
|
||||||
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
|
memcpy(embd_w.data(), (float *) ggml_get_data(inpL) + (n_vocab*(N-1)), sizeof(float)*n_vocab);
|
||||||
|
|
||||||
|
|
||||||
|
spdlog::debug("used_mem = {}\n", ggml_used_mem(ctx0));
|
||||||
|
|
||||||
if (mem_per_token == 0) {
|
if (mem_per_token == 0) {
|
||||||
mem_per_token = ggml_used_mem(ctx0)/N;
|
mem_per_token = ggml_used_mem(ctx0) / N; //* 4;
|
||||||
|
spdlog::debug("Set mem_per_token={} / {} * {} = {}", ggml_used_mem(ctx0), N, 4, mem_per_token);
|
||||||
|
|
||||||
}
|
}
|
||||||
spdlog::debug("used_mem = {}\n", ggml_used_mem(ctx0));
|
|
||||||
//printf("used_mem = %zu\n", ggml_used_mem(ctx0));
|
|
||||||
|
|
||||||
ggml_free(ctx0);
|
ggml_free(ctx0);
|
||||||
|
|
||||||
@ -685,7 +686,13 @@ std::stringstream GPTNEOXModel::predict_impl(std::string prompt, int max_length,
|
|||||||
|
|
||||||
std::vector<float> logits;
|
std::vector<float> logits;
|
||||||
|
|
||||||
gpt_neox_eval((*model), config.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
|
std::vector<gpt_vocab::id> test = {};
|
||||||
|
|
||||||
|
for(int i=0;i<64;i++){
|
||||||
|
test.push_back(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
gpt_neox_eval((*model), config.n_threads, 0, test, logits, mem_per_token);
|
||||||
|
|
||||||
const int64_t t_start_us = ggml_time_us();
|
const int64_t t_start_us = ggml_time_us();
|
||||||
|
|
||||||
|
@ -37,7 +37,7 @@ bool starcoder_eval(
|
|||||||
const int n_head = hparams.n_head;
|
const int n_head = hparams.n_head;
|
||||||
const int n_vocab = hparams.n_vocab;
|
const int n_vocab = hparams.n_vocab;
|
||||||
|
|
||||||
static size_t buf_size = 256u*1024*1024;
|
static size_t buf_size = 512u*1024*1024;
|
||||||
static void * buf = malloc(buf_size);
|
static void * buf = malloc(buf_size);
|
||||||
|
|
||||||
// use 2 scratch buffers
|
// use 2 scratch buffers
|
||||||
@ -48,8 +48,10 @@ bool starcoder_eval(
|
|||||||
static size_t scr1_size = 512u*1024*1024;
|
static size_t scr1_size = 512u*1024*1024;
|
||||||
static void * scr1 = malloc(scr1_size);
|
static void * scr1 = malloc(scr1_size);
|
||||||
|
|
||||||
if (mem_per_token > 0 && mem_per_token*N > buf_size) {
|
if (mem_per_token > 0 && 2*mem_per_token*N > buf_size) {
|
||||||
const size_t buf_size_new = 1.1*(mem_per_token*N); // add 10% to account for ggml object overhead
|
const size_t buf_size_new = 2*(mem_per_token*N); // add 10% to account for ggml object overhead
|
||||||
|
|
||||||
|
if(buf_size_new > buf_size){
|
||||||
spdlog::debug("{}: reallocating buffer from {} to {} bytes\n", __func__, buf_size, buf_size_new);
|
spdlog::debug("{}: reallocating buffer from {} to {} bytes\n", __func__, buf_size, buf_size_new);
|
||||||
|
|
||||||
// reallocate
|
// reallocate
|
||||||
@ -61,12 +63,15 @@ bool starcoder_eval(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
struct ggml_init_params params = {
|
struct ggml_init_params params = {
|
||||||
/*.mem_size =*/ buf_size,
|
/*.mem_size =*/ buf_size,
|
||||||
/*.mem_buffer =*/ buf,
|
/*.mem_buffer =*/ buf,
|
||||||
/*.no_alloc =*/ false,
|
/*.no_alloc =*/ false,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
|
||||||
struct ggml_context * ctx0 = ggml_init(params);
|
struct ggml_context * ctx0 = ggml_init(params);
|
||||||
struct ggml_cgraph gf = {};
|
struct ggml_cgraph gf = {};
|
||||||
|
|
||||||
@ -338,7 +343,9 @@ bool starcoder_eval(
|
|||||||
if (mem_per_token == 0) {
|
if (mem_per_token == 0) {
|
||||||
mem_per_token = ggml_used_mem(ctx0)/N;
|
mem_per_token = ggml_used_mem(ctx0)/N;
|
||||||
}
|
}
|
||||||
//printf("used_mem = %zu MB\n", ggml_used_mem(ctx0)/(1024*1024));
|
|
||||||
|
spdlog::debug("{}: used mem buf={} bytes", __func__, ggml_used_mem(ctx0));
|
||||||
|
|
||||||
|
|
||||||
ggml_free(ctx0);
|
ggml_free(ctx0);
|
||||||
|
|
||||||
@ -743,11 +750,22 @@ std::stringstream StarcoderModel::predict_impl(std::string prompt, int max_lengt
|
|||||||
size_t mem_per_token = 0;
|
size_t mem_per_token = 0;
|
||||||
|
|
||||||
std::vector<float> logits;
|
std::vector<float> logits;
|
||||||
|
std::vector<gpt_vocab::id> test = {};
|
||||||
|
|
||||||
|
for(int i=0;i<64;i++){
|
||||||
|
test.push_back(i);
|
||||||
|
}
|
||||||
|
|
||||||
|
spdlog::debug("{}: calculate required memory per token", __func__);
|
||||||
|
starcoder_eval((*model), config.n_threads, 0, test, logits, mem_per_token);
|
||||||
|
spdlog::debug("{}: mem_per_token={}", __func__, mem_per_token);
|
||||||
|
spdlog::debug("{}: total mem needed for prompt = {}*{}={}", __func__, embd_inp.size(), mem_per_token, embd_inp.size()*mem_per_token);
|
||||||
|
|
||||||
starcoder_eval((*model), config.n_threads, 0, { 0, 1, 2, 3 }, logits, mem_per_token);
|
|
||||||
|
|
||||||
for (int i = embd.size(); i < embd_inp.size() + n_predict; i++) {
|
for (int i = embd.size(); i < embd_inp.size() + n_predict; i++) {
|
||||||
// predict
|
// predict
|
||||||
|
spdlog::debug("{}: process token #{}: ", __func__, i);
|
||||||
|
|
||||||
if (embd.size() > 0) {
|
if (embd.size() > 0) {
|
||||||
const int64_t t_start_us = ggml_time_us();
|
const int64_t t_start_us = ggml_time_us();
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user