mirror of
https://github.com/nomic-ai/gpt4all.git
synced 2024-09-20 07:45:54 +00:00
8d19ef3909
* backend: factor out common structs in model code prepping to hack on these by hopefully making there be fewer places to fix the same bug rename * use common buffer wrapper instead of manual malloc * fix replit compile warnings
37 lines
596 B
C++
37 lines
596 B
C++
#pragma once
|
|
#include <cstdint>
|
|
#include <cstddef>
|
|
#include <ggml.h>
|
|
|
|
struct llm_buffer {
|
|
uint8_t * addr = NULL;
|
|
size_t size = 0;
|
|
|
|
void resize(size_t size) {
|
|
delete[] addr;
|
|
addr = new uint8_t[size];
|
|
this->size = size;
|
|
}
|
|
|
|
~llm_buffer() {
|
|
delete[] addr;
|
|
}
|
|
};
|
|
|
|
struct llm_kv_cache {
|
|
struct ggml_tensor * k;
|
|
struct ggml_tensor * v;
|
|
|
|
struct ggml_context * ctx = NULL;
|
|
|
|
llm_buffer buf;
|
|
|
|
int n; // number of tokens currently in the cache
|
|
|
|
~llm_kv_cache() {
|
|
if (ctx) {
|
|
ggml_free(ctx);
|
|
}
|
|
}
|
|
};
|