gpt4all/llm.cpp

405 lines
12 KiB
C++
Raw Normal View History

2023-04-09 03:28:39 +00:00
#include "llm.h"
2023-04-19 01:10:06 +00:00
#include "download.h"
2023-04-09 03:28:39 +00:00
#include <QCoreApplication>
#include <QDir>
#include <QFile>
2023-04-11 03:34:34 +00:00
#include <QProcess>
2023-04-09 03:28:39 +00:00
#include <QResource>
2023-04-10 20:33:14 +00:00
#include <fstream>
2023-04-09 03:28:39 +00:00
class MyLLM: public LLM { };
Q_GLOBAL_STATIC(MyLLM, llmInstance)
LLM *LLM::globalInstance()
{
return llmInstance();
}
static LLModel::PromptContext s_ctx;
static QString modelFilePath(const QString &modelName)
{
QString appPath = QCoreApplication::applicationDirPath()
+ QDir::separator() + "ggml-" + modelName + ".bin";
QFileInfo infoAppPath(appPath);
if (infoAppPath.exists())
return appPath;
QString downloadPath = Download::globalInstance()->downloadLocalModelsPath()
+ QDir::separator() + "ggml-" + modelName + ".bin";
QFileInfo infoLocalPath(downloadPath);
if (infoLocalPath.exists())
return downloadPath;
return QString();
}
LLMObject::LLMObject()
2023-04-09 03:28:39 +00:00
: QObject{nullptr}
, m_llmodel(nullptr)
, m_responseTokens(0)
, m_responseLogits(0)
2023-04-09 03:28:39 +00:00
{
moveToThread(&m_llmThread);
connect(&m_llmThread, &QThread::started, this, &LLMObject::loadModel);
2023-04-09 03:28:39 +00:00
m_llmThread.setObjectName("llm thread");
m_llmThread.start();
}
bool LLMObject::loadModel()
2023-04-09 03:28:39 +00:00
{
const QList<QString> models = modelList();
if (models.isEmpty()) {
2023-04-19 01:10:06 +00:00
// try again when we get a list of models
connect(Download::globalInstance(), &Download::modelListChanged, this,
&LLMObject::loadModel, Qt::SingleShotConnection);
return false;
}
return loadModelPrivate(models.first());
}
bool LLMObject::loadModelPrivate(const QString &modelName)
{
if (isModelLoaded() && m_modelName == modelName)
2023-04-09 03:28:39 +00:00
return true;
if (isModelLoaded()) {
resetContext();
delete m_llmodel;
m_llmodel = nullptr;
emit isModelLoadedChanged();
}
bool isGPTJ = false;
QString filePath = modelFilePath(modelName);
QFileInfo info(filePath);
if (info.exists()) {
2023-04-09 03:28:39 +00:00
auto fin = std::ifstream(filePath.toStdString(), std::ios::binary);
uint32_t magic;
fin.read((char *) &magic, sizeof(magic));
fin.seekg(0);
isGPTJ = magic == 0x67676d6c;
if (isGPTJ) {
m_llmodel = new GPTJ;
m_llmodel->loadModel(modelName.toStdString(), fin);
} else {
m_llmodel = new LLamaModel;
m_llmodel->loadModel(filePath.toStdString());
}
2023-04-09 03:28:39 +00:00
emit isModelLoadedChanged();
2023-04-18 13:46:03 +00:00
emit threadCountChanged();
2023-04-09 03:28:39 +00:00
}
if (m_llmodel)
setModelName(info.completeBaseName().remove(0, 5)); // remove the ggml- prefix
return m_llmodel;
2023-04-09 03:28:39 +00:00
}
2023-04-18 13:46:03 +00:00
void LLMObject::setThreadCount(int32_t n_threads) {
m_llmodel->setThreadCount(n_threads);
emit threadCountChanged();
}
int32_t LLMObject::threadCount() {
2023-04-20 11:17:07 +00:00
if (!m_llmodel)
return 1;
2023-04-18 13:46:03 +00:00
return m_llmodel->threadCount();
}
bool LLMObject::isModelLoaded() const
2023-04-09 03:28:39 +00:00
{
return m_llmodel && m_llmodel->isModelLoaded();
2023-04-09 03:28:39 +00:00
}
2023-04-17 18:11:41 +00:00
void LLMObject::regenerateResponse()
2023-04-09 03:28:39 +00:00
{
s_ctx.n_past -= m_responseTokens;
s_ctx.n_past = std::max(0, s_ctx.n_past);
s_ctx.logits.erase(s_ctx.logits.end() -= m_responseLogits, s_ctx.logits.end());
m_responseTokens = 0;
m_responseLogits = 0;
2023-04-09 03:28:39 +00:00
m_response = std::string();
emit responseChanged();
2023-04-09 03:28:39 +00:00
}
2023-04-17 18:11:41 +00:00
void LLMObject::resetResponse()
{
m_responseTokens = 0;
m_responseLogits = 0;
m_response = std::string();
emit responseChanged();
}
void LLMObject::resetContext()
{
2023-04-17 18:11:41 +00:00
regenerateResponse();
s_ctx = LLModel::PromptContext();
}
std::string remove_leading_whitespace(const std::string& input) {
auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
2023-04-16 18:04:25 +00:00
return !std::isspace(c);
});
return std::string(first_non_whitespace, input.end());
}
std::string trim_whitespace(const std::string& input) {
auto first_non_whitespace = std::find_if(input.begin(), input.end(), [](unsigned char c) {
return !std::isspace(c);
});
auto last_non_whitespace = std::find_if(input.rbegin(), input.rend(), [](unsigned char c) {
return !std::isspace(c);
}).base();
return std::string(first_non_whitespace, last_non_whitespace);
}
QString LLMObject::response() const
2023-04-09 03:28:39 +00:00
{
return QString::fromStdString(remove_leading_whitespace(m_response));
2023-04-09 03:28:39 +00:00
}
QString LLMObject::modelName() const
{
return m_modelName;
}
void LLMObject::setModelName(const QString &modelName)
{
m_modelName = modelName;
emit modelNameChanged();
emit modelListChanged();
}
void LLMObject::modelNameChangeRequested(const QString &modelName)
{
if (!loadModelPrivate(modelName))
qWarning() << "ERROR: Could not load model" << modelName;
}
QList<QString> LLMObject::modelList() const
{
// Build a model list from exepath and from the localpath
QList<QString> list;
QString exePath = QCoreApplication::applicationDirPath() + QDir::separator();
QString localPath = Download::globalInstance()->downloadLocalModelsPath();
{
QDir dir(exePath);
dir.setNameFilters(QStringList() << "ggml-*.bin");
QStringList fileNames = dir.entryList();
for (QString f : fileNames) {
QString filePath = exePath + f;
QFileInfo info(filePath);
QString name = info.completeBaseName().remove(0, 5);
if (info.exists()) {
if (name == m_modelName)
list.prepend(name);
else
list.append(name);
}
}
}
if (localPath != exePath) {
QDir dir(localPath);
dir.setNameFilters(QStringList() << "ggml-*.bin");
QStringList fileNames = dir.entryList();
for (QString f : fileNames) {
QString filePath = localPath + f;
QFileInfo info(filePath);
QString name = info.completeBaseName().remove(0, 5);
if (info.exists() && !list.contains(name)) { // don't allow duplicates
if (name == m_modelName)
list.prepend(name);
else
list.append(name);
}
}
}
if (list.isEmpty()) {
if (exePath != localPath) {
qWarning() << "ERROR: Could not find any applicable models in"
<< exePath << "nor" << localPath;
} else {
qWarning() << "ERROR: Could not find any applicable models in"
<< exePath;
}
return QList<QString>();
}
return list;
}
bool LLMObject::handleResponse(const std::string &response)
2023-04-09 03:28:39 +00:00
{
#if 0
printf("%s", response.c_str());
fflush(stdout);
#endif
++m_responseTokens;
if (!response.empty()) {
m_response.append(response);
emit responseChanged();
}
2023-04-15 20:33:14 +00:00
// Stop generation if we encounter prompt or response tokens
QString r = QString::fromStdString(m_response);
if (r.contains("### Prompt:") || r.contains("### Response:"))
return false;
2023-04-09 03:28:39 +00:00
return !m_stopGenerating;
}
bool LLMObject::prompt(const QString &prompt, const QString &prompt_template, int32_t n_predict, int32_t top_k, float top_p,
float temp, int32_t n_batch)
2023-04-09 03:28:39 +00:00
{
if (!isModelLoaded())
return false;
QString instructPrompt = prompt_template.arg(prompt);
2023-04-15 20:33:14 +00:00
2023-04-09 03:28:39 +00:00
m_stopGenerating = false;
auto func = std::bind(&LLMObject::handleResponse, this, std::placeholders::_1);
2023-04-09 03:28:39 +00:00
emit responseStarted();
qint32 logitsBefore = s_ctx.logits.size();
m_llmodel->prompt(instructPrompt.toStdString(), func, s_ctx, n_predict, top_k, top_p, temp, n_batch);
m_responseLogits += s_ctx.logits.size() - logitsBefore;
std::string trimmed = trim_whitespace(m_response);
if (trimmed != m_response) {
m_response = trimmed;
emit responseChanged();
}
2023-04-09 03:28:39 +00:00
emit responseStopped();
2023-04-09 03:28:39 +00:00
return true;
}
LLM::LLM()
: QObject{nullptr}
, m_llmodel(new LLMObject)
2023-04-09 03:28:39 +00:00
, m_responseInProgress(false)
{
2023-04-19 01:10:06 +00:00
connect(Download::globalInstance(), &Download::modelListChanged, this, &LLM::modelListChanged, Qt::QueuedConnection);
connect(m_llmodel, &LLMObject::isModelLoadedChanged, this, &LLM::isModelLoadedChanged, Qt::QueuedConnection);
connect(m_llmodel, &LLMObject::responseChanged, this, &LLM::responseChanged, Qt::QueuedConnection);
connect(m_llmodel, &LLMObject::responseStarted, this, &LLM::responseStarted, Qt::QueuedConnection);
connect(m_llmodel, &LLMObject::responseStopped, this, &LLM::responseStopped, Qt::QueuedConnection);
connect(m_llmodel, &LLMObject::modelNameChanged, this, &LLM::modelNameChanged, Qt::QueuedConnection);
connect(m_llmodel, &LLMObject::modelListChanged, this, &LLM::modelListChanged, Qt::QueuedConnection);
2023-04-18 13:46:03 +00:00
connect(m_llmodel, &LLMObject::threadCountChanged, this, &LLM::threadCountChanged, Qt::QueuedConnection);
connect(this, &LLM::promptRequested, m_llmodel, &LLMObject::prompt, Qt::QueuedConnection);
connect(this, &LLM::modelNameChangeRequested, m_llmodel, &LLMObject::modelNameChangeRequested, Qt::QueuedConnection);
// The following are blocking operations and will block the gui thread, therefore must be fast
// to respond to
2023-04-17 18:11:41 +00:00
connect(this, &LLM::regenerateResponseRequested, m_llmodel, &LLMObject::regenerateResponse, Qt::BlockingQueuedConnection);
connect(this, &LLM::resetResponseRequested, m_llmodel, &LLMObject::resetResponse, Qt::BlockingQueuedConnection);
connect(this, &LLM::resetContextRequested, m_llmodel, &LLMObject::resetContext, Qt::BlockingQueuedConnection);
2023-04-18 13:46:03 +00:00
connect(this, &LLM::setThreadCountRequested, m_llmodel, &LLMObject::setThreadCount, Qt::QueuedConnection);
2023-04-09 03:28:39 +00:00
}
bool LLM::isModelLoaded() const
{
return m_llmodel->isModelLoaded();
2023-04-09 03:28:39 +00:00
}
void LLM::prompt(const QString &prompt, const QString &prompt_template, int32_t n_predict, int32_t top_k, float top_p,
float temp, int32_t n_batch)
2023-04-09 03:28:39 +00:00
{
emit promptRequested(prompt, prompt_template, n_predict, top_k, top_p, temp, n_batch);
2023-04-09 03:28:39 +00:00
}
2023-04-17 18:11:41 +00:00
void LLM::regenerateResponse()
{
emit regenerateResponseRequested(); // blocking queued connection
}
2023-04-09 03:28:39 +00:00
void LLM::resetResponse()
{
emit resetResponseRequested(); // blocking queued connection
}
void LLM::resetContext()
{
emit resetContextRequested(); // blocking queued connection
}
2023-04-09 03:28:39 +00:00
void LLM::stopGenerating()
{
m_llmodel->stopGenerating();
2023-04-09 03:28:39 +00:00
}
QString LLM::response() const
{
return m_llmodel->response();
2023-04-09 03:28:39 +00:00
}
void LLM::responseStarted()
{
m_responseInProgress = true;
emit responseInProgressChanged();
}
void LLM::responseStopped()
{
m_responseInProgress = false;
emit responseInProgressChanged();
}
2023-04-11 03:34:34 +00:00
QString LLM::modelName() const
{
return m_llmodel->modelName();
}
void LLM::setModelName(const QString &modelName)
{
// doesn't block but will unload old model and load new one which the gui can see through changes
// to the isModelLoaded property
emit modelNameChangeRequested(modelName);
}
QList<QString> LLM::modelList() const
{
return m_llmodel->modelList();
}
2023-04-18 13:46:03 +00:00
void LLM::setThreadCount(int32_t n_threads) {
emit setThreadCountRequested(n_threads);
}
int32_t LLM::threadCount() {
return m_llmodel->threadCount();
}
2023-04-11 03:34:34 +00:00
bool LLM::checkForUpdates() const
{
#if defined(Q_OS_LINUX)
2023-04-11 16:16:04 +00:00
QString tool("maintenancetool");
2023-04-11 03:34:34 +00:00
#elif defined(Q_OS_WINDOWS)
2023-04-11 16:16:04 +00:00
QString tool("maintenancetool.exe");
2023-04-11 03:34:34 +00:00
#elif defined(Q_OS_DARWIN)
2023-04-12 21:57:02 +00:00
QString tool("../../../maintenancetool.app/Contents/MacOS/maintenancetool");
2023-04-11 03:34:34 +00:00
#endif
QString fileName = QCoreApplication::applicationDirPath()
+ QDir::separator() + ".." + QDir::separator() + tool;
if (!QFileInfo::exists(fileName)) {
qDebug() << "Couldn't find tool at" << fileName << "so cannot check for updates!";
return false;
}
return QProcess::startDetached(fileName);
}