gpt4all/gpt4all-chat/chatllm.h

174 lines
5.1 KiB
C
Raw Normal View History

#ifndef CHATLLM_H
#define CHATLLM_H
#include <QObject>
#include <QThread>
#include <QFileInfo>
2023-06-01 18:13:12 +00:00
#include "localdocs.h"
2023-06-22 19:44:49 +00:00
#include "modellist.h"
#include "../gpt4all-backend/llmodel.h"
enum LLModelType {
MPT_,
GPTJ_,
LLAMA_,
CHATGPT_,
2023-07-06 20:34:04 +00:00
BERT_,
};
struct LLModelInfo {
LLModel *model = nullptr;
QFileInfo fileInfo;
// NOTE: This does not store the model type or name on purpose as this is left for ChatLLM which
// must be able to serialize the information even if it is in the unloaded state
};
class TokenTimer : public QObject {
Q_OBJECT
public:
explicit TokenTimer(QObject *parent)
: QObject(parent)
, m_elapsed(0) {}
static int rollingAverage(int oldAvg, int newNumber, int n)
{
// i.e. to calculate the new average after then nth number,
// you multiply the old average by n1, add the new number, and divide the total by n.
return qRound(((float(oldAvg) * (n - 1)) + newNumber) / float(n));
}
void start() { m_tokens = 0; m_elapsed = 0; m_time.invalidate(); }
void stop() { handleTimeout(); }
void inc() {
if (!m_time.isValid())
m_time.start();
++m_tokens;
if (m_time.elapsed() > 999)
handleTimeout();
}
Q_SIGNALS:
void report(const QString &speed);
private Q_SLOTS:
void handleTimeout()
{
m_elapsed += m_time.restart();
emit report(QString("%1 tokens/sec").arg(m_tokens / float(m_elapsed / 1000.0f), 0, 'g', 2));
}
private:
QElapsedTimer m_time;
qint64 m_elapsed;
quint32 m_tokens;
};
class Chat;
class ChatLLM : public QObject
{
Q_OBJECT
Q_PROPERTY(bool isRecalc READ isRecalc NOTIFY recalcChanged)
public:
ChatLLM(Chat *parent, bool isServer = false);
2023-05-12 18:06:03 +00:00
virtual ~ChatLLM();
bool isModelLoaded() const;
void regenerateResponse();
void resetResponse();
void resetContext();
void stopGenerating() { m_stopGenerating = true; }
bool shouldBeLoaded() const { return m_shouldBeLoaded; }
void setShouldBeLoaded(bool b);
QString response() const;
2023-06-22 19:44:49 +00:00
ModelInfo modelInfo() const;
void setModelInfo(const ModelInfo &info);
bool isRecalc() const { return m_isRecalc; }
2023-05-02 15:19:17 +00:00
QString generatedName() const { return QString::fromStdString(m_nameResponse); }
2023-05-08 09:52:57 +00:00
bool serialize(QDataStream &stream, int version);
bool deserialize(QDataStream &stream, int version);
public Q_SLOTS:
bool prompt(const QList<QString> &collectionList, const QString &prompt);
bool loadDefaultModel();
2023-06-22 19:44:49 +00:00
bool loadModel(const ModelInfo &modelInfo);
void modelChangeRequested(const ModelInfo &modelInfo);
void forceUnloadModel();
void unloadModel();
void reloadModel();
2023-05-02 15:19:17 +00:00
void generateName();
void handleChatIdChanged(const QString &id);
void handleShouldBeLoadedChanged();
void handleThreadStarted();
2023-06-27 15:54:34 +00:00
void handleForceMetalChanged(bool forceMetal);
2023-09-13 14:32:08 +00:00
void handleDeviceChanged();
void processSystemPrompt();
Q_SIGNALS:
2023-06-22 19:44:49 +00:00
void recalcChanged();
void isModelLoadedChanged(bool);
void modelLoadingError(const QString &error);
void responseChanged(const QString &response);
void promptProcessing();
void responseStopped();
void sendStartup();
void sendModelLoaded();
void generatedNameChanged(const QString &name);
void stateChanged();
2023-05-11 20:46:25 +00:00
void threadStarted();
void shouldBeLoadedChanged();
2023-06-01 18:13:12 +00:00
void requestRetrieveFromDB(const QList<QString> &collections, const QString &text, int retrievalSize, QList<ResultInfo> *results);
void reportSpeed(const QString &speed);
2023-09-14 12:25:37 +00:00
void reportDevice(const QString &device);
void reportFallbackReason(const QString &fallbackReason);
2023-06-19 22:23:54 +00:00
void databaseResultsChanged(const QList<ResultInfo>&);
2023-06-22 19:44:49 +00:00
void modelInfoChanged(const ModelInfo &modelInfo);
2023-05-11 20:46:25 +00:00
protected:
bool promptInternal(const QList<QString> &collectionList, const QString &prompt, const QString &promptTemplate,
int32_t n_predict, int32_t top_k, float top_p, float temp, int32_t n_batch, float repeat_penalty,
int32_t repeat_penalty_tokens);
bool handlePrompt(int32_t token);
bool handleResponse(int32_t token, const std::string &response);
bool handleRecalculate(bool isRecalc);
2023-05-02 15:19:17 +00:00
bool handleNamePrompt(int32_t token);
bool handleNameResponse(int32_t token, const std::string &response);
bool handleNameRecalculate(bool isRecalc);
bool handleSystemPrompt(int32_t token);
bool handleSystemResponse(int32_t token, const std::string &response);
bool handleSystemRecalculate(bool isRecalc);
void saveState();
void restoreState();
protected:
LLModel::PromptContext m_ctx;
quint32 m_promptTokens;
quint32 m_promptResponseTokens;
private:
std::string m_response;
2023-05-02 15:19:17 +00:00
std::string m_nameResponse;
2023-06-22 19:44:49 +00:00
LLModelInfo m_llModelInfo;
LLModelType m_llModelType;
ModelInfo m_modelInfo;
TokenTimer *m_timer;
QByteArray m_state;
QThread m_llmThread;
std::atomic<bool> m_stopGenerating;
std::atomic<bool> m_shouldBeLoaded;
2023-06-19 22:24:11 +00:00
std::atomic<bool> m_isRecalc;
bool m_isServer;
2023-06-27 15:54:34 +00:00
bool m_forceMetal;
bool m_reloadingToChangeVariant;
bool m_processedSystemPrompt;
};
#endif // CHATLLM_H