From 045f6e6cdc6dedba996d5f3c630baeb18b3739ab Mon Sep 17 00:00:00 2001
From: Adam Treat <treat.adam@gmail.com>
Date: Fri, 15 Sep 2023 14:45:25 -0400
Subject: [PATCH] Link against ggml in bin so we can get the available devices
 without loading a model.

---
 gpt4all-backend/CMakeLists.txt     |  2 ++
 gpt4all-backend/llmodel.h          |  1 +
 gpt4all-backend/llmodel_shared.cpp | 27 +++++++++++++++++++++++++++
 gpt4all-chat/chatllm.cpp           |  6 ------
 gpt4all-chat/mysettings.cpp        |  8 ++++++++
 5 files changed, 38 insertions(+), 6 deletions(-)

diff --git a/gpt4all-backend/CMakeLists.txt b/gpt4all-backend/CMakeLists.txt
index 840bce67..762af492 100644
--- a/gpt4all-backend/CMakeLists.txt
+++ b/gpt4all-backend/CMakeLists.txt
@@ -134,6 +134,8 @@ add_library(llmodel
     llmodel_c.h llmodel_c.cpp
     dlhandle.h
 )
+target_link_libraries(llmodel PRIVATE ggml-mainline-default)
+target_compile_definitions(llmodel PRIVATE GGML_BUILD_VARIANT="default")
 target_compile_definitions(llmodel PRIVATE LIB_FILE_EXT="${CMAKE_SHARED_LIBRARY_SUFFIX}")
 
 set_target_properties(llmodel PROPERTIES
diff --git a/gpt4all-backend/llmodel.h b/gpt4all-backend/llmodel.h
index c90edd30..70d3b0b1 100644
--- a/gpt4all-backend/llmodel.h
+++ b/gpt4all-backend/llmodel.h
@@ -101,6 +101,7 @@ public:
     virtual bool initializeGPUDevice(int /*device*/) { return false; }
     virtual bool hasGPUDevice() { return false; }
     virtual bool usingGPUDevice() { return false; }
+    static std::vector<GPUDevice> availableGPUDevices();
 
 protected:
     // These are pure virtual because subclasses need to implement as the default implementation of
diff --git a/gpt4all-backend/llmodel_shared.cpp b/gpt4all-backend/llmodel_shared.cpp
index 89ba32b5..f3022f7e 100644
--- a/gpt4all-backend/llmodel_shared.cpp
+++ b/gpt4all-backend/llmodel_shared.cpp
@@ -4,6 +4,10 @@
 #include <iostream>
 #include <unordered_set>
 
+#ifdef GGML_USE_KOMPUTE
+#include "ggml-vulkan.h"
+#endif
+
 void LLModel::recalculateContext(PromptContext &promptCtx, std::function<bool(bool)> recalculate) {
     size_t i = 0;
     promptCtx.n_past = 0;
@@ -174,3 +178,26 @@ std::vector<float> LLModel::embedding(const std::string &/*text*/)
     }
     return std::vector<float>();
 }
+
+std::vector<LLModel::GPUDevice> LLModel::availableGPUDevices()
+{
+#if defined(GGML_USE_KOMPUTE)
+    std::vector<ggml_vk_device> vkDevices = ggml_vk_available_devices(0);
+
+    std::vector<LLModel::GPUDevice> devices;
+    for(const auto& vkDevice : vkDevices) {
+        LLModel::GPUDevice device;
+        device.index = vkDevice.index;
+        device.type = vkDevice.type;
+        device.heapSize = vkDevice.heapSize;
+        device.name = vkDevice.name;
+        device.vendor = vkDevice.vendor;
+
+        devices.push_back(device);
+    }
+
+    return devices;
+#else
+    return std::vector<LLModel::GPUDevice>();
+#endif
+}
diff --git a/gpt4all-chat/chatllm.cpp b/gpt4all-chat/chatllm.cpp
index 74208c17..ae6359c0 100644
--- a/gpt4all-chat/chatllm.cpp
+++ b/gpt4all-chat/chatllm.cpp
@@ -263,12 +263,6 @@ bool ChatLLM::loadModel(const ModelInfo &modelInfo)
             if (m_llModelInfo.model) {
                 // Update the settings that a model is being loaded and update the device list
                 MySettings::globalInstance()->setAttemptModelLoad(filePath);
-                std::vector<LLModel::GPUDevice> devices = m_llModelInfo.model->availableGPUDevices(0);
-                QVector<QString> deviceList{ "Auto" };
-                for (LLModel::GPUDevice &d : devices)
-                    deviceList << QString::fromStdString(d.name);
-                deviceList << "CPU";
-                MySettings::globalInstance()->setDeviceList(deviceList);
 
                 // Pick the best match for the device
                 QString actualDevice = m_llModelInfo.model->implementation().buildVariant() == "metal" ? "Metal" : "CPU";
diff --git a/gpt4all-chat/mysettings.cpp b/gpt4all-chat/mysettings.cpp
index 8db8c6eb..b1613786 100644
--- a/gpt4all-chat/mysettings.cpp
+++ b/gpt4all-chat/mysettings.cpp
@@ -1,5 +1,6 @@
 #include "mysettings.h"
 #include "modellist.h"
+#include "../gpt4all-backend/llmodel.h"
 
 #include <QDir>
 #include <QFile>
@@ -63,6 +64,13 @@ MySettings::MySettings()
     : QObject{nullptr}
 {
     QSettings::setDefaultFormat(QSettings::IniFormat);
+
+    std::vector<LLModel::GPUDevice> devices = LLModel::availableGPUDevices();
+    QVector<QString> deviceList{ "Auto" };
+    for (LLModel::GPUDevice &d : devices)
+        deviceList << QString::fromStdString(d.name);
+    deviceList << "CPU";
+    setDeviceList(deviceList);
 }
 
 Q_INVOKABLE QVector<QString> MySettings::deviceList() const