From e70a1c458b248f881959a7109d42d1e5da8fc5a4 Mon Sep 17 00:00:00 2001 From: Adam Treat Date: Fri, 19 May 2023 16:08:13 -0400 Subject: [PATCH] Add remove folder implementation. --- gpt4all-chat/localdocs.cpp | 139 ++++++++++++++++++++++++++++++++++--- 1 file changed, 128 insertions(+), 11 deletions(-) diff --git a/gpt4all-chat/localdocs.cpp b/gpt4all-chat/localdocs.cpp index 5d98a50d..fbe92c36 100644 --- a/gpt4all-chat/localdocs.cpp +++ b/gpt4all-chat/localdocs.cpp @@ -20,11 +20,11 @@ const auto INSERT_CHUNK_FTS_SQL = QLatin1String(R"( )"); const auto DELETE_CHUNKS_SQL = QLatin1String(R"( - DELETE FROM chunks WHERE document_id = ?; + delete from chunks WHERE document_id = ?; )"); const auto DELETE_CHUNKS_FTS_SQL = QLatin1String(R"( - DELETE FROM chunks_fts WHERE document_id = ?; + delete from chunks_fts WHERE document_id = ?; )"); const auto CHUNKS_SQL = QLatin1String(R"( @@ -76,7 +76,7 @@ bool addChunk(QSqlQuery &q, int document_id, int chunk_id, const QString &chunk_ return true; } -bool deleteChunksByDocumentId(QSqlQuery &q, int document_id) +bool removeChunksByDocumentId(QSqlQuery &q, int document_id) { { if (!q.prepare(DELETE_CHUNKS_SQL)) @@ -159,6 +159,10 @@ const auto INSERT_COLLECTION_SQL = QLatin1String(R"( insert into collections(collection_name, folder_id) values(?, ?); )"); +const auto DELETE_COLLECTION_SQL = QLatin1String(R"( + delete from collections where collection_name = ? and folder_id = ?; + )"); + const auto COLLECTIONS_SQL = QLatin1String(R"( create table collections(collection_name varchar, folder_id integer, unique(collection_name, folder_id)); )"); @@ -167,6 +171,10 @@ const auto SELECT_FOLDERS_FROM_COLLECTIONS_SQL = QLatin1String(R"( select folder_id from collections where collection_name = ?; )"); +const auto SELECT_COLLECTIONS_FROM_FOLDER_SQL = QLatin1String(R"( + select collection_name from collections where folder_id = ?; + )"); + bool addCollection(QSqlQuery &q, const QString &collection_name, int folder_id) { if (!q.prepare(INSERT_COLLECTION_SQL)) @@ -176,6 +184,15 @@ bool addCollection(QSqlQuery &q, const QString &collection_name, int folder_id) return q.exec(); } +bool removeCollection(QSqlQuery &q, const QString &collection_name, int folder_id) +{ + if (!q.prepare(DELETE_COLLECTION_SQL)) + return false; + q.addBindValue(collection_name); + q.addBindValue(folder_id); + return q.exec(); +} + bool selectFoldersFromCollection(QSqlQuery &q, const QString &collection_name, QList *folderIds) { if (!q.prepare(SELECT_FOLDERS_FROM_COLLECTIONS_SQL)) return false; @@ -187,12 +204,23 @@ bool selectFoldersFromCollection(QSqlQuery &q, const QString &collection_name, Q return true; } +bool selectCollectionsFromFolder(QSqlQuery &q, int folder_id, QList *collections) { + if (!q.prepare(SELECT_COLLECTIONS_FROM_FOLDER_SQL)) + return false; + q.addBindValue(folder_id); + if (!q.exec()) + return false; + while (q.next()) + collections->append(q.value(0).toString()); + return true; +} + const auto INSERT_FOLDERS_SQL = QLatin1String(R"( insert into folders(folder_path) values(?); )"); const auto DELETE_FOLDERS_SQL = QLatin1String(R"( - delete from folders where folder_path = ?; + delete from folders where id = ?; )"); const auto SELECT_FOLDERS_SQL = QLatin1String(R"( @@ -214,10 +242,10 @@ bool addFolderToDB(QSqlQuery &q, const QString &folder_path, int *folder_id) return true; } -bool removeFolderFromDB(QSqlQuery &q, const QString &folder_path) { +bool removeFolderFromDB(QSqlQuery &q, int folder_id) { if (!q.prepare(DELETE_FOLDERS_SQL)) return false; - q.addBindValue(folder_path); + q.addBindValue(folder_id); return q.exec(); } @@ -241,6 +269,10 @@ const auto UPDATE_DOCUMENT_TIME_SQL = QLatin1String(R"( update documents set document_time = ? where id = ?; )"); +const auto DELETE_DOCUMENTS_SQL = QLatin1String(R"( + delete from documents where id = ?; + )"); + const auto DOCUMENTS_SQL = QLatin1String(R"( create table documents(id integer primary key, folder_id integer, document_time integer, document_path varchar unique); )"); @@ -249,6 +281,10 @@ const auto SELECT_DOCUMENT_SQL = QLatin1String(R"( select id, document_time from documents where document_path = ?; )"); +const auto SELECT_DOCUMENTS_SQL = QLatin1String(R"( + select id from documents where folder_id = ?; + )"); + bool addDocument(QSqlQuery &q, int folder_id, qint64 document_time, const QString &document_path, int *document_id) { if (!q.prepare(INSERT_DOCUMENTS_SQL)) @@ -262,6 +298,13 @@ bool addDocument(QSqlQuery &q, int folder_id, qint64 document_time, const QStrin return true; } +bool removeDocument(QSqlQuery &q, int document_id) { + if (!q.prepare(DELETE_DOCUMENTS_SQL)) + return false; + q.addBindValue(document_id); + return q.exec(); +} + bool updateDocument(QSqlQuery &q, int id, qint64 document_time) { if (!q.prepare(UPDATE_DOCUMENT_TIME_SQL)) @@ -285,6 +328,17 @@ bool selectDocument(QSqlQuery &q, const QString &document_path, int *id, qint64 return true; } +bool selectDocuments(QSqlQuery &q, int folder_id, QList *documentIds) { + if (!q.prepare(SELECT_DOCUMENTS_SQL)) + return false; + q.addBindValue(folder_id); + if (!q.exec()) + return false; + while (q.next()) + documentIds->append(q.value(0).toInt()); + return true; +} + QSqlError initDb() { QString dbPath = Download::globalInstance()->downloadLocalModelsPath() @@ -439,8 +493,8 @@ void Database::scanQueue() if (!m_docsToScan.isEmpty()) QTimer::singleShot(0, this, &Database::scanQueue); return; } else { - if (!deleteChunksByDocumentId(q, existing_id)) { - return handleDocumentErrorAndScheduleNext("ERROR: Cannot delete chunks of document", + if (!removeChunksByDocumentId(q, existing_id)) { + return handleDocumentErrorAndScheduleNext("ERROR: Cannot remove chunks of document", existing_id, document_path, q.lastError()); } } @@ -592,11 +646,74 @@ void Database::removeFolder(const QString &collection, const QString &path) qDebug() << "removeFolder" << path; #endif - // FIXME: Determine if the folder is used by more than one collection + QSqlQuery q; + int folder_id = -1; - // FIXME: If not, then delete all chunks and documents associated with it + // See if the folder exists in the db + if (!selectFolder(q, path, &folder_id)) { + qWarning() << "ERROR: Cannot select folder from path" << path << q.lastError(); + return; + } - // FIXME: Remove it from the collections + // If we don't have a folder_id in the db, then something bad has happened + Q_ASSERT(folder_id != -1); + if (folder_id == -1) { + qWarning() << "ERROR: Collected folder does not exist in db" << path; + m_watcher->removePath(path); + return; + } + + // Determine if the folder is used by more than one collection + QList collections; + if (!selectCollectionsFromFolder(q, folder_id, &collections)) { + qWarning() << "ERROR: Cannot select collections from folder" << folder_id << q.lastError(); + return; + } + + // Remove it from the collections + if (!removeCollection(q, collection, folder_id)) { + qWarning() << "ERROR: Cannot remove collection" << collection << folder_id << q.lastError(); + return; + } + + // If the folder is associated with more than one collection, then return + if (collections.count() > 1) + return; + + // First remove all upcoming jobs associated with this folder by performing an opt-in filter + QQueue docsToScan; + for (DocumentInfo info : m_docsToScan) { + if (info.folder == folder_id) + continue; + docsToScan.append(info); + } + m_docsToScan = docsToScan; + emit docsToScanChanged(); + + // Get a list of all documents associated with folder + QList documentIds; + if (!selectDocuments(q, folder_id, &documentIds)) { + qWarning() << "ERROR: Cannot select documents" << folder_id << q.lastError(); + return; + } + + // Remove all chunks and documents associated with this folder + for (int document_id : documentIds) { + if (!removeChunksByDocumentId(q, document_id)) { + qWarning() << "ERROR: Cannot remove chunks of document_id" << document_id << q.lastError(); + return; + } + + if (!removeDocument(q, document_id)) { + qWarning() << "ERROR: Cannot remove document_id" << document_id << q.lastError(); + return; + } + } + + if (!removeFolderFromDB(q, folder_id)) { + qWarning() << "ERROR: Cannot remove folder_id" << folder_id << q.lastError(); + return; + } removeFolderFromWatch(path); }