From 045069c3e6144a63426494f6d6bfdaf7530c995b Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sun, 10 Jan 2021 22:25:14 +0100 Subject: [PATCH 01/10] Implement proper GXS message deletion notification GxsForums propagate message deletion notification as RsEvent --- libretroshare/src/gxs/rsgenexchange.cc | 2 +- libretroshare/src/gxs/rsgxsnotify.h | 10 + libretroshare/src/retroshare/rsgxsforums.h | 12 +- libretroshare/src/services/p3gxsforums.cc | 381 ++++++++++----------- retroshare-gui/src/gui/NewsFeed.cpp | 5 +- 5 files changed, 202 insertions(+), 208 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 5ddb0be26..9610fc1a6 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -4,7 +4,7 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012 Christopher Evi-Parker * - * Copyright (C) 2019 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * diff --git a/libretroshare/src/gxs/rsgxsnotify.h b/libretroshare/src/gxs/rsgxsnotify.h index 990acfe84..6f186e1fe 100644 --- a/libretroshare/src/gxs/rsgxsnotify.h +++ b/libretroshare/src/gxs/rsgxsnotify.h @@ -97,3 +97,13 @@ private: bool mMetaChange; }; +struct RsGxsBulkMsgDeletedChange : RsGxsNotify +{ + RsGxsBulkMsgDeletedChange( + const RsGxsGroupId& gid, const std::set& msgsId): + RsGxsNotify(gid), messagesId(msgsId) {} + + NotifyType getType() override { return TYPE_MESSAGE_DELETED; } + + const std::set messagesId; +}; diff --git a/libretroshare/src/retroshare/rsgxsforums.h b/libretroshare/src/retroshare/rsgxsforums.h index 83a961fc3..182a8177d 100644 --- a/libretroshare/src/retroshare/rsgxsforums.h +++ b/libretroshare/src/retroshare/rsgxsforums.h @@ -115,9 +115,10 @@ enum class RsForumEventCode: uint8_t READ_STATUS_CHANGED = 0x06, /// msg was read or marked unread STATISTICS_CHANGED = 0x07, /// suppliers and how many messages they have changed MODERATOR_LIST_CHANGED = 0x08, /// forum moderation list has changed. - SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed - PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned - DELETED_FORUM = 0x0c, /// forum was deleted by cleaning + SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed + PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned + DELETED_FORUM = 0x0c, /// forum was deleted by cleaning + DELETED_POSTS = 13 /// Posts deleted by cleaning }; struct RsGxsForumEvent: RsEvent @@ -128,7 +129,7 @@ struct RsGxsForumEvent: RsEvent RsForumEventCode mForumEventCode; RsGxsGroupId mForumGroupId; - RsGxsMessageId mForumMsgId; + std::set mForumMsgsId; std::list mModeratorsAdded; std::list mModeratorsRemoved; @@ -140,8 +141,7 @@ struct RsGxsForumEvent: RsEvent RsEvent::serial_process(j, ctx); RS_SERIAL_PROCESS(mForumEventCode); RS_SERIAL_PROCESS(mForumGroupId); - RS_SERIAL_PROCESS(mForumMsgId); - RS_SERIAL_PROCESS(mForumMsgId); + RS_SERIAL_PROCESS(mForumMsgsId); RS_SERIAL_PROCESS(mModeratorsAdded); RS_SERIAL_PROCESS(mModeratorsRemoved); } diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index a301997e6..b8d2194c5 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -192,221 +192,202 @@ RsSerialiser* p3GxsForums::setupSerialiser() void p3GxsForums::notifyChanges(std::vector &changes) { -#ifdef GXSFORUMS_DEBUG - std::cerr << "p3GxsForums::notifyChanges() : " << changes.size() << "changes to notify" << std::endl; -#endif + RS_DBG2(changes.size(), " changes to notify"); std::vector::iterator it; for(it = changes.begin(); it != changes.end(); ++it) { - RsGxsMsgChange *msgChange = dynamic_cast(*it); - - if (msgChange) + rs_view_ptr gxsChange = *it; + switch(gxsChange->getType()) { - if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW || msgChange->getType() == RsGxsNotify::TYPE_PUBLISHED) /* message received */ - if (rsEvents) + case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]] + case RsGxsNotify::TYPE_PUBLISHED: + { + rs_view_ptr msgChange = + dynamic_cast(*it); + rs_view_ptr groupChange = + dynamic_cast(*it); + + if(msgChange) /* Message received*/ + { + auto ev = std::make_shared(); + ev->mForumMsgsId.insert(msgChange->mMsgId); + ev->mForumGroupId = msgChange->mGroupId; + ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE; + rsEvents->postEvent(ev); + } + else if(groupChange) /* Group received */ + { + bool unknown; + { + RS_STACK_MUTEX(mKnownForumsMutex); + unknown = ( mKnownForums.find(gxsChange->mGroupId) + == mKnownForums.end() ); + mKnownForums[gxsChange->mGroupId] = time(nullptr); + IndicateConfigChanged(); + } + + if(unknown) { auto ev = std::make_shared(); - ev->mForumMsgId = msgChange->mMsgId; - ev->mForumGroupId = msgChange->mGroupId; - ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE; + ev->mForumGroupId = gxsChange->mGroupId; + ev->mForumEventCode = RsForumEventCode::NEW_FORUM; rsEvents->postEvent(ev); } - -#ifdef NOT_USED_YET - if (!msgChange->metaChange()) - { -#ifdef GXSCHANNELS_DEBUG - std::cerr << "p3GxsForums::notifyChanges() Found Message Change Notification"; - std::cerr << std::endl; -#endif - - std::map > &msgChangeMap = msgChange->msgChangeMap; - for(auto mit = msgChangeMap.begin(); mit != msgChangeMap.end(); ++mit) - { -#ifdef GXSCHANNELS_DEBUG - std::cerr << "p3GxsForums::notifyChanges() Msgs for Group: " << mit->first; - std::cerr << std::endl; -#endif - bool enabled = false; - if (autoDownloadEnabled(mit->first, enabled) && enabled) - { -#ifdef GXSCHANNELS_DEBUG - std::cerr << "p3GxsChannels::notifyChanges() AutoDownload for Group: " << mit->first; - std::cerr << std::endl; -#endif - - /* problem is most of these will be comments and votes, - * should make it occasional - every 5mins / 10minutes TODO */ - unprocessedGroups.push_back(mit->first); - } - } + else + RS_DBG1( " Not notifying already known forum ", + gxsChange->mGroupId ); } -#endif + break; } - else + case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed { - if (rsEvents) + auto ev = std::make_shared(); + ev->mForumGroupId = gxsChange->mGroupId; + ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED; + rsEvents->postEvent(ev); + break; + } + case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED: + { + auto ev = std::make_shared(); + ev->mForumGroupId = gxsChange->mGroupId; + ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED; + rsEvents->postEvent(ev); + break; + } + case RsGxsNotify::TYPE_MESSAGE_DELETED: + { + rs_view_ptr delChange = + dynamic_cast(gxsChange); + + if(!delChange) { - RsGxsGroupChange *grpChange = dynamic_cast(*it); - if (grpChange) - { - switch (grpChange->getType()) - { - case RsGxsNotify::TYPE_PROCESSED: // happens when the group is subscribed - { - auto ev = std::make_shared(); - ev->mForumGroupId = grpChange->mGroupId; - ev->mForumEventCode = RsForumEventCode::SUBSCRIBE_STATUS_CHANGED; - rsEvents->postEvent(ev); - } - break; - - case RsGxsNotify::TYPE_GROUP_SYNC_PARAMETERS_UPDATED: - { - auto ev = std::make_shared(); - ev->mForumGroupId = grpChange->mGroupId; - ev->mForumEventCode = RsForumEventCode::SYNC_PARAMETERS_UPDATED; - rsEvents->postEvent(ev); - } - break; - - case RsGxsNotify::TYPE_PUBLISHED: - case RsGxsNotify::TYPE_RECEIVED_NEW: - { - /* group received */ - - bool unknown; - { - RS_STACK_MUTEX(mKnownForumsMutex); - unknown = (mKnownForums.find(grpChange->mGroupId)==mKnownForums.end()); - mKnownForums[grpChange->mGroupId] = time(nullptr); - IndicateConfigChanged(); - } - - if(unknown) - { - auto ev = std::make_shared(); - ev->mForumGroupId = grpChange->mGroupId; - ev->mForumEventCode = RsForumEventCode::NEW_FORUM; - rsEvents->postEvent(ev); - } - else - RsInfo() << __PRETTY_FUNCTION__ - << " Not notifying already known forum " - << grpChange->mGroupId << std::endl; - } - break; - - case RsGxsNotify::TYPE_GROUP_DELETED: - { - auto ev = std::make_shared(); - ev->mForumGroupId = grpChange->mGroupId; - ev->mForumEventCode = RsForumEventCode::DELETED_FORUM; - rsEvents->postEvent(ev); - } - break; - - case RsGxsNotify::TYPE_STATISTICS_CHANGED: - { - auto ev = std::make_shared(); - ev->mForumGroupId = grpChange->mGroupId; - ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED; - rsEvents->postEvent(ev); - - RS_STACK_MUTEX(mKnownForumsMutex); - mKnownForums[grpChange->mGroupId] = time(nullptr); - IndicateConfigChanged(); - } - break; - - case RsGxsNotify::TYPE_UPDATED: - { - // Happens when the group data has changed. In this case we need to analyse the old and new group in order to detect possible notifications for clients - - RsGxsForumGroupItem *old_forum_grp_item = dynamic_cast(grpChange->mOldGroupItem); - RsGxsForumGroupItem *new_forum_grp_item = dynamic_cast(grpChange->mNewGroupItem); - - if(old_forum_grp_item == nullptr || new_forum_grp_item == nullptr) - { - RsErr() << __PRETTY_FUNCTION__ << " received GxsGroupUpdate item with mOldGroup and mNewGroup not of type RsGxsForumGroupItem or NULL. This is inconsistent!" << std::endl; - delete grpChange; - continue; - } - - // First of all, we check if there is a difference between the old and new list of moderators - - std::list added_mods, removed_mods; - - for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids) - if(old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == old_forum_grp_item->mGroup.mAdminList.ids.end()) - added_mods.push_back(gxs_id); - - for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids) - if(new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) == new_forum_grp_item->mGroup.mAdminList.ids.end()) - removed_mods.push_back(gxs_id); - - if(!added_mods.empty() || !removed_mods.empty()) - { - auto ev = std::make_shared(); - - ev->mForumGroupId = new_forum_grp_item->meta.mGroupId; - ev->mModeratorsAdded = added_mods; - ev->mModeratorsRemoved = removed_mods; - ev->mForumEventCode = RsForumEventCode::MODERATOR_LIST_CHANGED; - - rsEvents->postEvent(ev); - } - - // check the list of pinned posts - - std::list added_pins, removed_pins; - - for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids) - if(old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == old_forum_grp_item->mGroup.mPinnedPosts.ids.end()) - added_pins.push_back(msg_id); - - for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids) - if(new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) == new_forum_grp_item->mGroup.mPinnedPosts.ids.end()) - removed_pins.push_back(msg_id); - - if(!added_pins.empty() || !removed_pins.empty()) - { - auto ev = std::make_shared(); - - ev->mForumGroupId = new_forum_grp_item->meta.mGroupId; - ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED; - - rsEvents->postEvent(ev); - } - - if( old_forum_grp_item->mGroup.mDescription != new_forum_grp_item->mGroup.mDescription - || old_forum_grp_item->meta.mGroupName != new_forum_grp_item->meta.mGroupName - || old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags - || old_forum_grp_item->meta.mAuthorId != new_forum_grp_item->meta.mAuthorId - || old_forum_grp_item->meta.mCircleId != new_forum_grp_item->meta.mCircleId - ) - { - auto ev = std::make_shared(); - ev->mForumGroupId = new_forum_grp_item->meta.mGroupId; - ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM; - rsEvents->postEvent(ev); - } - } - break; - - - default: - RsErr() << " Got a GXS event of type " << grpChange->getType() << " Currently not handled." << std::endl; - break; - } - } + RS_ERR( "Got mismatching notification type: ", + gxsChange->getType() ); + print_stacktrace(); + goto cleanup; } + + auto ev = std::make_shared(); + ev->mForumEventCode = RsForumEventCode::DELETED_POSTS; + ev->mForumGroupId = delChange->mGroupId; + ev->mForumMsgsId = delChange->messagesId; + break; + } + case RsGxsNotify::TYPE_GROUP_DELETED: + { + auto ev = std::make_shared(); + ev->mForumGroupId = gxsChange->mGroupId; + ev->mForumEventCode = RsForumEventCode::DELETED_FORUM; + rsEvents->postEvent(ev); + break; + } + case RsGxsNotify::TYPE_STATISTICS_CHANGED: + { + auto ev = std::make_shared(); + ev->mForumGroupId = gxsChange->mGroupId; + ev->mForumEventCode = RsForumEventCode::STATISTICS_CHANGED; + rsEvents->postEvent(ev); + + RS_STACK_MUTEX(mKnownForumsMutex); + mKnownForums[gxsChange->mGroupId] = time(nullptr); + IndicateConfigChanged(); + break; + } + case RsGxsNotify::TYPE_UPDATED: + { + /* Happens when the group data has changed. In this case we need to + * analyse the old and new group in order to detect possible + * notifications for clients */ + + rs_view_ptr grpChange = + dynamic_cast(*it); + + RsGxsForumGroupItem* old_forum_grp_item = + dynamic_cast(grpChange->mOldGroupItem); + RsGxsForumGroupItem* new_forum_grp_item = + dynamic_cast(grpChange->mNewGroupItem); + + if( old_forum_grp_item == nullptr || new_forum_grp_item == nullptr) + { + RS_ERR( "received GxsGroupUpdate item with mOldGroup and " + "mNewGroup not of type RsGxsForumGroupItem or NULL. " + "This is inconsistent!"); + print_stacktrace(); + goto cleanup; + } + + /* First of all, we check if there is a difference between the old + * and new list of moderators */ + + std::list added_mods, removed_mods; + for(auto& gxs_id: new_forum_grp_item->mGroup.mAdminList.ids) + if( old_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) + == old_forum_grp_item->mGroup.mAdminList.ids.end() ) + added_mods.push_back(gxs_id); + + for(auto& gxs_id: old_forum_grp_item->mGroup.mAdminList.ids) + if( new_forum_grp_item->mGroup.mAdminList.ids.find(gxs_id) + == new_forum_grp_item->mGroup.mAdminList.ids.end() ) + removed_mods.push_back(gxs_id); + + if(!added_mods.empty() || !removed_mods.empty()) + { + auto ev = std::make_shared(); + + ev->mForumGroupId = new_forum_grp_item->meta.mGroupId; + ev->mModeratorsAdded = added_mods; + ev->mModeratorsRemoved = removed_mods; + ev->mForumEventCode = RsForumEventCode::MODERATOR_LIST_CHANGED; + + rsEvents->postEvent(ev); + } + + // check the list of pinned posts + std::list added_pins, removed_pins; + + for(auto& msg_id: new_forum_grp_item->mGroup.mPinnedPosts.ids) + if( old_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) + == old_forum_grp_item->mGroup.mPinnedPosts.ids.end() ) + added_pins.push_back(msg_id); + + for(auto& msg_id: old_forum_grp_item->mGroup.mPinnedPosts.ids) + if( new_forum_grp_item->mGroup.mPinnedPosts.ids.find(msg_id) + == new_forum_grp_item->mGroup.mPinnedPosts.ids.end() ) + removed_pins.push_back(msg_id); + + if(!added_pins.empty() || !removed_pins.empty()) + { + auto ev = std::make_shared(); + ev->mForumGroupId = new_forum_grp_item->meta.mGroupId; + ev->mForumEventCode = RsForumEventCode::PINNED_POSTS_CHANGED; + rsEvents->postEvent(ev); + } + + if( old_forum_grp_item->mGroup.mDescription != new_forum_grp_item->mGroup.mDescription + || old_forum_grp_item->meta.mGroupName != new_forum_grp_item->meta.mGroupName + || old_forum_grp_item->meta.mGroupFlags != new_forum_grp_item->meta.mGroupFlags + || old_forum_grp_item->meta.mAuthorId != new_forum_grp_item->meta.mAuthorId + || old_forum_grp_item->meta.mCircleId != new_forum_grp_item->meta.mCircleId ) + { + auto ev = std::make_shared(); + ev->mForumGroupId = new_forum_grp_item->meta.mGroupId; + ev->mForumEventCode = RsForumEventCode::UPDATED_FORUM; + rsEvents->postEvent(ev); + } + + break; } - /* shouldn't need to worry about groups - as they need to be subscribed to */ + default: + RS_ERR( "Got a GXS event of type ", gxsChange->getType(), + " Currently not handled." ); + break; + } - delete *it; +cleanup: + delete *it; } } @@ -1072,7 +1053,7 @@ void p3GxsForums::setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& { auto ev = std::make_shared(); - ev->mForumMsgId = msgId.second; + ev->mForumMsgsId.insert(msgId.second); ev->mForumGroupId = msgId.first; ev->mForumEventCode = RsForumEventCode::READ_STATUS_CHANGED; rsEvents->postEvent(ev); @@ -1103,7 +1084,7 @@ std::error_condition p3GxsForums::setPostKeepForever( { auto ev = std::make_shared(); ev->mForumGroupId = forumId; - ev->mForumMsgId = postId; + ev->mForumMsgsId.insert(postId); ev->mForumEventCode = RsForumEventCode::UPDATED_MESSAGE; rsEvents->postEvent(ev); return std::error_condition(); diff --git a/retroshare-gui/src/gui/NewsFeed.cpp b/retroshare-gui/src/gui/NewsFeed.cpp index afef19b16..3c86de318 100644 --- a/retroshare-gui/src/gui/NewsFeed.cpp +++ b/retroshare-gui/src/gui/NewsFeed.cpp @@ -270,7 +270,10 @@ void NewsFeed::handleForumEvent(std::shared_ptr event) case RsForumEventCode::UPDATED_MESSAGE: case RsForumEventCode::NEW_MESSAGE: - addFeedItem(new GxsForumMsgItem(this, NEWSFEED_NEW_FORUM, pe->mForumGroupId, pe->mForumMsgId, false, true)); + for(const auto& postId: pe->mForumMsgsId) + addFeedItem(new GxsForumMsgItem( + this, NEWSFEED_NEW_FORUM, pe->mForumGroupId, postId, + false, true )); break; default: break; From cef43fe04830cf7e3ba83d4fabe09c8b51221839 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 13 Jan 2021 17:24:03 +0100 Subject: [PATCH 02/10] Notify one deletion per event as Cyril suggested --- libretroshare/src/gxs/rsgenexchange.cc | 13 +++++++++---- libretroshare/src/gxs/rsgxsnotify.h | 10 +++++----- libretroshare/src/retroshare/rsgxsforums.h | 10 +++++----- libretroshare/src/services/p3gxsforums.cc | 12 ++++++------ retroshare-gui/src/gui/NewsFeed.cpp | 4 ++-- 5 files changed, 27 insertions(+), 22 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 9610fc1a6..99725cd70 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -2717,10 +2717,15 @@ void RsGenExchange::processMessageDelete() msgDeleted.push_back(note.msgIds); } - for(const auto& msgreq:msgDeleted) - for(const auto& msgit:msgreq) - for(const auto& msg:msgit.second) - mNotifications.push_back(new RsGxsMsgChange(RsGxsNotify::TYPE_MESSAGE_DELETED,msgit.first,msg, false)); + /* Three nested for looks like a performance bomb, but as Cyril says here + * https://github.com/RetroShare/RetroShare/pull/2218#pullrequestreview-565194022 + * this should actually not explode at all because it is just one message at + * time that get notified */ + for(const auto& msd : mMsgDeletePublish) + for(auto& msgMap : msd.mMsgs) + for(auto& msgId : msgMap.second) + mNotifications.push_back( + new RsGxsMsgDeletedChange(msgMap.first, msgId) ); mMsgDeletePublish.clear(); } diff --git a/libretroshare/src/gxs/rsgxsnotify.h b/libretroshare/src/gxs/rsgxsnotify.h index 6f186e1fe..030d9f26f 100644 --- a/libretroshare/src/gxs/rsgxsnotify.h +++ b/libretroshare/src/gxs/rsgxsnotify.h @@ -97,13 +97,13 @@ private: bool mMetaChange; }; -struct RsGxsBulkMsgDeletedChange : RsGxsNotify +struct RsGxsMsgDeletedChange : RsGxsNotify { - RsGxsBulkMsgDeletedChange( - const RsGxsGroupId& gid, const std::set& msgsId): - RsGxsNotify(gid), messagesId(msgsId) {} + RsGxsMsgDeletedChange( + const RsGxsGroupId& gid, const RsGxsMessageId& msgId): + RsGxsNotify(gid), messageId(msgId) {} NotifyType getType() override { return TYPE_MESSAGE_DELETED; } - const std::set messagesId; + const RsGxsMessageId messageId; }; diff --git a/libretroshare/src/retroshare/rsgxsforums.h b/libretroshare/src/retroshare/rsgxsforums.h index 182a8177d..1b35f2db9 100644 --- a/libretroshare/src/retroshare/rsgxsforums.h +++ b/libretroshare/src/retroshare/rsgxsforums.h @@ -118,7 +118,7 @@ enum class RsForumEventCode: uint8_t SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned DELETED_FORUM = 0x0c, /// forum was deleted by cleaning - DELETED_POSTS = 13 /// Posts deleted by cleaning + DELETED_POSTS = 0x0d /// Posts deleted by cleaning }; struct RsGxsForumEvent: RsEvent @@ -129,9 +129,9 @@ struct RsGxsForumEvent: RsEvent RsForumEventCode mForumEventCode; RsGxsGroupId mForumGroupId; - std::set mForumMsgsId; - std::list mModeratorsAdded; - std::list mModeratorsRemoved; + RsGxsMessageId mForumMsgId; + std::list mModeratorsAdded; + std::list mModeratorsRemoved; ///* @see RsEvent @see RsSerializable void serial_process( @@ -141,7 +141,7 @@ struct RsGxsForumEvent: RsEvent RsEvent::serial_process(j, ctx); RS_SERIAL_PROCESS(mForumEventCode); RS_SERIAL_PROCESS(mForumGroupId); - RS_SERIAL_PROCESS(mForumMsgsId); + RS_SERIAL_PROCESS(mForumMsgId); RS_SERIAL_PROCESS(mModeratorsAdded); RS_SERIAL_PROCESS(mModeratorsRemoved); } diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index b8d2194c5..a57d4e68f 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -211,7 +211,7 @@ void p3GxsForums::notifyChanges(std::vector &changes) if(msgChange) /* Message received*/ { auto ev = std::make_shared(); - ev->mForumMsgsId.insert(msgChange->mMsgId); + ev->mForumMsgId = msgChange->mMsgId; ev->mForumGroupId = msgChange->mGroupId; ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE; rsEvents->postEvent(ev); @@ -258,8 +258,8 @@ void p3GxsForums::notifyChanges(std::vector &changes) } case RsGxsNotify::TYPE_MESSAGE_DELETED: { - rs_view_ptr delChange = - dynamic_cast(gxsChange); + rs_view_ptr delChange = + dynamic_cast(gxsChange); if(!delChange) { @@ -272,7 +272,7 @@ void p3GxsForums::notifyChanges(std::vector &changes) auto ev = std::make_shared(); ev->mForumEventCode = RsForumEventCode::DELETED_POSTS; ev->mForumGroupId = delChange->mGroupId; - ev->mForumMsgsId = delChange->messagesId; + ev->mForumMsgId = delChange->messageId; break; } case RsGxsNotify::TYPE_GROUP_DELETED: @@ -1053,7 +1053,7 @@ void p3GxsForums::setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& { auto ev = std::make_shared(); - ev->mForumMsgsId.insert(msgId.second); + ev->mForumMsgId = msgId.second; ev->mForumGroupId = msgId.first; ev->mForumEventCode = RsForumEventCode::READ_STATUS_CHANGED; rsEvents->postEvent(ev); @@ -1084,7 +1084,7 @@ std::error_condition p3GxsForums::setPostKeepForever( { auto ev = std::make_shared(); ev->mForumGroupId = forumId; - ev->mForumMsgsId.insert(postId); + ev->mForumMsgId = postId; ev->mForumEventCode = RsForumEventCode::UPDATED_MESSAGE; rsEvents->postEvent(ev); return std::error_condition(); diff --git a/retroshare-gui/src/gui/NewsFeed.cpp b/retroshare-gui/src/gui/NewsFeed.cpp index 3c86de318..4e59685b3 100644 --- a/retroshare-gui/src/gui/NewsFeed.cpp +++ b/retroshare-gui/src/gui/NewsFeed.cpp @@ -270,9 +270,9 @@ void NewsFeed::handleForumEvent(std::shared_ptr event) case RsForumEventCode::UPDATED_MESSAGE: case RsForumEventCode::NEW_MESSAGE: - for(const auto& postId: pe->mForumMsgsId) addFeedItem(new GxsForumMsgItem( - this, NEWSFEED_NEW_FORUM, pe->mForumGroupId, postId, + this, NEWSFEED_NEW_FORUM, + pe->mForumGroupId, pe->mForumMsgId, false, true )); break; From ab349a8157711c61ad4030c4f1557e6af37fef52 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 15 Jan 2021 16:42:29 +0100 Subject: [PATCH 03/10] Use unannotated pointer as per Cyril taste --- libretroshare/src/services/p3gxsforums.cc | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index a57d4e68f..cd7cbb5fe 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -197,16 +197,14 @@ void p3GxsForums::notifyChanges(std::vector &changes) std::vector::iterator it; for(it = changes.begin(); it != changes.end(); ++it) { - rs_view_ptr gxsChange = *it; + RsGxsNotify* gxsChange = *it; switch(gxsChange->getType()) { case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]] case RsGxsNotify::TYPE_PUBLISHED: { - rs_view_ptr msgChange = - dynamic_cast(*it); - rs_view_ptr groupChange = - dynamic_cast(*it); + RsGxsMsgChange* msgChange = dynamic_cast(*it); + RsGxsGroupChange* groupChange = dynamic_cast(*it); if(msgChange) /* Message received*/ { @@ -258,7 +256,7 @@ void p3GxsForums::notifyChanges(std::vector &changes) } case RsGxsNotify::TYPE_MESSAGE_DELETED: { - rs_view_ptr delChange = + RsGxsMsgDeletedChange* delChange = dynamic_cast(gxsChange); if(!delChange) @@ -301,8 +299,7 @@ void p3GxsForums::notifyChanges(std::vector &changes) * analyse the old and new group in order to detect possible * notifications for clients */ - rs_view_ptr grpChange = - dynamic_cast(*it); + RsGxsGroupChange* grpChange = dynamic_cast(*it); RsGxsForumGroupItem* old_forum_grp_item = dynamic_cast(grpChange->mOldGroupItem); From 1b551d809f419db94e0a2ae88be803ac8115cee4 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Mon, 8 Feb 2021 17:04:04 +0100 Subject: [PATCH 04/10] First working prototype of GXS service search Channels are now able to take advantage of the new deep search goodies Rebase on top of master 2021/10/19 --- .../src/deep_search/channelsindex.cpp | 169 +++-- .../src/deep_search/channelsindex.hpp | 39 +- libretroshare/src/deep_search/commonutils.cpp | 154 ++++- libretroshare/src/deep_search/commonutils.hpp | 34 +- libretroshare/src/deep_search/filesindex.cpp | 83 ++- libretroshare/src/deep_search/filesindex.hpp | 23 +- .../src/file_sharing/directory_storage.cc | 6 +- libretroshare/src/ft/ftserver.cc | 4 +- libretroshare/src/gxs/rsgenexchange.cc | 42 +- libretroshare/src/gxs/rsgxsnetservice.cc | 134 ++-- libretroshare/src/gxs/rsgxsnetservice.h | 62 +- libretroshare/src/gxs/rsgxsnettunnel.cc | 607 +++++++++++++----- libretroshare/src/gxs/rsgxsnettunnel.h | 62 +- libretroshare/src/gxs/rsgxsutil.cc | 145 +---- libretroshare/src/gxs/rsgxsutil.h | 36 +- libretroshare/src/gxs/rsnxs.h | 62 +- .../rsmemory.cc => gxs/rsnxsobserver.cpp} | 89 ++- libretroshare/src/gxs/rsnxsobserver.h | 104 ++- libretroshare/src/libretroshare.pro | 12 +- libretroshare/src/retroshare/rsgxschannels.h | 147 +++-- libretroshare/src/retroshare/rsgxsiface.h | 62 +- .../src/retroshare/rsgxsifacetypes.h | 2 +- libretroshare/src/rsitems/rsgxschannelitems.h | 65 +- libretroshare/src/rsitems/rsitem.h | 5 +- libretroshare/src/rsitems/rsserviceids.h | 14 +- libretroshare/src/services/p3gxschannels.h | 74 +-- libretroshare/src/util/rsdebug.cc | 9 +- libretroshare/src/util/rsdebug.h | 10 +- libretroshare/src/util/rsmemory.h | 70 +- .../src/gui/gxschannels/GxsChannelDialog.cpp | 4 +- 30 files changed, 1455 insertions(+), 874 deletions(-) rename libretroshare/src/{util/rsmemory.cc => gxs/rsnxsobserver.cpp} (55%) diff --git a/libretroshare/src/deep_search/channelsindex.cpp b/libretroshare/src/deep_search/channelsindex.cpp index cd1c374fc..86a1e341a 100644 --- a/libretroshare/src/deep_search/channelsindex.cpp +++ b/libretroshare/src/deep_search/channelsindex.cpp @@ -1,8 +1,8 @@ /******************************************************************************* * RetroShare full text indexing and search implementation based on Xapian * * * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * - * Copyright (C) 2019 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License version 3 as * @@ -20,16 +20,23 @@ #include "deep_search/channelsindex.hpp" #include "deep_search/commonutils.hpp" +#include "retroshare/rsinit.h" +#include "util/rsdebuglevel3.h" -uint32_t DeepChannelsIndex::search( +/*static*/ std::string DeepChannelsIndex::dbDefaultPath() +{ return RsAccounts::AccountDirectory() + "/deep_channels_xapian_db"; } + +std::error_condition DeepChannelsIndex::search( const std::string& queryStr, std::vector& results, uint32_t maxResults ) { + RS_DBG3(queryStr); + results.clear(); std::unique_ptr dbPtr( - DeepSearch::openReadOnlyDatabase(dbPath()) ); - if(!dbPtr) return 0; + DeepSearch::openReadOnlyDatabase(mDbPath) ); + if(!dbPtr) return std::errc::bad_file_descriptor; Xapian::Database& db(*dbPtr); @@ -63,17 +70,13 @@ uint32_t DeepChannelsIndex::search( results.push_back(s); } - return static_cast(results.size()); + return std::error_condition(); } -void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan) +std::error_condition DeepChannelsIndex::indexChannelGroup( + const RsGxsChannelGroup& chan ) { - std::unique_ptr dbPtr( - DeepSearch::openWritableDatabase( - dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); - if(!dbPtr) return; - - Xapian::WritableDatabase& db(*dbPtr); + RS_DBG4(chan); // Set up a TermGenerator that we'll use in indexing. Xapian::TermGenerator termgenerator; @@ -94,21 +97,8 @@ void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan) termgenerator.increase_termpos(); termgenerator.index_text(chan.mDescription); - RsUrl chanUrl; chanUrl - .setScheme("retroshare").setPath("/channel") - .setQueryKV("id", chan.mMeta.mGroupId.toStdString()); - const std::string idTerm("Q" + chanUrl.toString()); - - chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs)); - chanUrl.setQueryKV("name", chan.mMeta.mGroupName); - if(!chan.mMeta.mAuthorId.isNull()) - chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString()); - if(chan.mMeta.mSignFlags) - chanUrl.setQueryKV( "signFlags", - std::to_string(chan.mMeta.mSignFlags) ); - std::string rsLink(chanUrl.toString()); - // store the RS link so we are able to retrive it on matching search + const std::string rsLink(channelIndexId(chan.mMeta.mGroupId)); doc.add_value(URL_VALUENO, rsLink); // Store some fields for display purposes. @@ -117,35 +107,32 @@ void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan) // We use the identifier to ensure each object ends up in the // database only once no matter how many times we run the // indexer. "Q" prefix is a Xapian convention for unique id term. + const std::string idTerm("Q" + rsLink); doc.add_boolean_term(idTerm); - db.replace_document(idTerm, doc); + + mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db) + { db.replace_document(idTerm, doc); } ); + + return std::error_condition(); } -void DeepChannelsIndex::removeChannelFromIndex(RsGxsGroupId grpId) +std::error_condition DeepChannelsIndex::removeChannelFromIndex( + const RsGxsGroupId& grpId ) { + RS_DBG3(grpId); + // "Q" prefix is a Xapian convention for unique id term. - RsUrl chanUrl; chanUrl - .setScheme("retroshare").setPath("/channel") - .setQueryKV("id", grpId.toStdString()); - std::string idTerm("Q" + chanUrl.toString()); + const std::string idTerm("Q" + channelIndexId(grpId)); + mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db) + { db.delete_document(idTerm); } ); - std::unique_ptr dbPtr( - DeepSearch::openWritableDatabase( - dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); - if(!dbPtr) return; - - Xapian::WritableDatabase& db(*dbPtr); - db.delete_document(idTerm); + return std::error_condition(); } -void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post) +std::error_condition DeepChannelsIndex::indexChannelPost( + const RsGxsChannelPost& post ) { - std::unique_ptr dbPtr( - DeepSearch::openWritableDatabase( - dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); - if(!dbPtr) return; - - Xapian::WritableDatabase& db(*dbPtr); + RS_DBG4(post); // Set up a TermGenerator that we'll use in indexing. Xapian::TermGenerator termgenerator; @@ -160,21 +147,16 @@ void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post) termgenerator.index_text( DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" ); - // TODO: we should strip out HTML tags instead of skipping indexing - // Avoid indexing HTML - bool isPlainMsg = - post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>'; - - if(isPlainMsg) - termgenerator.index_text(post.mMsg, 1, "XD"); + // Avoid indexing RetroShare-gui HTML tags + const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg); + termgenerator.index_text( + DeepSearch::simpleTextHtmlExtract(post.mMsg), 1, "XD" ); // Index fields without prefixes for general search. termgenerator.index_text(post.mMeta.mMsgName); - if(isPlainMsg) - { - termgenerator.increase_termpos(); - termgenerator.index_text(post.mMsg); - } + + termgenerator.increase_termpos(); + termgenerator.index_text(cleanMsg); for(const RsGxsFile& attachment : post.mFiles) { @@ -184,47 +166,50 @@ void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post) termgenerator.index_text(attachment.mName); } - // We use the identifier to ensure each object ends up in the - // database only once no matter how many times we run the - // indexer. - RsUrl postUrl; postUrl - .setScheme("retroshare").setPath("/channel") - .setQueryKV("id", post.mMeta.mGroupId.toStdString()) - .setQueryKV("msgid", post.mMeta.mMsgId.toStdString()); - std::string idTerm("Q" + postUrl.toString()); - - postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs)); - postUrl.setQueryKV("name", post.mMeta.mMsgName); - postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString()); - std::string rsLink(postUrl.toString()); - // store the RS link so we are able to retrive it on matching search + const std::string rsLink(postIndexId(post.mMeta.mGroupId, post.mMeta.mMsgId)); doc.add_value(URL_VALUENO, rsLink); // Store some fields for display purposes. - if(isPlainMsg) - doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg); - else doc.set_data(post.mMeta.mMsgName); + doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg); + // We use the identifier to ensure each object ends up in the + // database only once no matter how many times we run the + // indexer. + const std::string idTerm("Q" + rsLink); doc.add_boolean_term(idTerm); - db.replace_document(idTerm, doc); + + mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db) + { db.replace_document(idTerm, doc); } ); + + return std::error_condition(); } -void DeepChannelsIndex::removeChannelPostFromIndex( +std::error_condition DeepChannelsIndex::removeChannelPostFromIndex( + const RsGxsGroupId& grpId, const RsGxsMessageId& msgId ) +{ + RS_DBG3(grpId, msgId); + + std::string idTerm("Q" + postIndexId(grpId, msgId)); + mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db) + { db.delete_document(idTerm); } ); + + return std::error_condition(); +} + +/*static*/ std::string DeepChannelsIndex::channelIndexId(RsGxsGroupId grpId) +{ + RsUrl channelIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL); + channelIndexId.setQueryKV( + RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString() ); + return channelIndexId.toString(); +} + +/*static*/ std::string DeepChannelsIndex::postIndexId( RsGxsGroupId grpId, RsGxsMessageId msgId ) { - RsUrl postUrl; postUrl - .setScheme("retroshare").setPath("/channel") - .setQueryKV("id", grpId.toStdString()) - .setQueryKV("msgid", msgId.toStdString()); - // "Q" prefix is a Xapian convention for unique id term. - std::string idTerm("Q" + postUrl.toString()); - - std::unique_ptr dbPtr( - DeepSearch::openWritableDatabase( - dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); - if(!dbPtr) return; - - Xapian::WritableDatabase& db(*dbPtr); - db.delete_document(idTerm); + RsUrl postIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL); + postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString()); + postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_MSG_ID_FIELD, msgId.toStdString()); + return postIndexId.toString(); } diff --git a/libretroshare/src/deep_search/channelsindex.hpp b/libretroshare/src/deep_search/channelsindex.hpp index 0a49629d9..b767ce2b4 100644 --- a/libretroshare/src/deep_search/channelsindex.hpp +++ b/libretroshare/src/deep_search/channelsindex.hpp @@ -1,8 +1,8 @@ /******************************************************************************* * RetroShare full text indexing and search implementation based on Xapian * * * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * - * Copyright (C) 2019 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License version 3 as * @@ -24,8 +24,8 @@ #include "util/rstime.h" #include "retroshare/rsgxschannels.h" -#include "retroshare/rsinit.h" #include "util/rsurl.h" +#include "deep_search/commonutils.hpp" struct DeepChannelsSearchResult { @@ -36,28 +36,34 @@ struct DeepChannelsSearchResult struct DeepChannelsIndex { + explicit DeepChannelsIndex(const std::string& dbPath) : + mDbPath(dbPath), mWriteQueue(dbPath) {} + /** * @brief Search indexed GXS groups and messages * @param[in] maxResults maximum number of acceptable search results, 0 for * no limits * @return search results count */ - static uint32_t search( const std::string& queryStr, - std::vector& results, - uint32_t maxResults = 100 ); + std::error_condition search( + const std::string& queryStr, + std::vector& results, + uint32_t maxResults = 100 ); - static void indexChannelGroup(const RsGxsChannelGroup& chan); + std::error_condition indexChannelGroup(const RsGxsChannelGroup& chan); - static void removeChannelFromIndex(RsGxsGroupId grpId); + std::error_condition removeChannelFromIndex(const RsGxsGroupId& grpId); - static void indexChannelPost(const RsGxsChannelPost& post); + std::error_condition indexChannelPost(const RsGxsChannelPost& post); - static void removeChannelPostFromIndex( - RsGxsGroupId grpId, RsGxsMessageId msgId ); + std::error_condition removeChannelPostFromIndex( + const RsGxsGroupId& grpId, const RsGxsMessageId& msgId ); - static uint32_t indexFile(const std::string& path); + static std::string dbDefaultPath(); private: + static std::string channelIndexId(RsGxsGroupId grpId); + static std::string postIndexId(RsGxsGroupId grpId, RsGxsMessageId msgId); enum : Xapian::valueno { @@ -68,10 +74,7 @@ private: BAD_VALUENO = Xapian::BAD_VALUENO }; - static const std::string& dbPath() - { - static const std::string dbDir = - RsAccounts::AccountDirectory() + "/deep_channels_xapian_db"; - return dbDir; - } + const std::string mDbPath; + + DeepSearch::StubbornWriteOpQueue mWriteQueue; }; diff --git a/libretroshare/src/deep_search/commonutils.cpp b/libretroshare/src/deep_search/commonutils.cpp index eecbd4ec6..cbe4ee27b 100644 --- a/libretroshare/src/deep_search/commonutils.cpp +++ b/libretroshare/src/deep_search/commonutils.cpp @@ -1,8 +1,8 @@ /******************************************************************************* * RetroShare full text indexing and search implementation based on Xapian * * * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * - * Copyright (C) 2019 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License version 3 as * @@ -18,39 +18,17 @@ * * *******************************************************************************/ +#include +#include + #include "deep_search/commonutils.hpp" #include "util/stacktrace.h" -#include "util/rsdebug.h" +#include "util/rsthreads.h" +#include "util/rsdebuglevel0.h" + namespace DeepSearch { - -std::unique_ptr openWritableDatabase( - const std::string& path, int flags, int blockSize ) -{ - try - { - std::unique_ptr dbPtr( - new Xapian::WritableDatabase(path, flags, blockSize) ); - return dbPtr; - } - catch(Xapian::DatabaseLockError) - { - RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock " - << path << std::endl; - print_stacktrace(); - } - catch(...) - { - RsErr() << __PRETTY_FUNCTION__ << " Xapian DB is apparently corrupted " - << "deleting it might help without causing any harm: " - << path << std::endl; - print_stacktrace(); - } - - return nullptr; -} - std::unique_ptr openReadOnlyDatabase( const std::string& path, int flags ) { @@ -60,12 +38,12 @@ std::unique_ptr openReadOnlyDatabase( new Xapian::Database(path, flags) ); return dbPtr; } - catch(Xapian::DatabaseOpeningError e) + catch(Xapian::DatabaseOpeningError& e) { RsWarn() << __PRETTY_FUNCTION__ << " " << e.get_msg() << ", probably nothing has been indexed yet." << std::endl; } - catch(Xapian::DatabaseLockError) + catch(Xapian::DatabaseLockError&) { RsErr() << __PRETTY_FUNCTION__ << " Failed aquiring Xapian DB lock " << path << std::endl; @@ -90,4 +68,116 @@ std::string timetToXapianDate(const rstime_t& time) return date; } +StubbornWriteOpQueue::~StubbornWriteOpQueue() +{ + auto fErr = flush(0); + if(fErr) + { + RS_FATAL( "Flush failed on destruction ", mOpStore.size(), + " operations irreparably lost ", fErr ); + print_stacktrace(); + } +} + +void StubbornWriteOpQueue::push(write_op op) +{ + RS_DBG4(""); + + { + std::unique_lock lock(mQueueMutex); + mOpStore.push(op); + } + + flush(); +} + +std::error_condition StubbornWriteOpQueue::flush( + rstime_t acceptDelay, rstime_t callTS ) +{ + RS_DBG4(""); + + { + // Return without attempt to open the database if the queue is empty + std::unique_lock lock(mQueueMutex); + if(mOpStore.empty()) return std::error_condition(); + } + + std::unique_ptr dbPtr; + try + { + dbPtr = std::make_unique( + mDbPath, Xapian::DB_CREATE_OR_OPEN ); + } + catch(Xapian::DatabaseLockError) + { + if(acceptDelay) + { + rstime_t tNow = time(nullptr); + rstime_t maxRemaining = tNow - (callTS + acceptDelay); + if(maxRemaining > 0) + { + std::chrono::milliseconds interval( + std::max(50l, maxRemaining*1000/5) ); + RS_DBG3( "Cannot acquire database write lock, retrying in:", + interval.count(), "ms" ); + RsThread::async([this, acceptDelay, callTS, interval]() + { + std::this_thread::sleep_for(interval); + flush(acceptDelay, callTS); + }); + return std::error_condition(); + } + else + { + RS_ERR(std::errc::timed_out, acceptDelay, callTS, tNow); + return std::errc::timed_out; + } + } + else return std::errc::resource_unavailable_try_again; + } + catch(...) + { + RS_ERR("Xapian DB ", mDbPath, " is apparently corrupted"); + print_stacktrace(); + return std::errc::io_error; + } + + std::unique_lock lock(mQueueMutex); + while(!mOpStore.empty()) + { + auto op = mOpStore.front(); mOpStore.pop(); + op(*dbPtr); + } + return std::error_condition(); +} + +std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc) +{ + if(rsHtmlDoc.empty()) return rsHtmlDoc; + + const bool isPlainMsg = + rsHtmlDoc[0] != '<' || rsHtmlDoc[rsHtmlDoc.size() - 1] != '>'; + if(isPlainMsg) return rsHtmlDoc; + + auto oSize = rsHtmlDoc.size(); + auto bodyTagBegin(rsHtmlDoc.find("= oSize) return rsHtmlDoc; + + auto bodyTagEnd(rsHtmlDoc.find(">", bodyTagBegin)); + if(bodyTagEnd >= oSize) return rsHtmlDoc; + + std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1)); + + std::string::size_type oPos; + std::string::size_type cPos; + while((oPos = retVal.find("<")) < retVal.size()) + { + if((cPos = retVal.find(">")) <= retVal.size()) + retVal.erase(oPos, 1+cPos-oPos); + else break; + } + + return retVal; +} + } diff --git a/libretroshare/src/deep_search/commonutils.hpp b/libretroshare/src/deep_search/commonutils.hpp index 28961bc09..5f47c39bd 100644 --- a/libretroshare/src/deep_search/commonutils.hpp +++ b/libretroshare/src/deep_search/commonutils.hpp @@ -1,8 +1,8 @@ /******************************************************************************* * RetroShare full text indexing and search implementation based on Xapian * * * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * - * Copyright (C) 2019 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License version 3 as * @@ -21,6 +21,9 @@ #include #include +#include +#include +#include #include "util/rstime.h" @@ -33,13 +36,34 @@ namespace DeepSearch { - -std::unique_ptr openWritableDatabase( - const std::string& path, int flags = 0, int blockSize = 0 ); +typedef std::function write_op; std::unique_ptr openReadOnlyDatabase( const std::string& path, int flags = 0 ); std::string timetToXapianDate(const rstime_t& time); +std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc); + +struct StubbornWriteOpQueue +{ + explicit StubbornWriteOpQueue(const std::string& dbPath): + mDbPath(dbPath) {} + + ~StubbornWriteOpQueue(); + + void push(write_op op); + + std::error_condition flush( + rstime_t acceptDelay = 20, rstime_t callTS = time(nullptr) ); + +private: + std::queue mOpStore; + rstime_t mLastFlush; + + std::mutex mQueueMutex; + + const std::string mDbPath; +}; + } diff --git a/libretroshare/src/deep_search/filesindex.cpp b/libretroshare/src/deep_search/filesindex.cpp index 3edcf9a97..9d5b09a72 100644 --- a/libretroshare/src/deep_search/filesindex.cpp +++ b/libretroshare/src/deep_search/filesindex.cpp @@ -1,8 +1,8 @@ /******************************************************************************* * RetroShare full text indexing and search implementation based on Xapian * * * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * - * Copyright (C) 2019 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License version 3 as * @@ -18,47 +18,47 @@ * * *******************************************************************************/ -#include "deep_search/filesindex.hpp" -#include "deep_search/commonutils.hpp" -#include "util/rsdebug.h" -#include "retroshare/rsinit.h" -#include "retroshare/rsversion.h" #include +#include "deep_search/filesindex.hpp" +#include "deep_search/commonutils.hpp" +#include "util/rsdebuglevel1.h" +#include "retroshare/rsinit.h" +#include "retroshare/rsversion.h" + /*static*/ std::multimap DeepFilesIndex::indexersRegister = {}; -bool DeepFilesIndex::indexFile( +std::error_condition DeepFilesIndex::indexFile( const std::string& path, const std::string& name, const RsFileHash& hash ) { - auto dbPtr = DeepSearch::openWritableDatabase( - mDbPath, Xapian::DB_CREATE_OR_OPEN ); - if(!dbPtr) return false; - Xapian::WritableDatabase& db(*dbPtr); - const std::string hashString = hash.toStdString(); const std::string idTerm("Q" + hashString); - Xapian::Document oldDoc; - Xapian::PostingIterator pIt = db.postlist_begin(idTerm); - if( pIt != db.postlist_end(idTerm) ) + auto db = DeepSearch::openReadOnlyDatabase(mDbPath); + if(db) { - oldDoc = db.get_document(*pIt); - if( oldDoc.get_value(INDEXER_VERSION_VALUENO) == - RS_HUMAN_READABLE_VERSION && - std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) == - indexersRegister.size() ) + Xapian::Document oldDoc; + Xapian::PostingIterator pIt = db->postlist_begin(idTerm); + if( pIt != db->postlist_end(idTerm) ) { - /* Looks like this file has already been indexed by this RetroShare - * exact version, so we can skip it. If the version was different it - * made sense to reindex it as better indexers might be available - * since last time it was indexed */ - Dbg3() << __PRETTY_FUNCTION__ << " skipping laready indexed file: " - << hash << " " << name << std::endl; - return true; + oldDoc = db->get_document(*pIt); + if( oldDoc.get_value(INDEXER_VERSION_VALUENO) == + RS_HUMAN_READABLE_VERSION && + std::stoull(oldDoc.get_value(INDEXERS_COUNT_VALUENO)) == + indexersRegister.size() ) + { + /* Looks like this file has already been indexed by this + * RetroShare exact version, so we can skip it. If the version + * was different it made sense to reindex it as better indexers + * might be available since last time it was indexed */ + RS_DBG3("skipping laready indexed file: ", hash, " ", name); + return std::error_condition(); + } } + db.reset(); // Release DB read lock ASAP } Xapian::Document doc; @@ -80,22 +80,21 @@ bool DeepFilesIndex::indexFile( doc.add_value( INDEXERS_COUNT_VALUENO, std::to_string(indexersRegister.size()) ); - db.replace_document(idTerm, doc); - return true; + mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db) + { db.replace_document(idTerm, doc); }); + + return std::error_condition(); } -bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash) +std::error_condition DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash) { - Dbg3() << __PRETTY_FUNCTION__ << " removing file from index: " - << hash << std::endl; + RS_DBG3(hash); - std::unique_ptr db = - DeepSearch::openWritableDatabase(mDbPath, Xapian::DB_CREATE_OR_OPEN); - if(!db) return false; + mWriteQueue.push([hash](Xapian::WritableDatabase& db) + { db.delete_document("Q" + hash.toStdString()); }); - db->delete_document("Q" + hash.toStdString()); - return true; + return std::error_condition(); } /*static*/ std::string DeepFilesIndex::dbDefaultPath() @@ -104,20 +103,20 @@ bool DeepFilesIndex::removeFileFromIndex(const RsFileHash& hash) /*static*/ bool DeepFilesIndex::registerIndexer( int order, const DeepFilesIndex::IndexerFunType& indexerFun ) { - Dbg1() << __PRETTY_FUNCTION__ << " " << order << std::endl; + RS_DBG1(order); indexersRegister.insert(std::make_pair(order, indexerFun)); return true; } -uint32_t DeepFilesIndex::search( +std::error_condition DeepFilesIndex::search( const std::string& queryStr, std::vector& results, uint32_t maxResults ) { results.clear(); auto dbPtr = DeepSearch::openReadOnlyDatabase(mDbPath); - if(!dbPtr) return 0; + if(!dbPtr) return std::errc::bad_file_descriptor; Xapian::Database& db(*dbPtr); // Set up a QueryParser with a stemmer and suitable prefixes. @@ -151,7 +150,7 @@ uint32_t DeepFilesIndex::search( results.push_back(s); } - return static_cast(results.size()); + return std::error_condition(); } diff --git a/libretroshare/src/deep_search/filesindex.hpp b/libretroshare/src/deep_search/filesindex.hpp index f811e5e9c..b337bd072 100644 --- a/libretroshare/src/deep_search/filesindex.hpp +++ b/libretroshare/src/deep_search/filesindex.hpp @@ -1,8 +1,8 @@ /******************************************************************************* * RetroShare full text indexing and search implementation based on Xapian * * * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * - * Copyright (C) 2019 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License version 3 as * @@ -19,9 +19,6 @@ *******************************************************************************/ #pragma once -#include "retroshare/rstypes.h" -#include "util/rsdebug.h" - #include #include #include @@ -29,6 +26,9 @@ #include #include +#include "retroshare/rstypes.h" +#include "deep_search/commonutils.hpp" + struct DeepFilesSearchResult { DeepFilesSearchResult() : mWeight(0) {} @@ -41,7 +41,8 @@ struct DeepFilesSearchResult class DeepFilesIndex { public: - DeepFilesIndex(const std::string& dbPath) : mDbPath(dbPath) {} + explicit DeepFilesIndex(const std::string& dbPath): + mDbPath(dbPath), mWriteQueue(dbPath) {} /** * @brief Search indexed files @@ -49,7 +50,7 @@ public: * no limits * @return search results count */ - uint32_t search( const std::string& queryStr, + std::error_condition search( const std::string& queryStr, std::vector& results, uint32_t maxResults = 100 ); @@ -57,7 +58,7 @@ public: * @return false if file could not be indexed because of error or * unsupported type, true otherwise. */ - bool indexFile( + std::error_condition indexFile( const std::string& path, const std::string& name, const RsFileHash& hash ); @@ -65,7 +66,7 @@ public: * @brief Remove file entry from database * @return false on error, true otherwise. */ - bool removeFileFromIndex(const RsFileHash& hash); + std::error_condition removeFileFromIndex(const RsFileHash& hash); static std::string dbDefaultPath(); @@ -96,8 +97,8 @@ private: const std::string mDbPath; + DeepSearch::StubbornWriteOpQueue mWriteQueue; + /** Storage for indexers function by order */ static std::multimap indexersRegister; - - RS_SET_CONTEXT_DEBUG_LEVEL(1) }; diff --git a/libretroshare/src/file_sharing/directory_storage.cc b/libretroshare/src/file_sharing/directory_storage.cc index 402cae533..583ac5882 100644 --- a/libretroshare/src/file_sharing/directory_storage.cc +++ b/libretroshare/src/file_sharing/directory_storage.cc @@ -4,8 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2016 Mr.Alice * - * Copyright (C) 2021 Gioacchino Mazzurco * - * Copyright (C) 2021 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -549,7 +549,7 @@ bool LocalDirectoryStorage::updateHash( fInfo.storage_permission_flags & DIR_FLAGS_ANONYMOUS_SEARCH ) { DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath()); - ret &= dfi.indexFile(fInfo.path, fInfo.fname, hash); + ret &= !dfi.indexFile(fInfo.path, fInfo.fname, hash); } #endif // def RS_DEEP_FILES_INDEX diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 7d5dadb12..bf745e9e8 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -5,7 +5,7 @@ * * * Copyright (C) 2008 Robert Fernie * * Copyright (C) 2018-2021 Gioacchino Mazzurco * - * Copyright (C) 2020-2021 Asociación Civil Altermundi * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -2037,7 +2037,7 @@ bool ftServer::receiveSearchRequest( std::vector dRes; DeepFilesIndex dfi(DeepFilesIndex::dbDefaultPath()); - if(dfi.search(searchReq.queryString, dRes, maxAllowsHits) > 0) + if(!dfi.search(searchReq.queryString, dRes, maxAllowsHits)) { RsFileSearchResultItem resIt; diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 99725cd70..0a9db1e58 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -5,6 +5,7 @@ * * * Copyright (C) 2012 Christopher Evi-Parker * * Copyright (C) 2019-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -193,47 +194,6 @@ RsGenExchange::RsGenExchange( VALIDATE_MAX_WAITING_TIME(60) { mDataAccess = new RsGxsDataAccess(gds); - - // Perform an early checking/cleaning of the db. Will eliminate groups and messages that do not match their hash - -#ifdef RS_DEEP_CHANNEL_INDEX - // This code is only called because it of deep indexing in channels. But loading - // the entire set of messages in order to provide indexing is pretty bad (very costly and slowly - // eats memory, as many tests have shown. Not because of leaks, but because new threads are - // apparently attributed large stacks and pages of memory are not regained by the system maybe because it thinks - // that RS will use them again. - // - // * the deep check should be implemented differently, in an incremental way. For instance in notifyChanges() of each - // service (e.g. channels here) should update the index when a new message is received. The question to how old messages - // are processed is open. I believe that they shouldn't. New users will progressively process them. - // - // * integrity check (re-hashing of message data) is not needed. Message signature already ensures that all messages received are - // unalterated. The only problem (possibly very rare) is that a message is locally corrupted and not deleted (because of no check). - // It will therefore never be replaced by the correct one from friends. The cost of re-hashing the whole db data regularly - // doesn't counterbalance such a low risk. - // - if(mServType == RS_SERVICE_GXS_TYPE_CHANNELS) - { - std::vector grpsToDel; - GxsMsgReq msgsToDel; - - RsGxsSinglePassIntegrityCheck::check(mServType,mGixs,mDataStore, - this, *mSerialiser, - grpsToDel,msgsToDel); - - for(auto& grpId: grpsToDel) - { - uint32_t token2=0; - deleteGroup(token2,grpId); - } - - if(!msgsToDel.empty()) - { - uint32_t token1=0; - deleteMsgs(token1,msgsToDel); - } - } -#endif } void RsGenExchange::setNetworkExchangeService(RsNetworkExchangeService *ns) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index e1b70a670..8038b63f4 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2012-2012 by Christopher Evi-Parker * + * Copyright (C) 2012 Christopher Evi-Parker * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -5204,13 +5206,14 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st return mGxsNetTunnel->turtleSearchRequest(match_string,this) ; } -#ifndef RS_DEEP_CHANNEL_INDEX static bool termSearch(const std::string& src, const std::string& substring) { - /* always ignore case */ - return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() ); + /* always ignore case */ + return src.end() != std::search( + src.begin(), src.end(), substring.begin(), substring.end(), + RsRegularExpression::CompareCharIC() ); } -#endif // ndef RS_DEEP_CHANNEL_INDEX + bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map& group_infos) { @@ -5246,7 +5249,8 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id) return true ; } -void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std::list& group_infos ) +void RsGxsNetService::receiveTurtleSearchResults( + TurtleRequestId req, const std::list& group_infos ) { std::set groupsToNotifyResults; @@ -5276,26 +5280,20 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std for (const RsGxsGroupSummary& gps : group_infos) { -#ifndef RS_DEEP_CHANNEL_INDEX +#ifdef TO_REMOVE + /* Because of deep search is enabled search results may bring more + * info then we already have also about post that are indexed by + * xapian, so we don't apply this filter anymore. */ + /* Only keep groups that are not locally known, and groups that are * not already in the mDistantSearchResults structure. - * mDataStore may in some situations allocate an empty group meta data, so it's important - * to test that the group meta is both non null and actually corresponds to the group id we seek. */ + * mDataStore may in some situations allocate an empty group meta + * data, so it's important to test that the group meta is both non + * null and actually corresponds to the group id we seek. */ + auto& meta(grpMeta[gps.mGroupId]); + if(meta != nullptr && meta->mGroupId == gps.mGroupId) continue; +#endif // def TO_REMOVE - auto& meta(grpMeta[gps.mGroupId]); - - if(meta != nullptr && meta->mGroupId == gps.mGroupId) - continue; - -#ifdef NXS_NET_DEBUG_9 - std::cerr << " group " << gps.mGroupId << " is not known. Adding it to search results..." << std::endl; -#endif - -#else // ndef RS_DEEP_CHANNEL_INDEX - /* When deep search is enabled search results may bring more info - * then we already have also about post that are indexed by xapian, - * so we don't apply this filter in this case. */ -#endif const RsGxsGroupId& grpId(gps.mGroupId); groupsToNotifyResults.insert(grpId); @@ -5332,18 +5330,19 @@ void RsGxsNetService::receiveTurtleSearchResults( TurtleRequestId req, const std mObserver->receiveDistantSearchResults(req, grpId); } -void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len) +void RsGxsNetService::receiveTurtleSearchResults( + TurtleRequestId req, + const uint8_t* encrypted_group_data, uint32_t encrypted_group_data_len ) { #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl; #endif - auto it = mSearchRequests.find(req); - - if(mSearchRequests.end() == it) - { - std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ; - return; - } + auto it = mSearchRequests.find(req); + if(mSearchRequests.end() == it) + { + RS_WARN("Received search results for unknown request: ", req); + return; + } RsGxsGroupId grpId = it->second; uint8_t encryption_master_key[32]; @@ -5417,56 +5416,36 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsig mObserver->receiveDistantSearchResults(req, grpId); } +std::error_condition RsGxsNetService::distantSearchRequest( + rs_owner_ptr searchData, uint32_t dataSize, + RsServiceType serviceType, TurtleRequestId& requestId ) +{ + return mGxsNetTunnel->turtleSearchRequest( + searchData, dataSize, serviceType, requestId ); +} + +std::error_condition RsGxsNetService::handleDistantSearchRequest( + rs_view_ptr requestData, uint32_t requestSize, + rs_owner_ptr& resultData, uint32_t& resultSize ) +{ + RS_DBG(""); + return mObserver->handleDistantSearchRequest( + requestData, requestSize, resultData, resultSize ); +} + +std::error_condition RsGxsNetService::receiveDistantSearchResult( + const TurtleRequestId requestId, + rs_owner_ptr& resultData, uint32_t& resultSize ) +{ + return mObserver->receiveDistantSearchResult( + requestId, resultData, resultSize ); +} + bool RsGxsNetService::search( const std::string& substring, std::list& group_infos ) { group_infos.clear(); -#ifdef RS_DEEP_CHANNEL_INDEX - -#warning TODO: filter deep index search result to non circle-restricted groups. -// /!\ -// /!\ These results should be filtered to only return results coming from a non restricted group! -// /!\ - - std::vector results; - DeepChannelsIndex::search(substring, results); - - for(auto dsr : results) - { - RsUrl rUrl(dsr.mUrl); - const auto& uQ(rUrl.query()); - auto rit = uQ.find("id"); - if(rit != rUrl.query().end()) - { - RsGroupNetworkStats stats; - RsGxsGroupId grpId(rit->second); - if( !grpId.isNull() && getGroupNetworkStats(grpId, stats) ) - { - RsGxsGroupSummary s; - - s.mGroupId = grpId; - - if((rit = uQ.find("name")) != uQ.end()) - s.mGroupName = rit->second; - if((rit = uQ.find("signFlags")) != uQ.end()) - s.mSignFlags = static_cast(std::stoul(rit->second)); - if((rit = uQ.find("publishTs")) != uQ.end()) - s.mPublishTs = static_cast(std::stoll(rit->second)); - if((rit = uQ.find("authorId")) != uQ.end()) - s.mAuthorId = RsGxsId(rit->second); - - s.mSearchContext = dsr.mSnippet; - - s.mNumberOfMessages = stats.mMaxVisibleCount; - s.mLastMessageTs = stats.mLastGroupModificationTS; - s.mPopularity = stats.mSuppliers; - - group_infos.push_back(s); - } - } - } -#else // RS_DEEP_CHANNEL_INDEX RsGxsGrpMetaTemporaryMap grpMetaMap; { RS_STACK_MUTEX(mNxsMutex) ; @@ -5492,12 +5471,11 @@ bool RsGxsNetService::search( const std::string& substring, group_infos.push_back(s); } -#endif // RS_DEEP_CHANNEL_INDEX #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl; #endif - return !group_infos.empty(); + return !group_infos.empty(); } bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len) diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 295b8232a..a4b448003 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2012-2012 by Christopher Evi-Parker * + * Copyright (C) 2012 Christopher Evi-Parker * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -19,8 +21,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef RSGXSNETSERVICE_H -#define RSGXSNETSERVICE_H +#pragma once #include #include @@ -130,18 +131,53 @@ public: virtual bool msgAutoSync() const override { return mAllowMsgSync; } virtual bool grpAutoSync() const override { return mGrpAutoSync; } - /*! - * \brief Search methods. - * These four methods are used to request distant search and receive the results. - * \param group_id - */ - virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)override ; - virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)override ; - virtual bool search(const std::string& substring,std::list& group_infos) override ; + /// @see RsNetworkExchangeService + std::error_condition distantSearchRequest( + rs_owner_ptr searchData, uint32_t dataSize, + RsServiceType serviceType, TurtleRequestId& requestId ) override; + + /// @see RsNetworkExchangeService + std::error_condition handleDistantSearchRequest( + rs_view_ptr requestData, uint32_t requestSize, + rs_owner_ptr& resultData, uint32_t& resultSize ) override; + + /// @see RsNetworkExchangeService + std::error_condition receiveDistantSearchResult( + const TurtleRequestId requestId, + rs_owner_ptr& resultData, uint32_t& resultSize ) override; + + /** Request group data via turtle search + * @param group_id */ + TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override; + + /** + * @brief Search for matching groups names over turtle search. + * @deprecated this method is kept mostly for retrocompatibility with older + * peers, newly implemented search functions should instead be based on the + * service generic search. + * @see RsNetworkExchangeService + */ + RS_DEPRECATED_FOR(distantSearchRequest) + TurtleRequestId turtleSearchRequest(const std::string& match_string) override; + + /** @see RsNetworkExchangeService + * @deprecated kept for retrocompatibility with older peers, new code should + * instead be based on the service generic search */ + RS_DEPRECATED_FOR(receiveDistantSearchResult) + void receiveTurtleSearchResults( + TurtleRequestId req, + const uint8_t* encrypted_group_data, + uint32_t encrypted_group_data_len ) override; + + /** + * @deprecated kept for retrocompatibility with older peers, new code should + * instead be based on the service generic search */ + RS_DEPRECATED_FOR(handleRemoteSearchRequest) + virtual bool search( const std::string& substring, + std::list& group_infos) override; virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)override ; virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos)override ; - virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)override ; virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos)override ; virtual bool clearDistantSearchResults(const TurtleRequestId& id)override ; @@ -629,5 +665,3 @@ private: bool mUseMetaCache; }; - -#endif // RSGXSNETSERVICE_H diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 9b7a3089a..7f70ffa42 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2018 by Cyril Soler * + * Copyright (C) 2018 Cyril Soler * + * Copyright (C) 2019-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -27,6 +29,8 @@ #include "gxs/rsnxs.h" #include "rsgxsnettunnel.h" +/*extern*/ RsGxsDistSync* rsGxsDistSync = nullptr; + //#define DEBUG_RSGXSNETTUNNEL 1 #define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } @@ -36,42 +40,93 @@ static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1; static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100; -RsGxsDistSync *rsGxsDistSync = NULL; RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") { mRandomBias.clear(); - mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20); // adds some variance in order to avoid doing all this tasks at once across services - mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20); - mLastDump = time(NULL) + (RSRandom::random_u32()%20); + /* adds some variance in order to avoid doing all this tasks at once across + * services */ + auto now = time(nullptr); + mLastKeepAlive = now + (RsRandom::random_u32()%20); + mLastAutoWash = now + (RsRandom::random_u32()%20); + mLastDump = now + (RsRandom::random_u32()%20); } -//===========================================================================================================================================// -// Transport Items // -//===========================================================================================================================================// +//============================================================================// +// Transport Items // +//============================================================================// -const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ; +enum class RsGxsNetTunnelItemSubtypes : uint8_t +{ + VIRTUAL_PEER = 0x01, + KEEP_ALIVE = 0x02, + + RANDOM_BIAS = 0x03, + + /// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REQUEST + SEARCH_SUBSTRING = 0x04, + + SEARCH_GROUP_REQUEST = 0x05, + + // SEARCH_GROUP_SUMMARY = 0x06, removed + + SEARCH_GROUP_DATA = 0x07, + + /// @deprecated kept only for retrocompatibility @see SERVICE_SEARCH_REPLY + SEARCH_GROUP_SUMMARY = 0x08, + + /** Generic search request generated and handled by specific service + * (channels, forums...) */ + SERVICE_SEARCH_REQUEST = 0x09, + + /** Generic search reply generated and handled by specific service + * (channels, forums...) */ + SERVICE_SEARCH_REPLY = 0x0a +}; + +RS_DEPRECATED_FOR(RsServiceType::GXS_DISTANT) +constexpr uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = + static_cast(RsServiceType::GXS_DISTANT); + +RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes) const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; +RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes) const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ; +RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes) const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ; +RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes) const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ; +RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes) const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ; -// const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06; // DEPRECATED +RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes) const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ; +RS_DEPRECATED_FOR(RsGxsNetTunnelItemSubtypes) const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x08; +// Do not add new subitems types as const, use RsGxsNetTunnelItemSubtypes instead -class RsGxsNetTunnelItem: public RsItem +struct RsGxsNetTunnelItem: RsItem { public: - explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype) + explicit RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype): + RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL, + static_cast(subtype) ) + { + /* no priority. All items are encapsulated into generic Turtle items + * anyway. */ + } + + virtual ~RsGxsNetTunnelItem() = default; + virtual void clear() {} + + RS_DEPRECATED_FOR("RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes subtype)") + explicit RsGxsNetTunnelItem(uint8_t item_subtype): + RsItem( RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_NET_TUNNEL, + item_subtype ) { // no priority. All items are encapsulated into generic Turtle items anyway. } - - virtual ~RsGxsNetTunnelItem() {} - virtual void clear() {} }; class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem @@ -113,7 +168,86 @@ public: Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization. }; -class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem +struct RsGxsServiceTurtleSearchReqItem: RsGxsNetTunnelItem +{ + RsGxsServiceTurtleSearchReqItem(): + RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST), + mServiceType(RsServiceType::NONE), mSearchData(nullptr), + mSearchDataSize(0) {} + + explicit RsGxsServiceTurtleSearchReqItem(RsServiceType service): + RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST), + mServiceType(service), mSearchData(nullptr), mSearchDataSize(0) {} + + /// Type of the service which originated the search request + RsServiceType mServiceType; + + uint8_t* mSearchData; /// Service search request data + uint32_t mSearchDataSize; /// Search data size + + /// @see RsSerializable + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RS_SERIAL_PROCESS(mServiceType); + + RsTypeSerializer::RawMemoryWrapper prox(mSearchData, mSearchDataSize); + RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData"); + } + + /// @see RsItem + void clear() override + { + free(mSearchData); + mSearchData = nullptr; + mSearchDataSize = 0; + } + + ~RsGxsServiceTurtleSearchReqItem() override { clear(); } +}; + +struct RsGxsServiceTurtleSearchReplyItem: RsGxsNetTunnelItem +{ + RsGxsServiceTurtleSearchReplyItem(): + RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY), + mServiceType(RsServiceType::NONE), mReplyData(nullptr), + mReplyDataSize(0) {} + + explicit RsGxsServiceTurtleSearchReplyItem(RsServiceType service): + RsGxsNetTunnelItem(RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY), + mServiceType(service), mReplyData(nullptr), mReplyDataSize(0) {} + + /// Type of the service which originated the search request + RsServiceType mServiceType; + + uint8_t* mReplyData; /// Service search reply data + uint32_t mReplyDataSize; /// Search reply data size + + /// @see RsSerializable + void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RS_SERIAL_PROCESS(mServiceType); + + RsTypeSerializer::RawMemoryWrapper prox(mReplyData, mReplyDataSize); + RsTypeSerializer::serial_process(j, ctx, prox, "mSearchData"); + } + + /// @see RsItem + void clear() override + { + free(mReplyData); + mReplyData = nullptr; + mReplyDataSize = 0; + } + + ~RsGxsServiceTurtleSearchReplyItem() override { clear(); } +}; + +class RS_DEPRECATED_FOR(RsGxsServiceTurtleSearchItem) +RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem { public: explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {} @@ -164,6 +298,7 @@ public: RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ; } }; + class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem { public: @@ -193,28 +328,41 @@ public: class RsGxsNetTunnelSerializer: public RsServiceSerializer { public: - RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {} + RsGxsNetTunnelSerializer(): + RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {} virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const { if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL) { - GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl; - return NULL ; + RS_ERR( "received item with wrong service ID ", service); + print_stacktrace(); + return nullptr; } - switch(item_subtype) + switch(static_cast(item_subtype)) { - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER : return new RsGxsNetTunnelVirtualPeerItem ; - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ; - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ; - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem; - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem; - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem; - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem; + case RsGxsNetTunnelItemSubtypes::VIRTUAL_PEER: + return new RsGxsNetTunnelVirtualPeerItem; + case RsGxsNetTunnelItemSubtypes::KEEP_ALIVE: + return new RsGxsNetTunnelKeepAliveItem; + case RsGxsNetTunnelItemSubtypes::RANDOM_BIAS: + return new RsGxsNetTunnelRandomBiasItem; + case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING: + return new RsGxsNetTunnelTurtleSearchSubstringItem; + case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST: + return new RsGxsNetTunnelTurtleSearchGroupRequestItem; + case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY: + return new RsGxsNetTunnelTurtleSearchGroupSummaryItem; + case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA: + return new RsGxsNetTunnelTurtleSearchGroupDataItem; + case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST: + return new RsGxsServiceTurtleSearchReqItem; + case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY: + return new RsGxsServiceTurtleSearchReplyItem; default: - GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl; - return NULL ; + RS_ERR("Unkonown item type: ", static_cast(item_subtype)); + return nullptr; } } }; @@ -993,7 +1141,9 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr return mTurtle->turtleSearch(mem,size,this) ; } -TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) +TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest( + const std::string& match_string, + RsNetworkExchangeService* client_service ) { GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl; @@ -1002,7 +1152,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma search_item.service = client_service->serviceType() ; uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ; - unsigned char *mem = (unsigned char*)rs_malloc(size) ; + uint8_t* mem = rs_malloc(size); if(mem == NULL) return 0 ; @@ -1013,151 +1163,304 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma return mTurtle->turtleSearch(mem,size,this) ; } -bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits) +std::error_condition RsGxsNetTunnelService::turtleSearchRequest( + rs_owner_ptr searchData, uint32_t dataSize, + RsServiceType serviceType, TurtleRequestId& requestId ) { - GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; + if(!searchData || !dataSize || serviceType == RsServiceType::NONE) + return std::errc::invalid_argument; - RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ; + RsGxsServiceTurtleSearchReqItem searchItem(serviceType); + searchItem.mSearchDataSize = dataSize; + searchItem.mSearchData = searchData; - RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast(item) ; + RsGxsNetTunnelSerializer tSerializer; - if(substring_sr != NULL) - { - GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl; + uint32_t size = tSerializer.size(&searchItem); + uint8_t* buf = rs_malloc(size); - max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ; + tSerializer.serialise(&searchItem, buf, &size); - std::list results ; - RsNetworkExchangeService *service = nullptr; + requestId = mTurtle->turtleSearch(buf, size, this); + if(!requestId) return std::errc::result_out_of_range; - { - RS_STACK_MUTEX(mGxsNetTunnelMtx); - - auto it = mSearchableServices.find(substring_sr->service) ; - - if(it != mSearchableServices.end()) - service = it->second; - } - - if(service != nullptr && service->search(substring_sr->substring_match,results)) - { - RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ; - - GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl; - - search_result_item.service = substring_sr->service ; - search_result_item.group_infos = results ; - - search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ; - search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ; - - delete item; - - if(search_result_data == NULL) - return false ; - - RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size); - - return true ; - } - } - - RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast(item) ; - - if(substring_gr != NULL) - { - RS_STACK_MUTEX(mGxsNetTunnelMtx); - auto it = mSearchableServices.find(substring_gr->service) ; - - max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ; - - unsigned char *encrypted_group_data = NULL ; - uint32_t encrypted_group_data_len = 0 ; - - if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len)) - { - RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ; - - search_result_item.service = substring_gr->service ; - search_result_item.encrypted_group_data = encrypted_group_data ; - search_result_item.encrypted_group_data_len = encrypted_group_data_len; - - search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ; - search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ; - - if(search_result_data == NULL) - return false ; - - RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size); - - delete item; - return true ; - } - } - - delete item; - return false ; + return std::error_condition(); } -void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len) +rs_view_ptr +RsGxsNetTunnelService::retrievieSearchableServiceLocking(uint16_t serviceType) { - RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len); + RS_STACK_MUTEX(mGxsNetTunnelMtx); + auto it = mSearchableServices.find(serviceType); + if( it != mSearchableServices.end()) return it->second; + return nullptr; +} - GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl; +bool RsGxsNetTunnelService::receiveSearchRequest( + uint8_t* search_request_data, uint32_t search_request_data_len, + uint8_t*& search_result_data, uint32_t& search_result_data_size, + uint32_t& max_allowed_hits ) +{ + /* Must return true only if there are matching results available, false in + * all other cases. @see RsTurleClientService */ - RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast(item) ; + RS_DBG3(""); - if(result_gs != NULL) + RsGxsNetTunnelSerializer tSerializer; + + std::unique_ptr item; + item.reset(tSerializer.deserialise( + search_request_data, &search_request_data_len )); + + if(!item) { - GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; - -#ifdef DEBUG_RSGXSNETTUNNEL - for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) - std::cerr << " group " << (*it).mGroupId << ": " << (*it).mGroupName << ", " << (*it).mNumberOfMessages << " messages, last is " << time(NULL)-(*it).mLastMessageTs << " secs ago." << std::endl; -#endif - - auto it = mSearchableServices.find(result_gs->service) ; - - if(it == mSearchableServices.end()) - { - GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl; - delete item; - return ; - } - - it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ; - - delete item; - return ; + RS_ERR( "Deserialization failed: ", + search_request_data, search_request_data_len, item.get() ); + print_stacktrace(); + return false; } - RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast(item) ; - - if(result_gd != NULL) - { - GXS_NET_TUNNEL_DEBUG() << " : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl; - - auto it = mSearchableServices.find(result_gd->service) ; - - if(it == mSearchableServices.end()) + switch(static_cast(item->PacketSubType())) + { + case RsGxsNetTunnelItemSubtypes::SEARCH_SUBSTRING: + { + if(!search_result_data) { - GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl; - delete item; - return ; + RS_ERR( "Got item with TURTLE_SEARCH_SUBSTRING without space for " + "results!" ); + print_stacktrace(); + break; } - it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ; + auto substring_sr = + dynamic_cast(item.get()); + if(!substring_sr) + { + RS_WARN( "Got item with TURTLE_SEARCH_SUBSTRING subtype: ", + item->PacketSubType(), " but casting failed!"); + break; + } - result_gd->encrypted_group_data = NULL ; // prevents deletion - delete item; + max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH; + std::list results; + auto tService = retrievieSearchableServiceLocking(substring_sr->service); + if(tService && tService->search(substring_sr->substring_match, results)) + { + RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item; + search_result_item.service = substring_sr->service; + search_result_item.group_infos = results; + search_result_data_size = tSerializer.size(&search_result_item); + search_result_data = rs_malloc(search_result_data_size); - return ; - } + tSerializer.serialise( + &search_result_item, search_result_data, + &search_result_data_size ); - GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl; + return true; + } + + break; + } + case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_REQUEST: + { + auto *substring_gr = + dynamic_cast(item.get()); + + if(!substring_gr) + { + RS_WARN( "Got item with TURTLE_SEARCH_GROUP_REQUEST subtype: ", + item->PacketSubType(), " but casting failed!" ); + break; + } + + max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA; + uint8_t* encrypted_group_data = nullptr; + uint32_t encrypted_group_data_len = 0; + + auto tService = retrievieSearchableServiceLocking(substring_gr->service); + if(tService && tService->search( + substring_gr->hashed_group_id, + encrypted_group_data, encrypted_group_data_len )) + { + RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item; + search_result_item.service = substring_gr->service; + search_result_item.encrypted_group_data = encrypted_group_data; + search_result_item.encrypted_group_data_len = encrypted_group_data_len; + search_result_data_size = tSerializer.size(&search_result_item); + search_result_data = rs_malloc(search_result_data_size); + + tSerializer.serialise( + &search_result_item, + search_result_data, &search_result_data_size ); + return true; + } + break; + } + case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REQUEST: + { + RS_DBG3("SERVICE_SEARCH_REQUEST"); + + auto searchItem = + static_cast(item.get()); + + max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH; + + uint16_t sType = static_cast(searchItem->mServiceType); + auto sService = retrievieSearchableServiceLocking(sType); + if(!sService) + { + RS_WARN("Got search request for non searchable service: ", sType); + break; + } + + RsGxsServiceTurtleSearchReplyItem replyItem(searchItem->mServiceType); + + auto errc = sService->handleDistantSearchRequest( + searchItem->mSearchData, searchItem->mSearchDataSize, + replyItem.mReplyData, replyItem.mReplyDataSize ); + if(errc) + { + // Some error has been reported by the searchable service + RS_WARN("searchable service: ", sType , " reported: ", errc); + break; + } + + if( (!replyItem.mReplyData && replyItem.mReplyDataSize) || + (replyItem.mReplyData && !replyItem.mReplyDataSize) ) + { + // Inconsistent behaviour from searcheable service + RS_ERR( "searchable service: ", sType , " silently failed handling " + "inconsistent result mReplyData: ", replyItem.mReplyData, + " mReplyDataSize: ", replyItem.mReplyDataSize ); + break; + } + + /* Our node have 0 matching results */ + if(!replyItem.mReplyData && !replyItem.mReplyDataSize) + break; + + search_result_data_size = tSerializer.size(&replyItem); + search_result_data = rs_malloc(search_result_data_size); + + tSerializer.serialise( + &replyItem, search_result_data, &search_result_data_size ); + + return true; + } + default: + RS_WARN("Got unknown item type: ", item->PacketSubType()); + break; + } + + return false; } -void RsGxsNetTunnelService::getStatistics(std::map& groups, std::map& virtual_peers, std::map &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const +void RsGxsNetTunnelService::receiveSearchResult( + TurtleSearchRequestId request_id, + uint8_t* search_result_data, uint32_t search_result_data_len ) +{ + RS_DBG3(request_id); + + std::unique_ptr item; + item.reset(RsGxsNetTunnelSerializer().deserialise( + search_result_data,&search_result_data_len )); + + auto castFailedWarn = [](const uint8_t subtype) + { + RS_WARN( "Got item with subtype: ", subtype, + " but cast failed!" ); + }; + + auto searchableServiceGet = [this](const auto pservice) + { + auto service = static_cast(pservice); + auto it = mSearchableServices.find(service); + if(it == mSearchableServices.end()) + { + RS_WARN( "got item for service ", service, + " which is not in the searchable services list." ); + return static_cast(nullptr); + } + + return it->second; + }; + + const auto tSubtype = item->PacketSubType(); + switch (static_cast(tSubtype)) + { + case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_SUMMARY: + { + auto result_gs = + dynamic_cast( + item.get() ); + + if(!result_gs) + { + castFailedWarn(tSubtype); + break; + } + + RS_DBG2( " got result is of type group summary result for service ", + result_gs->service ); + + auto service = searchableServiceGet(result_gs->service); + if(service) + service->receiveTurtleSearchResults( + request_id, result_gs->group_infos ); + return; + } + case RsGxsNetTunnelItemSubtypes::SEARCH_GROUP_DATA: + { + auto result_gd = + dynamic_cast(item.get()); + + if(!result_gd) + { + castFailedWarn(tSubtype); + break; + } + + RS_DBG2("got group data result for service: ", result_gd->service); + + auto service = searchableServiceGet(result_gd->service); + if(service) + service->receiveTurtleSearchResults( + request_id, + result_gd->encrypted_group_data, + result_gd->encrypted_group_data_len ); + + /* Ensure ownershipt is passed down preventing deletion */ + result_gd->encrypted_group_data = nullptr; + break; + } + case RsGxsNetTunnelItemSubtypes::SERVICE_SEARCH_REPLY: + { + auto searchReply = + static_cast(item.get()); + + auto service = searchableServiceGet(searchReply->mServiceType); + if(service) + service->receiveDistantSearchResult( + request_id, + searchReply->mReplyData, + searchReply->mReplyDataSize ); + + /* Ensure memory ownership is passed down preventing deletion */ + searchReply->mReplyData = nullptr; + break; + } + default: + RS_WARN("got item of unknown type: ", item->PacketSubType()); + break; + } +} + +void RsGxsNetTunnelService::getStatistics( + std::map& groups, + std::map& virtual_peers, + std::map& + turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const { groups = mGroups ; virtual_peers = mVirtualPeers ; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 7c18ab54c..c7f9ac1f3 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2018 by Cyril Soler * + * Copyright (C) 2018 Cyril Soler * + * Copyright (C) 2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -23,6 +25,7 @@ #pragma once #include +#include #include "turtle/p3turtle.h" #include "retroshare/rsgxsdistsync.h" @@ -100,7 +103,7 @@ // and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen. // -class RsGxsNetTunnelItem ; +struct RsGxsNetTunnelItem; class RsNetworkExchangeService ; class RsGxsNetTunnelService: @@ -108,8 +111,8 @@ class RsGxsNetTunnelService: public RsGxsDistSync { public: - RsGxsNetTunnelService() ; - virtual ~RsGxsNetTunnelService() ; + RsGxsNetTunnelService(); + ~RsGxsNetTunnelService() override; /*! * \brief registerSearchableService @@ -181,24 +184,38 @@ public: */ void dump() const; - /*! - * \brief connectToTurtleRouter - * Should be called after allocating a RsGxsNetTunnelService - * \param tr turtle router object - */ - virtual void connectToTurtleRouter(p3turtle *tr) ; + /*! + * Should be called after allocating a RsGxsNetTunnelService + * \param tr turtle router object + */ + void connectToTurtleRouter(p3turtle *tr) override; - TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ; - TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ; + /** Gxs services (channels, forums...) are supposed to use this to request + * searches on distant peers */ + std::error_condition turtleSearchRequest( + rs_owner_ptr searchData, uint32_t dataSize, + RsServiceType serviceType, TurtleRequestId& requestId ); - /*! - * \brief receiveSearchRequest - * See RsTurtleClientService::@ - */ - virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits); - virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len); + ///@see RsTurtleClientService + bool receiveSearchRequest( + unsigned char* search_request_data, + uint32_t search_request_data_len, + unsigned char*& search_result_data, + uint32_t& search_result_data_len, + uint32_t& max_allowed_hits ) override; - void threadTick() override; /// @see RsTickingThread + ///@see RsTurtleClientService + virtual void receiveSearchResult( + TurtleSearchRequestId request_id, + unsigned char* search_result_data, + uint32_t search_result_data_len ) override; + + TurtleRequestId turtleGroupRequest( + const RsGxsGroupId& group_id, + RsNetworkExchangeService* client_service ); + + /// @see RsTickingThread + void threadTick() override; // Overloads p3Config @@ -213,6 +230,11 @@ public: std::map& turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias) const; + RS_DEPRECATED + TurtleRequestId turtleSearchRequest( + const std::string& match_string, + RsNetworkExchangeService* client_service ); + protected: // interaction with turtle router @@ -233,6 +255,8 @@ private: void sendKeepAlivePackets() ; void handleIncoming(RsGxsNetTunnelItem *item) ; void flush_pending_items(); + rs_view_ptr retrievieSearchableServiceLocking( + uint16_t serviceType ); std::map mGroups ; // groups on the client and server side diff --git a/libretroshare/src/gxs/rsgxsutil.cc b/libretroshare/src/gxs/rsgxsutil.cc index f64fd236f..4a0f3e372 100644 --- a/libretroshare/src/gxs/rsgxsutil.cc +++ b/libretroshare/src/gxs/rsgxsutil.cc @@ -3,8 +3,8 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2013-2013 by Christopher Evi-Parker * - * Copyright (C) 2018 Gioacchino Mazzurco * + * Copyright (C) 2013 Christopher Evi-Parker * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -29,12 +29,6 @@ #include "pqi/pqihash.h" #include "gxs/rsgixs.h" -#ifdef RS_DEEP_CHANNEL_INDEX -# include "deep_search/channelsindex.hpp" -# include "services/p3gxschannels.h" -# include "rsitems/rsgxschannelitems.h" -#endif - // The goals of this set of methods is to check GXS messages and groups for consistency, mostly // re-ferifying signatures and hashes, to make sure that the data hasn't been tempered. This shouldn't // happen anyway, but we still conduct these test as an extra safety measure. @@ -197,9 +191,8 @@ bool RsGxsCleanUp::clean(RsGxsGroupId& next_group_to_check,std::vector& grpsToDel, GxsMsgReq& msgsToDel) +bool RsGxsSinglePassIntegrityCheck::check( + uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds, + std::vector& grpsToDel, GxsMsgReq& msgsToDel ) { #ifdef DEBUG_GXSUTIL GXSUTIL_DEBUG() << "Parsing all groups and messages data in service " << std::hex << mds->serviceType() << " for integrity check. Could take a while..." << std::endl; #endif -#ifdef RS_DEEP_CHANNEL_INDEX - bool isGxsChannels = mGenExchangeClient->serviceType() == RS_SERVICE_GXS_TYPE_CHANNELS; - std::set indexedGroups; -#endif // first take out all the groups std::map grp; @@ -393,55 +380,14 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, } else msgIds.erase(msgIds.find(grp->grpId)); // could not get them, so group is removed from list. - -#ifdef RS_DEEP_CHANNEL_INDEX - // This should be moved to p3gxschannels. It is really not the place for this here! - - if( isGxsChannels - && grp->metaData->mCircleType == GXS_CIRCLE_TYPE_PUBLIC - && grp->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) - { - RsGxsGrpMetaData meta; - meta.deserialise(grp->meta.bin_data, grp->meta.bin_len); - - uint32_t blz = grp->grp.bin_len; - RsItem* rIt = mSerializer.deserialise(grp->grp.bin_data, - &blz); - - if( RsGxsChannelGroupItem* cgIt = - dynamic_cast(rIt) ) - { - RsGxsChannelGroup cg; - cgIt->toChannelGroup(cg, false); - cg.mMeta = meta; - - indexedGroups.insert(grp->grpId); - DeepChannelsIndex::indexChannelGroup(cg); - } - else - { - std::cerr << __PRETTY_FUNCTION__ << " Group: " - << meta.mGroupId.toStdString() << " " - << meta.mGroupName - << " doesn't seems a channel, please " - << "report to developers" - << std::endl; - print_stacktrace(); - } - - delete rIt; - } -#endif // def RS_DEEP_CHANNEL_INDEX - } - else - { - std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting group " << grp->grpId << " with wrong hash or null/corrupted meta data. meta=" << grp->metaData << std::endl; - grpsToDel.push_back(grp->grpId); -#ifdef RS_DEEP_CHANNEL_INDEX - if(isGxsChannels) - DeepChannelsIndex::removeChannelFromIndex(grp->grpId); -#endif // def RS_DEEP_CHANNEL_INDEX } + else + { + RS_WARN( "deleting group ", grp->grpId, + " with wrong hash or null/corrupted meta data. meta=", + grp->metaData ); + grpsToDel.push_back(grp->grpId); + } delete grp; } @@ -469,15 +415,9 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, if(nxsMsg) nxsMsgS.insert(nxsMsg->msgId); - for (auto& msgId:msgIdV) - if(nxsMsgS.find(msgId) == nxsMsgS.end()) - { - msgsToDel[grpId].insert(msgId); -#ifdef RS_DEEP_CHANNEL_INDEX - if(isGxsChannels) - DeepChannelsIndex::removeChannelPostFromIndex(grpId, msgId); -#endif // def RS_DEEP_CHANNEL_INDEX - } + for (auto& msgId:msgIdV) + if(nxsMsgS.find(msgId) == nxsMsgS.end()) + msgsToDel[grpId].insert(msgId); } for(auto mit = msgs.begin(); mit != msgs.end(); ++mit) @@ -495,54 +435,11 @@ bool RsGxsSinglePassIntegrityCheck::check(uint16_t service_type, RsGixs *mgixs, if(msg->metaData == NULL || currHash != msg->metaData->mHash) { - std::cerr << __PRETTY_FUNCTION__ <<" (EE) deleting message " << msg->msgId << " in group " << msg->grpId << " with wrong hash or null/corrupted meta data. meta=" << (void*)msg->metaData << std::endl; + RS_WARN( "deleting message ", msg->msgId, " in group ", + msg->grpId, + " with wrong hash or null/corrupted meta data. meta=", + static_cast(msg->metaData) ); msgsToDel[msg->grpId].insert(msg->msgId); -#ifdef RS_DEEP_CHANNEL_INDEX - if(isGxsChannels) - DeepChannelsIndex::removeChannelPostFromIndex( - msg->grpId, msg->msgId ); -#endif // def RS_DEEP_CHANNEL_INDEX - } - else if (subscribed_groups.count(msg->metaData->mGroupId)) - { -#ifdef RS_DEEP_CHANNEL_INDEX - // This should be moved to p3gxschannels. It is really not the place for this here! - - if( isGxsChannels && indexedGroups.count(msg->metaData->mGroupId) ) - { - RsGxsMsgMetaData meta; - meta.deserialise(msg->meta.bin_data, &msg->meta.bin_len); - - uint32_t blz = msg->msg.bin_len; - RsItem* rIt = mSerializer.deserialise(msg->msg.bin_data, - &blz); - - if( RsGxsChannelPostItem* cgIt = - dynamic_cast(rIt) ) - { - RsGxsChannelPost cg; - cgIt->toChannelPost(cg, false); - cg.mMeta = meta; - - DeepChannelsIndex::indexChannelPost(cg); - } - else if(dynamic_cast(rIt)) {} - else if(dynamic_cast(rIt)) {} - else - { - std::cerr << __PRETTY_FUNCTION__ << " Message: " - << meta.mMsgId.toStdString() - << " in group: " - << meta.mGroupId.toStdString() << " " - << " doesn't seems a channel post, please " - << "report to developers" - << std::endl; - print_stacktrace(); - } - - delete rIt; - } -#endif // def RS_DEEP_CHANNEL_INDEX } delete msg; diff --git a/libretroshare/src/gxs/rsgxsutil.h b/libretroshare/src/gxs/rsgxsutil.h index 34f1b2baa..21fd6bb06 100644 --- a/libretroshare/src/gxs/rsgxsutil.h +++ b/libretroshare/src/gxs/rsgxsutil.h @@ -3,8 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2013-2013 by Christopher Evi-Parker * - * Copyright (C) 2018 Gioacchino Mazzurco * + * Copyright (C) 2013 Christopher Evi-Parker * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -162,18 +163,9 @@ class RsGxsIntegrityCheck : public RsThread enum CheckState { CheckStart, CheckChecking }; public: - - - /*! - * - * @param dataService - * @param mGroupTS - * @param chunkSize - * @param sleepPeriod - */ - RsGxsIntegrityCheck(RsGeneralDataService* const dataService, - RsGenExchange *genex, RsSerialType&, - RsGixs *gixs); + RsGxsIntegrityCheck( RsGeneralDataService* const dataService, + RsGenExchange* genex, RsSerialType&, + RsGixs* gixs ); static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds); bool isDone(); @@ -201,19 +193,9 @@ private: class RsGxsSinglePassIntegrityCheck { public: - - /*! - * - * @param dataService - * @param mGroupTS - * @param chunkSize - * @param sleepPeriod - */ - static bool check(uint16_t service_type, RsGixs *mgixs, RsGeneralDataService *mds -#ifdef RS_DEEP_CHANNEL_INDEX - , RsGenExchange* mGenExchangeClient, RsSerialType& mSerializer -#endif - , std::vector& grpsToDel, GxsMsgReq& msgsToDel); + static bool check( + uint16_t service_type, RsGixs* mgixs, RsGeneralDataService* mds, + std::vector& grpsToDel, GxsMsgReq& msgsToDel ); }; class GroupUpdate diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index fb6238f92..2f9bb25a6 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -3,8 +3,10 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2011-2011 by Robert Fernie * - * Copyright 2011-2011 by Christopher Evi-Parker * + * Copyright (C) 2011 Robert Fernie * + * Copyright (C) 2011 Christopher Evi-Parker * + * Copyright (C) 2019-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -20,17 +22,15 @@ * along with this program. If not, see . * * * *******************************************************************************/ - -#ifndef RSGNP_H -#define RSGNP_H +#pragma once #include #include -#include "util/rstime.h" #include #include #include +#include "util/rstime.h" #include "services/p3service.h" #include "retroshare/rsreputations.h" #include "retroshare/rsidentity.h" @@ -61,9 +61,8 @@ class RsNetworkExchangeService { public: - - RsNetworkExchangeService(){ return;} - virtual ~RsNetworkExchangeService() {} + RsNetworkExchangeService() = default; + virtual ~RsNetworkExchangeService() = default; virtual uint16_t serviceType() const =0; /*! @@ -85,9 +84,24 @@ public: virtual bool msgAutoSync() const =0; virtual bool grpAutoSync() const =0; - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// - /// DISTANT SEARCH FUNCTIONS /// - /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////// + /// DISTANT SEARCH FUNCTIONS /// + //////////////////////////////////////////////////////////////////////////// + + /// Trigger remote generic GXS service search + virtual std::error_condition distantSearchRequest( + rs_owner_ptr searchData, uint32_t dataSize, + RsServiceType serviceType, TurtleRequestId& requestId ) = 0; + + /// Handle remote generic GXS services search requests to specific service + virtual std::error_condition handleDistantSearchRequest( + rs_view_ptr requestData, uint32_t requestSize, + rs_owner_ptr& resultData, uint32_t& resultSize ) = 0; + + /// Receive remote generic GXS services search result + virtual std::error_condition receiveDistantSearchResult( + const TurtleRequestId requestId, + rs_owner_ptr& resultData, uint32_t& resultSize ) = 0; /*! * \brief turtleGroupRequest @@ -115,13 +129,17 @@ public: */ virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos)=0; - /*! - * \brief receiveTurtleSearchResults - * Called by turtle (through RsGxsNetTunnel) when new data is received - * \param req Turtle search request ID associated with this result - * \param encrypted_group_data Group data - */ - virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0; + /*! + * \brief receiveTurtleSearchResults + * Called by turtle (through RsGxsNetTunnel) when new data is received + * \param req Turtle search request ID associated with this result + * \param encrypted_group_data Group data + */ + RS_DEPRECATED_FOR("receiveDistantSearchResult") + virtual void receiveTurtleSearchResults( + TurtleRequestId req, + rs_owner_ptr encrypted_group_data, + uint32_t encrypted_group_data_len ) = 0; /*! * \brief retrieveTurtleSearchResults @@ -141,7 +159,9 @@ public: virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0; virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSearchResults&)=0; - virtual bool search(const std::string& substring,std::list& group_infos) =0; + RS_DEPRECATED_FOR("handleDistantSearchRequest and distantSearchRequest") + virtual bool search(const std::string& substring,std::list& group_infos) =0; + virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0; /*! @@ -306,5 +326,3 @@ public: } } }; - -#endif // RSGNP_H diff --git a/libretroshare/src/util/rsmemory.cc b/libretroshare/src/gxs/rsnxsobserver.cpp similarity index 55% rename from libretroshare/src/util/rsmemory.cc rename to libretroshare/src/gxs/rsnxsobserver.cpp index 2b162c77d..2b5ab0d8a 100644 --- a/libretroshare/src/util/rsmemory.cc +++ b/libretroshare/src/gxs/rsnxsobserver.cpp @@ -1,53 +1,36 @@ -/******************************************************************************* - * libretroshare/src/util: rsmemory.cc * - * * - * libretroshare: retroshare core library * - * * - * Copyright 2012-2012 by Cyril Soler * - * * - * This program is free software: you can redistribute it and/or modify * - * it under the terms of the GNU Lesser General Public License as * - * published by the Free Software Foundation, either version 3 of the * - * License, or (at your option) any later version. * - * * - * This program is distributed in the hope that it will be useful, * - * but WITHOUT ANY WARRANTY; without even the implied warranty of * - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * - * GNU Lesser General Public License for more details. * - * * - * You should have received a copy of the GNU Lesser General Public License * - * along with this program. If not, see . * - * * - *******************************************************************************/ -#include "util/rsmemory.h" - -void *rs_malloc(size_t size) -{ - static const size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024 ; // 1Gb should be enough for everything! - - if(size == 0) - { - std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl; - print_stacktrace() ; - return NULL ; - } - - if(size > SAFE_MEMALLOC_THRESHOLD) - { - std::cerr << "(EE) Memory allocation error. A chunk of size larger than " << SAFE_MEMALLOC_THRESHOLD << " was requested. Callstack:" << std::endl; - print_stacktrace() ; - return NULL ; - } - - void *mem = malloc(size) ; - - if(mem == NULL) - { - std::cerr << "(EE) Memory allocation error for a chunk of " << size << " bytes. Callstack:" << std::endl; - print_stacktrace() ; - return NULL ; - } - - return mem ; -} - +/******************************************************************************* + * RetroShare General eXchange System * + * * + * Copyright (C) 2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "gxs/rsnxsobserver.h" + +const RsNxsObserverErrorCategory RsNxsObserverErrorCategory::instance; + +std::error_condition RsNxsObserverErrorCategory::default_error_condition(int ev) +const noexcept +{ + switch(static_cast(ev)) + { + case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER: + return std::errc::operation_not_supported; + default: + return std::error_condition(ev, *this); + } +} diff --git a/libretroshare/src/gxs/rsnxsobserver.h b/libretroshare/src/gxs/rsnxsobserver.h index 2da5067a4..5725df0f0 100644 --- a/libretroshare/src/gxs/rsnxsobserver.h +++ b/libretroshare/src/gxs/rsnxsobserver.h @@ -3,7 +3,10 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2011-2012 by Robert Fernie, Evi-Parker Christopher * + * Copyright (C) 2011-2012 Robert Fernie * + * Copyright (C) 2011-2012 Christopher Evi-Parker * + * Copyright (C) 2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -19,21 +22,61 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef RSNXSOBSERVER_H -#define RSNXSOBSERVER_H +#pragma once -#include +#include +#include + +#include "retroshare/rsgxsiface.h" #include "rsitems/rsnxsitems.h" +#include "util/rsdebug.h" -typedef uint32_t TurtleRequestId ; +typedef uint32_t TurtleRequestId; + +enum class RsNxsObserverErrorNum : int32_t +{ + NOT_OVERRIDDEN_BY_OBSERVER = 2004, +}; + +struct RsNxsObserverErrorCategory: std::error_category +{ + const char* name() const noexcept override + { return "RetroShare NXS Observer"; } + + std::string message(int ev) const override + { + switch (static_cast(ev)) + { + case RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER: + return "Method not overridden by observer"; + default: + return rsErrorNotInCategory(ev, name()); + } + } + + std::error_condition default_error_condition(int ev) const noexcept override; + + const static RsNxsObserverErrorCategory instance; +}; + + +namespace std +{ +/** Register RsNxsObserverErrorNum as an error condition enum, must be in std + * namespace */ +template<> struct is_error_condition_enum : true_type {}; +} + +/** Provide RsJsonApiErrorNum conversion to std::error_condition, must be in + * same namespace of RsJsonApiErrorNum */ +inline std::error_condition make_error_condition(RsNxsObserverErrorNum e) noexcept +{ + return std::error_condition( + static_cast(e), RsNxsObserverErrorCategory::instance ); +}; class RsNxsObserver { -public: - - RsNxsObserver() {} - - public: /*! @@ -56,6 +99,42 @@ public: std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl; } + /** If advanced search functionalities like deep indexing are supported at + * observer/service level, this method should be overridden to handle search + * requests there. + * @param[in] requestData search query + * @param[in] requestSize search query size + * @param[out] resultData results data + * @param[out] resultSize results data size + * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is + * returned to inform the caller that this method was not overridden by the + * observer so do not use it for other meanings. */ + virtual std::error_condition handleDistantSearchRequest( + rs_view_ptr requestData, uint32_t requestSize, + rs_owner_ptr& resultData, uint32_t& resultSize ) + { + (void) requestData; (void) requestSize; + (void) resultData; (void) resultSize; + return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER; + } + + /** If advanced search functionalities like deep indexing are supported at + * observer/service level, this method should be overridden to handle search + * results there. + * @param[in] requestId search query id + * @param[out] resultData results data + * @param[out] resultSize results data size + * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is + * returned to inform the caller that this method was not overridden by the + * observer so do not use it for other meanings. */ + virtual std::error_condition receiveDistantSearchResult( + const TurtleRequestId requestId, + rs_owner_ptr& resultData, uint32_t& resultSize ) + { + (void) requestId; (void) resultData; (void) resultSize; + return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER; + } + /*! * @param grpId group id */ @@ -70,6 +149,7 @@ public: * @param grpId group id */ virtual void notifyChangedGroupStats(const RsGxsGroupId &grpId) = 0; -}; -#endif // RSNXSOBSERVER_H + RsNxsObserver() = default; + virtual ~RsNxsObserver() = default; +}; diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index c654b2e61..d410e8a52 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -633,7 +633,6 @@ SOURCES += util/folderiterator.cc \ util/rsexpr.cc \ util/smallobject.cc \ util/rsdir.cc \ - util/rsmemory.cc \ util/rsdiscspace.cc \ util/rsnet.cc \ util/rsnet_ss.cc \ @@ -712,7 +711,8 @@ SOURCES += rsitems/rsnxsitems.cc \ gxs/gxstokenqueue.cc \ gxs/rsgxsnetutils.cc \ gxs/rsgxsutil.cc \ - gxs/rsgxsrequesttypes.cc + gxs/rsgxsrequesttypes.cc \ + gxs/rsnxsobserver.cpp # gxs tunnels HEADERS += gxstunnel/p3gxstunnel.h \ @@ -936,6 +936,14 @@ rs_jsonapi { SOURCES += jsonapi/jsonapi.cpp } +rs_deep_forums_index { + HEADERS *= deep_search/commonutils.hpp + SOURCES *= deep_search/commonutils.cpp + + HEADERS += deep_search/forumsindex.hpp + SOURCES += deep_search/forumsindex.cpp +} + rs_deep_channels_index { HEADERS *= deep_search/commonutils.hpp SOURCES *= deep_search/commonutils.cpp diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index 421417bdc..bd09950f4 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -4,8 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012 Robert Fernie * - * Copyright (C) 2018-2020 Gioacchino Mazzurco * - * Copyright (C) 2019-2020 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -46,6 +46,11 @@ class RsGxsChannels; extern RsGxsChannels* rsGxsChannels; +/* TODO: At this abstraction level that turtle is used for distant searches + * should be an hidden implementation detail. As of today a bit of those + * implementation details leaks around because we use TurtleRequestId in this + * interface */ + struct RsGxsChannelGroup : RsSerializable, RsGxsGenericGroupData { RsGxsChannelGroup() : mAutoDownload(false) {} @@ -108,29 +113,38 @@ struct RsGxsChannelPost : RsSerializable, RsGxsGenericMsgData enum class RsChannelEventCode: uint8_t { UNKNOWN = 0x00, - NEW_CHANNEL = 0x01, // emitted when new channel is received - UPDATED_CHANNEL = 0x02, // emitted when existing channel is updated - NEW_MESSAGE = 0x03, // new message reeived in a particular channel (group and msg id) - UPDATED_MESSAGE = 0x04, // existing message has been updated in a particular channel - RECEIVED_PUBLISH_KEY = 0x05, // publish key for this channel has been received - SUBSCRIBE_STATUS_CHANGED = 0x06, // subscription for channel mChannelGroupId changed. - READ_STATUS_CHANGED = 0x07, // existing message has been read or set to unread - RECEIVED_DISTANT_SEARCH_RESULT = 0x08, // result for the given group id available for the given turtle request id - STATISTICS_CHANGED = 0x09, // stats (nb of supplier friends, how many msgs they have etc) has changed - SYNC_PARAMETERS_UPDATED = 0x0a, // sync and storage times have changed - NEW_COMMENT = 0x0b, // new comment arrived/published. mChannelThreadId gives the ID of the commented message - NEW_VOTE = 0x0c, // new vote arrived/published. mChannelThreadId gives the ID of the votes message comment - DELETED_CHANNEL = 0x0d, // channel was deleted by auto-cleaning system + NEW_CHANNEL = 0x01, /// emitted when new channel is received + UPDATED_CHANNEL = 0x02, /// emitted when existing channel is updated + NEW_MESSAGE = 0x03, /// new message reeived in a particular channel (group and msg id) + UPDATED_MESSAGE = 0x04, /// existing message has been updated in a particular channel + RECEIVED_PUBLISH_KEY = 0x05, /// publish key for this channel has been received + SUBSCRIBE_STATUS_CHANGED = 0x06, /// subscription for channel mChannelGroupId changed. + READ_STATUS_CHANGED = 0x07, /// existing message has been read or set to unread + + /** Result for the given group id available for the given turtle request id + * @deprecated kept for retrocompatibility with old search system new code + * should use @see DISTANT_SEARCH_RESULT instead */ + RECEIVED_TURTLE_SEARCH_RESULT = 0x08, + + STATISTICS_CHANGED = 0x09, /// stats (nb of supplier friends, how many msgs they have etc) has changed + SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed + NEW_COMMENT = 0x0b, /// new comment arrived/published. mChannelThreadId gives the ID of the commented message + NEW_VOTE = 0x0c, /// new vote arrived/published. mChannelThreadId gives the ID of the votes message comment + DELETED_CHANNEL = 0x0d, /// channel was deleted by auto-cleaning system + DELETED_POST = 0x0e, /// Post deleted (usually by cleaning) + DISTANT_SEARCH_RESULT = 0x0f /// Distant search result received }; struct RsGxsChannelEvent: RsEvent { - RsGxsChannelEvent(): RsEvent(RsEventType::GXS_CHANNELS), mChannelEventCode(RsChannelEventCode::UNKNOWN) {} + RsGxsChannelEvent(): + RsEvent(RsEventType::GXS_CHANNELS), + mChannelEventCode(RsChannelEventCode::UNKNOWN) {} RsChannelEventCode mChannelEventCode; RsGxsGroupId mChannelGroupId; RsGxsMessageId mChannelMsgId; - RsGxsMessageId mChannelThreadId; + RsGxsMessageId mChannelThreadId; ///* @see RsEvent @see RsSerializable void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override @@ -143,13 +157,14 @@ struct RsGxsChannelEvent: RsEvent } }; -// This event is used to factor multiple search results notifications in a single event. - -struct RsGxsChannelSearchResultEvent: RsEvent +/** This event is used to factor multiple search results notifications in a + * single event.*/ +struct RS_DEPRECATED_FOR(RsGxsChannelDistantSearchResultEvent) +RsGxsChannelSearchResultEvent: RsEvent { RsGxsChannelSearchResultEvent(): RsEvent(RsEventType::GXS_CHANNELS), - mChannelEventCode(RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT) {} + mChannelEventCode(RsChannelEventCode::RECEIVED_TURTLE_SEARCH_RESULT) {} RsChannelEventCode mChannelEventCode; std::map > mSearchResultsMap; @@ -164,6 +179,29 @@ struct RsGxsChannelSearchResultEvent: RsEvent } }; +/** This event is fired once distant search results are received */ +struct RsGxsChannelDistantSearchResultEvent: RsEvent +{ + RsGxsChannelDistantSearchResultEvent(): + RsEvent(RsEventType::GXS_CHANNELS), + mChannelEventCode(RsChannelEventCode::DISTANT_SEARCH_RESULT) {} + + RsChannelEventCode mChannelEventCode; + TurtleRequestId mSearchId; + std::vector mSearchResults; + + ///* @see RsEvent @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RsEvent::serial_process(j, ctx); + + RS_SERIAL_PROCESS(mChannelEventCode); + RS_SERIAL_PROCESS(mSearchId); + RS_SERIAL_PROCESS(mSearchResults); + } +}; + class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService { public: @@ -386,11 +424,15 @@ public: * @brief Get channel content summaries * @jsonapi{development} * @param[in] channelId id of the channel of which the content is requested + * @param[in] contentIds ids of requested contents, if empty summaries of + * all messages are reqeusted * @param[out] summaries storage for summaries * @return false if something failed, true otherwhise */ - virtual bool getContentSummaries( const RsGxsGroupId& channelId, - std::vector& summaries ) = 0; + virtual std::error_condition getContentSummaries( + const RsGxsGroupId& channelId, + const std::set& contentIds, + std::vector& summaries ) = 0; /** * @brief Toggle post read status. Blocking API. @@ -422,22 +464,23 @@ public: virtual bool subscribeToChannel( const RsGxsGroupId& channelId, bool subscribe ) = 0; - /** - * \brief Retrieve statistics about the channel service + /** + * @brief Retrieve statistics about the channel service * @jsonapi{development} - * \param[out] stat Statistics structure - * \return - */ - virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0; + * @param[out] stat storage for statistics + * @return true on success false otherwise + */ + virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0; - /** - * \brief Retrieve statistics about the given channel + /** + * @brief Retrieve statistics about the given channel * @jsonapi{development} - * \param[in] channelId Id of the channel group - * \param[out] stat Statistics structure - * \return - */ - virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0; + * @param[in] channelId Id of the channel group + * @param[out] stat storage for statistics + * @return true on success false otherwise + */ + virtual bool getChannelStatistics( + const RsGxsGroupId& channelId, GxsGroupStatistic& stat ) =0; /// default base URL used for channels links @see exportChannelLink static const std::string DEFAULT_CHANNEL_BASE_URL; @@ -496,14 +539,18 @@ public: std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0; /** - * @brief Search the turtle reachable network for matching channels + * @brief Search the whole reachable network for matching channels and + * contents * @jsonapi{development} - * An @see RsGxsChannelSearchResultEvent is emitted when matching channels - * arrives from the network + * An @see RsGxsChannelSearchResultEvent is emitted when matching results + * arrives from the network * @param[in] matchString string to search into the channels - * @return search id + * @param[out] searchId storage for search id, useful to track search events + * and retrieve search results + * @return success or error details */ - virtual TurtleRequestId turtleSearchRequest(const std::string& matchString)=0; + virtual std::error_condition distantSearchRequest( + const std::string& matchString, TurtleRequestId& searchId ) = 0; /** * @brief Retrieve available search results @@ -533,16 +580,18 @@ public: * @param[out] distantGroup storage for group data * @return false on error, true otherwise */ - virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0; + virtual bool getDistantSearchResultGroupData( + const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0; - /** - * @brief getDistantSearchStatus - * Returns the status of ongoing search: unknown (probably not even searched), known as a search result, - * data request ongoing and data available - */ - virtual DistantSearchGroupStatus getDistantSearchStatus(const RsGxsGroupId& group_id) =0; + /** + * @brief Get the status of ongoing search + * @return unknown (probably not even searched), known as a search result, + * data request ongoing and data available + */ + virtual DistantSearchGroupStatus getDistantSearchStatus( + const RsGxsGroupId& group_id ) =0; - /** + /** * @brief Clear accumulated search results * @jsonapi{development} * @param[in] reqId search id diff --git a/libretroshare/src/retroshare/rsgxsiface.h b/libretroshare/src/retroshare/rsgxsiface.h index ce64bf4ed..eca8b009e 100644 --- a/libretroshare/src/retroshare/rsgxsiface.h +++ b/libretroshare/src/retroshare/rsgxsiface.h @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012 Christopher Evi-Parker * - * Copyright (C) 2019 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -31,13 +32,63 @@ #include "rsitems/rsserviceids.h" #include "retroshare/rsevents.h" +/*! + * This structure is used to transport GXS search results. + * It contains the group information as well as a context string to tell where + * the information was found. + * Keep it small as to make search responses as light as possible. + * It differs from RsGxsGroupSearchResults because it supports also results from + * message matches not just groups. + */ +struct RsGxsSearchResult : RsSerializable +{ + RsGxsSearchResult(): mPublishTs(0) {} + + /** Id of the group which match*/ + RsGxsGroupId mGroupId; + + /** Title of the group which match */ + std::string mGroupName; + + /** Optional message id if the search match is against a message */ + RsGxsMessageId mMsgId; + + /** Optional message title if the search match is against a message */ + std::string mMsgName; + + /** Author id of the element which matched (group or message) */ + RsGxsId mAuthorId; + + /** Publish timestamp of the element which matched (group or message) */ + rstime_t mPublishTs; + + /** A snippet of content around the exact match */ + std::string mSearchContext; + + /// @see RsSerializable::serial_process + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) + { + RS_SERIAL_PROCESS(mGroupId); + RS_SERIAL_PROCESS(mGroupName); + RS_SERIAL_PROCESS(mMsgId); + RS_SERIAL_PROCESS(mMsgName); + RS_SERIAL_PROCESS(mAuthorId); + RS_SERIAL_PROCESS(mPublishTs); + RS_SERIAL_PROCESS(mSearchContext); + } + + virtual ~RsGxsSearchResult() = default; +}; + /*! * This structure is used to transport group summary information when a GXS * service is searched. It contains the group information as well as a context * string to tell where the information was found. It is more compact than a * GroupMeta object, so as to make search responses as light as possible. */ -struct RsGxsGroupSummary : RsSerializable +struct RS_DEPRECATED_FOR(RsGxsSearchResult) +RsGxsGroupSummary : RsSerializable { RsGxsGroupSummary() : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), @@ -78,8 +129,12 @@ struct RsGxsGroupSummary : RsSerializable * strings to tell where the information was found. It is more compact than a * GroupMeta object, so as to make search responses as light as possible. */ -struct RsGxsGroupSearchResults : RsSerializable +struct RS_DEPRECATED_FOR(RsGxsSearchResult) +RsGxsGroupSearchResults : RsSerializable { + /* TODO: This seems exactly the same as RsGxsGroupSummary + mSearchContexts + * do we really need both? */ + RsGxsGroupSearchResults() : mPublishTs(0), mNumberOfMessages(0),mLastMessageTs(0), mSignFlags(0),mPopularity(0) {} @@ -113,6 +168,7 @@ struct RsGxsGroupSearchResults : RsSerializable virtual ~RsGxsGroupSearchResults() = default; }; + /*! * Stores ids of changed gxs groups and messages. * It is used to notify about GXS changes. diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index 0f78b3738..d6232a0d6 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -63,7 +63,7 @@ struct RsGroupMetaData : RsSerializable mCircleType(0x0001), mAuthenFlags(0), mSubscribeFlags(0), mPop(0), mVisibleMsgCount(0), mLastPost(0), mGroupStatus(0) {} - virtual ~RsGroupMetaData() {} + virtual ~RsGroupMetaData() = default; void operator =(const RsGxsGrpMetaData& rGxsMeta); RsGroupMetaData(const RsGxsGrpMetaData& rGxsMeta) { operator=(rGxsMeta); } diff --git a/libretroshare/src/rsitems/rsgxschannelitems.h b/libretroshare/src/rsitems/rsgxschannelitems.h index fa52c8cfc..598400966 100644 --- a/libretroshare/src/rsitems/rsgxschannelitems.h +++ b/libretroshare/src/rsitems/rsgxschannelitems.h @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2012-2012 by Robert Fernie * + * Copyright (C) 2012 Robert Fernie * + * Copyright (C) 2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -19,8 +21,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef RS_GXS_CHANNEL_ITEMS_H -#define RS_GXS_CHANNEL_ITEMS_H +#pragma once #include @@ -30,14 +31,23 @@ #include "serialiser/rstlvfileitem.h" #include "serialiser/rstlvimage.h" - +#include "serialiser/rsserializable.h" #include "retroshare/rsgxschannels.h" - #include "serialiser/rsserializer.h" - #include "util/rsdir.h" +enum class RsGxsChannelItems : uint8_t +{ + GROUP_ITEM = 0x02, + POST_ITEM = 0x03, + SEARCH_REQUEST = 0x04, + SEARCH_REPLY = 0x05, +}; + +RS_DEPRECATED_FOR(RsGxsChannelItems) const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_GROUP_ITEM = 0x02; + +RS_DEPRECATED_FOR(RsGxsChannelItems) const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_POST_ITEM = 0x03; class RsGxsChannelGroupItem : public RsGxsGrpItem @@ -79,6 +89,47 @@ public: RsTlvImage mThumbnail; }; +struct RsGxsChannelsSearchRequest : RsSerializable +{ + RsGxsChannelsSearchRequest() : mType(RsGxsChannelItems::SEARCH_REQUEST) {} + + /// Just for easier back and forward compatibility + RsGxsChannelItems mType; + + /// Store search match string + std::string mQuery; + + /// @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RS_SERIAL_PROCESS(mType); + RS_SERIAL_PROCESS(mQuery); + } + + ~RsGxsChannelsSearchRequest() override = default; +}; + +struct RsGxsChannelsSearchReply : RsSerializable +{ + RsGxsChannelsSearchReply() : mType(RsGxsChannelItems::SEARCH_REPLY) {} + + /// Just for easier back and forward compatibility + RsGxsChannelItems mType; + + /// Results storage + std::vector mResults; + + /// @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RS_SERIAL_PROCESS(mType); + RS_SERIAL_PROCESS(mResults); + } + + ~RsGxsChannelsSearchReply() override = default; +}; class RsGxsChannelSerialiser : public RsGxsCommentSerialiser { @@ -89,5 +140,3 @@ public: virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ; }; - -#endif /* RS_GXS_CHANNEL_ITEMS_H */ diff --git a/libretroshare/src/rsitems/rsitem.h b/libretroshare/src/rsitems/rsitem.h index 1c8ea9789..3ef44f859 100644 --- a/libretroshare/src/rsitems/rsitem.h +++ b/libretroshare/src/rsitems/rsitem.h @@ -86,9 +86,8 @@ struct RsItem : RsMemoryManagement::SmallObject, RsSerializable virtual void serial_process(RsGenericSerializer::SerializeJob, RsGenericSerializer::SerializeContext&)// = 0; { - std::cerr << "(EE) RsItem::serial_process(...) called by an item using" - << "new serialization classes, but not derived! Class is " - << typeid(*this).name() << std::endl; + RS_ERR( "called by an item using new serialization system without " + "overriding Class is: ", typeid(*this).name() ); print_stacktrace(); } diff --git a/libretroshare/src/rsitems/rsserviceids.h b/libretroshare/src/rsitems/rsserviceids.h index ead349e32..eb08db07c 100644 --- a/libretroshare/src/rsitems/rsserviceids.h +++ b/libretroshare/src/rsitems/rsserviceids.h @@ -28,7 +28,9 @@ enum class RsServiceType : uint16_t { - NONE = 0, /// To detect non-initialized reads + /// To detect non-initialized items + NONE = 0, + GOSSIP_DISCOVERY = 0x0011, CHAT = 0x0012, MSG = 0x0013, @@ -46,7 +48,10 @@ enum class RsServiceType : uint16_t GWEMAIL_MAIL = 0x0025, SERVICE_CONTROL = 0x0026, DISTANT_CHAT = 0x0027, + + /// For GXS identity authenticated tunnels, do not confuse with @GXS_DISTANT GXS_TUNNEL = 0x0028, + BANLIST = 0x0101, STATUS = 0x0102, NXS = 0x0200, @@ -58,6 +63,7 @@ enum class RsServiceType : uint16_t POSTED = 0x0216, CHANNELS = 0x0217, GXSCIRCLE = 0x0218, + /// not gxs, but used with identities. REPUTATION = 0x0219, GXS_RECOGN = 0x0220, @@ -68,13 +74,13 @@ enum class RsServiceType : uint16_t CHANNELS_CONFIG = 0x0317, RTT = 0x1011, /// Round Trip Time - - /***************** IDS ALLOCATED FOR PLUGINS ******************/ - // 2000+ PLUGIN_ARADO_ID = 0x2001, PLUGIN_QCHESS_ID = 0x2002, PLUGIN_FEEDREADER = 0x2003, + /// GXS distant sync and search do not confuse with @see GXS_TUNNEL + GXS_DISTANT = 0x2233, + /// Reserved for packet slicing probes. PACKET_SLICING_PROBE = 0xAABB, diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index b5fe1426f..58af0b22d 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012 Robert Fernie * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -22,6 +23,8 @@ *******************************************************************************/ #pragma once +#include +#include #include "retroshare/rsgxschannels.h" #include "services/p3gxscommon.h" @@ -30,9 +33,7 @@ #include "util/rsmemory.h" #include "util/rsdebug.h" #include "util/rstickevent.h" - -#include -#include +#include "deep_search/channelsindex.hpp" // This class is only a helper to parse the channel group service string. @@ -56,6 +57,11 @@ class p3GxsChannels: public RsGenExchange, public RsGxsChannels, public: p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes, RsGixs* gixs ); + + /// @see RsGxsChannels + std::error_condition distantSearchRequest( + const std::string& matchString, TurtleRequestId& searchId ) override; + virtual RsServiceInfo getServiceInfo() override; virtual void service_tick() override; @@ -69,7 +75,7 @@ protected: virtual bool loadList(std::list& loadList) override; // @see p3Config::loadList(std::list&) virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override; - virtual TurtleRequestId turtleSearchRequest(const std::string& match_string) override; + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) override; virtual bool clearDistantSearchResults(TurtleRequestId req) override; virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) override; @@ -112,24 +118,6 @@ virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled) virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory) override; virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory) override; -#ifdef TO_REMOVE - /// @see RsGxsChannels::turtleSearchRequest - virtual bool turtleSearchRequest(const std::string& matchString, - const std::function& multiCallback, - rstime_t maxWait = 300 ) override; - - /// @see RsGxsChannels::turtleChannelRequest - virtual bool turtleChannelRequest( - const RsGxsGroupId& channelId, - const std::function& multiCallback, - rstime_t maxWait = 300 ) override; - - /// @see RsGxsChannels::localSearchRequest - virtual bool localSearchRequest(const std::string& matchString, - const std::function& multiCallback, - rstime_t maxWait = 30 ) override; -#endif - /** * Receive results from turtle search @see RsGenExchange @see RsNxsObserver * @see RsGxsNetService::receiveTurtleSearchResults @@ -215,9 +203,10 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override; const std::set &contentIds, std::vector &comments) override; - /// Implementation of @see RsGxsChannels::getContentSummaries - bool getContentSummaries( + /// @see RsGxsChannels + std::error_condition getContentSummaries( const RsGxsGroupId& channelId, + const std::set& contentIds, std::vector& summaries ) override; /// Implementation of @see RsGxsChannels::getChannelStatistics @@ -297,6 +286,17 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override; virtual bool shareChannelKeys( const RsGxsGroupId& channelId, const std::set& peers ) override; +#ifdef RS_DEEP_CHANNEL_INDEX + /// @see RsNxsObserver + std::error_condition handleDistantSearchRequest( + rs_view_ptr requestData, uint32_t requestSize, + rs_owner_ptr& resultData, uint32_t& resultSize ) override; + + std::error_condition receiveDistantSearchResult( + const TurtleRequestId requestId, + rs_owner_ptr& resultData, uint32_t& resultSize ) override; +#endif + /// Implementation of @see RsGxsChannels::createChannel RS_DEPRECATED_FOR(createChannelV2) bool createChannel(RsGxsChannelGroup& channel) override; @@ -313,7 +313,6 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override; RS_DEPRECATED_FOR(createVoteV2) bool createVote(RsGxsVote& vote) override; - protected: // Overloaded from GxsTokenQueue for Request callbacks. virtual void handleResponse(uint32_t token, uint32_t req_type @@ -329,7 +328,6 @@ static uint32_t channelsAuthenPolicy(); void request_SpecificSubscribedGroups(const std::list &groups); void load_SubscribedGroups(const uint32_t &token); - void request_SpecificUnprocessedPosts(std::list > &ids); void request_GroupUnprocessedPosts(const std::list &grouplist); void load_unprocessedPosts(uint32_t token); @@ -390,26 +388,8 @@ bool generateGroup(uint32_t &token, std::string groupName); rstime_t mLastDistantSearchNotificationTS; std::map > mSearchResultsToNotify; -#ifdef TO_REMOVE - /** Store search callbacks with timeout*/ - std::map< - TurtleRequestId, - std::pair< - std::function, - std::chrono::system_clock::time_point > - > mSearchCallbacksMap; - RsMutex mSearchCallbacksMapMutex; - /** Store distant channels requests callbacks with timeout*/ - std::map< - TurtleRequestId, - std::pair< - std::function, - std::chrono::system_clock::time_point > - > mDistantChannelsCallbacksMap; - RsMutex mDistantChannelsCallbacksMapMutex; - - /// Cleanup mSearchCallbacksMap and mDistantChannelsCallbacksMap - void cleanTimedOutCallbacks(); +#ifdef RS_DEEP_CHANNEL_INDEX + DeepChannelsIndex mDeepIndex; #endif }; diff --git a/libretroshare/src/util/rsdebug.cc b/libretroshare/src/util/rsdebug.cc index 0a4777ab0..ac121bca3 100644 --- a/libretroshare/src/util/rsdebug.cc +++ b/libretroshare/src/util/rsdebug.cc @@ -3,9 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright (C) 2004-2008 by Robert Fernie * - * Copyright (C) 2020 Gioacchino Mazzurco * - * Copyright (C) 2020 Asociación Civil Altermundi * + * Copyright (C) 2004-2008 Robert Fernie * + * Copyright (C) 2020-2021 Gioacchino Mazzurco * + * Copyright (C) 2020-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -36,6 +36,9 @@ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName) " not available in category: " + categoryName; } +std::error_condition rs_errno_to_condition(int errno_code) +{ return std::make_error_condition(static_cast(errno_code)); } + //////////////////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////////////////// diff --git a/libretroshare/src/util/rsdebug.h b/libretroshare/src/util/rsdebug.h index 6409e9492..4eddac240 100644 --- a/libretroshare/src/util/rsdebug.h +++ b/libretroshare/src/util/rsdebug.h @@ -2,8 +2,8 @@ * RetroShare debugging utilities * * * * Copyright (C) 2004-2008 Robert Fernie * - * Copyright (C) 2019-2020 Gioacchino Mazzurco * - * Copyright (C) 2020 Asociación Civil Altermundi * + * Copyright (C) 2019-2021 Gioacchino Mazzurco * + * Copyright (C) 2020-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -65,6 +65,12 @@ std::ostream &operator<<(std::ostream& out, const std::error_condition& err); * the message around */ std::string rsErrorNotInCategory(int errNum, const std::string& categoryName); +/** Convert C errno codes to modern C++11 std::error_condition, this is quite + * useful to use toghether with C functions used around the code like `malloc`, + * `socket` etc to let errors bubble up comprensibly to upper layers C++11 code + */ +std::error_condition rs_errno_to_condition(int errno_code); + template struct t_RsLogger : std::ostringstream diff --git a/libretroshare/src/util/rsmemory.h b/libretroshare/src/util/rsmemory.h index eb2889a6f..8afc48cf7 100644 --- a/libretroshare/src/util/rsmemory.h +++ b/libretroshare/src/util/rsmemory.h @@ -3,8 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2012 Cyril Soler * - * Copyright 2019-2020 Gioacchino Mazzurco * + * Copyright (C) 2012 Cyril Soler * + * Copyright (C) 2019-2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -25,8 +26,10 @@ #include #include #include +#include #include "util/stacktrace.h" +#include "util/rsdebug.h" /** * @brief Shorthand macro to declare optional functions output parameters @@ -108,7 +111,66 @@ template using rs_view_ptr = T*; * @see http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2019/p1408r0.pdf */ template using rs_owner_ptr = T*; -void *rs_malloc(size_t size) ; + +/// 1Gb should be enough for everything! +static constexpr size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024; + +/** Comfortable templated safer malloc, just use it specifing the type of the + * pointer to be returned without need of ugly casting the returned pointer + * `uint8_t* ptr = rs_malloc(40);` + * @param[in] size number of bytes to allocate + * @param[out] ec optional storage for error details. Value is meaningful only + * whem nullptr is returned. + * @return nullptr on error, pointer to the allocated chuck of memory on success + */ +template rs_owner_ptr rs_malloc( + size_t size, + rs_view_ptr ec = nullptr ) +{ + if(size == 0) + { + if(!ec) + { + RS_ERR("A chunk of size 0 was requested"); + print_stacktrace(); + exit(static_cast(std::errc::invalid_argument)); + } + + *ec = std::errc::invalid_argument; + return nullptr; + } + + if(size > SAFE_MEMALLOC_THRESHOLD) + { + if(!ec) + { + RS_ERR( "A chunk of size larger than ", SAFE_MEMALLOC_THRESHOLD, + " was requested" ); + exit(static_cast(std::errc::argument_out_of_domain)); + } + + *ec = std::errc::argument_out_of_domain; + return nullptr; + } + + void* mem = malloc(size); + if(!mem) + { + if(!ec) + { + RS_ERR( "Allocation failed for a chunk of ", size, + " bytes with: ", errno); + print_stacktrace(); + exit(errno); + } + + *ec = rs_errno_to_condition(errno); + return nullptr; + } + + return static_cast>(mem); +} + /** @deprecated use std::unique_ptr instead // This is a scope guard to release the memory block when going of of the current scope. @@ -128,7 +190,7 @@ void *rs_malloc(size_t size) ; // // } // mem gets freed automatically */ -class RsTemporaryMemory +class RS_DEPRECATED_FOR("std::unique_ptr") RsTemporaryMemory { public: explicit RsTemporaryMemory(size_t s) diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index e18a161a3..baf715a96 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -411,7 +411,9 @@ void GxsChannelDialog::clearDistantSearchResults(TurtleRequestId id) TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string) { - return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ; + TurtleRequestId searchId; + rsGxsChannels->distantSearchRequest(search_string.toStdString(), searchId); + return searchId; } bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map& group_infos) From 9c38eed6482c70a109143c0a7d960ed12ff9f4c1 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 19 Feb 2021 23:23:02 +0100 Subject: [PATCH 05/10] Implement deep indexing and search for forums RsGxsNetTunnelService::receiveSearchRequest handle no results case properly RsNxsObserver::handleDistantSearchRequest improve method behaviour documentation RsTurtleClientService Improve documentation --- libretroshare/src/deep_search/commonutils.cpp | 20 + libretroshare/src/deep_search/forumsindex.cpp | 208 ++++++++++ libretroshare/src/deep_search/forumsindex.hpp | 81 ++++ libretroshare/src/gxs/rsgxsnetservice.cc | 6 + libretroshare/src/gxs/rsnxsobserver.h | 8 +- libretroshare/src/retroshare/rsgxschannels.h | 2 +- libretroshare/src/retroshare/rsgxsforums.h | 76 +++- libretroshare/src/rsitems/rsgxsforumitems.h | 62 ++- libretroshare/src/rsserver/rsinit.cc | 34 +- libretroshare/src/services/p3gxsforums.cc | 363 ++++++++++++++++-- libretroshare/src/services/p3gxsforums.h | 52 ++- .../src/turtle/turtleclientservice.h | 73 ++-- retroshare.pri | 6 + 13 files changed, 902 insertions(+), 89 deletions(-) create mode 100644 libretroshare/src/deep_search/forumsindex.cpp create mode 100644 libretroshare/src/deep_search/forumsindex.hpp diff --git a/libretroshare/src/deep_search/commonutils.cpp b/libretroshare/src/deep_search/commonutils.cpp index cbe4ee27b..c3b9c5342 100644 --- a/libretroshare/src/deep_search/commonutils.cpp +++ b/libretroshare/src/deep_search/commonutils.cpp @@ -168,13 +168,33 @@ std::string simpleTextHtmlExtract(const std::string& rsHtmlDoc) std::string retVal(rsHtmlDoc.substr(bodyTagEnd+1)); + // strip also CSS inside + oSize = retVal.size(); + auto styleTagBegin(retVal.find("", styleTagBegin)); + if(styleEnd < oSize) + retVal.erase(styleTagBegin, 8+styleEnd-styleTagBegin); + } + std::string::size_type oPos; std::string::size_type cPos; + int itCount = 0; while((oPos = retVal.find("<")) < retVal.size()) { if((cPos = retVal.find(">")) <= retVal.size()) retVal.erase(oPos, 1+cPos-oPos); else break; + + // Avoid infinite loop with crafty input + if(itCount > 1000) + { + RS_WARN( "Breaking stripping loop due to max allowed iterations ", + "rsHtmlDoc: ", rsHtmlDoc, " retVal: ", retVal ); + break; + } + ++itCount; } return retVal; diff --git a/libretroshare/src/deep_search/forumsindex.cpp b/libretroshare/src/deep_search/forumsindex.cpp new file mode 100644 index 000000000..acc7aed9a --- /dev/null +++ b/libretroshare/src/deep_search/forumsindex.cpp @@ -0,0 +1,208 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "deep_search/forumsindex.hpp" +#include "deep_search/commonutils.hpp" +#include "retroshare/rsinit.h" +#include "retroshare/rsgxsforums.h" +#include "util/rsdebuglevel4.h" + +std::error_condition DeepForumsIndex::search( + const std::string& queryStr, + std::vector& results, uint32_t maxResults ) +{ + results.clear(); + + std::unique_ptr dbPtr( + DeepSearch::openReadOnlyDatabase(mDbPath) ); + if(!dbPtr) return std::errc::bad_file_descriptor; + + Xapian::Database& db(*dbPtr); + + // Set up a QueryParser with a stemmer and suitable prefixes. + Xapian::QueryParser queryparser; + //queryparser.set_stemmer(Xapian::Stem("en")); + queryparser.set_stemming_strategy(queryparser.STEM_SOME); + // Start of prefix configuration. + //queryparser.add_prefix("title", "S"); + //queryparser.add_prefix("description", "XD"); + // End of prefix configuration. + + // And parse the query. + Xapian::Query query = queryparser.parse_query(queryStr); + + // Use an Enquire object on the database to run the query. + Xapian::Enquire enquire(db); + enquire.set_query(query); + + Xapian::MSet mset = enquire.get_mset( + 0, maxResults ? maxResults : db.get_doccount() ); + + for( Xapian::MSetIterator m = mset.begin(); m != mset.end(); ++m ) + { + const Xapian::Document& doc = m.get_document(); + DeepForumsSearchResult s; + s.mUrl = doc.get_value(URL_VALUENO); +#if XAPIAN_AT_LEAST(1,3,5) + s.mSnippet = mset.snippet(doc.get_data()); +#endif // XAPIAN_AT_LEAST(1,3,5) + results.push_back(s); + } + + return std::error_condition(); +} + +/*static*/ std::string DeepForumsIndex::forumIndexId(const RsGxsGroupId& grpId) +{ + RsUrl forumIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL); + forumIndexId.setQueryKV( + RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString() ); + return forumIndexId.toString(); +} + +/*static*/ std::string DeepForumsIndex::postIndexId( + const RsGxsGroupId& grpId, const RsGxsMessageId& msgId ) +{ + RsUrl postIndexId(RsGxsForums::DEFAULT_FORUM_BASE_URL); + postIndexId.setQueryKV(RsGxsForums::FORUM_URL_ID_FIELD, grpId.toStdString()); + postIndexId.setQueryKV(RsGxsForums::FORUM_URL_MSG_ID_FIELD, msgId.toStdString()); + return postIndexId.toString(); +} + +std::error_condition DeepForumsIndex::indexForumGroup( + const RsGxsForumGroup& forum ) +{ + // Set up a TermGenerator that we'll use in indexing. + Xapian::TermGenerator termgenerator; + //termgenerator.set_stemmer(Xapian::Stem("en")); + + // We make a document and tell the term generator to use this. + Xapian::Document doc; + termgenerator.set_document(doc); + + // Index each field with a suitable prefix. + termgenerator.index_text(forum.mMeta.mGroupName, 1, "G"); + termgenerator.index_text( + DeepSearch::timetToXapianDate(forum.mMeta.mPublishTs), 1, "D" ); + termgenerator.index_text(forum.mDescription, 1, "XD"); + + // Index fields without prefixes for general search. + termgenerator.index_text(forum.mMeta.mGroupName); + termgenerator.increase_termpos(); + termgenerator.index_text(forum.mDescription); + + // store the RS link so we are able to retrive it on matching search + const std::string rsLink(forumIndexId(forum.mMeta.mGroupId)); + doc.add_value(URL_VALUENO, rsLink); + + /* Store some fields for display purposes. Retrieved later to provide the + * matching snippet on search */ + doc.set_data(forum.mMeta.mGroupName + "\n" + forum.mDescription); + + /* We use the identifier to ensure each object ends up in the database only + * once no matter how many times we run the indexer. + * "Q" prefix is a Xapian convention for unique id term. */ + const std::string idTerm("Q" + rsLink); + doc.add_boolean_term(idTerm); + + mWriteQueue.push([idTerm, doc](Xapian::WritableDatabase& db) + { db.replace_document(idTerm, doc); } ); + + return std::error_condition(); +} + +std::error_condition DeepForumsIndex::removeForumFromIndex( + const RsGxsGroupId& grpId ) +{ + mWriteQueue.push([grpId](Xapian::WritableDatabase& db) + { db.delete_document("Q" + forumIndexId(grpId)); }); + + return std::error_condition(); +} + +std::error_condition DeepForumsIndex::indexForumPost(const RsGxsForumMsg& post) +{ + RS_DBG4(post); + + const auto& groupId = post.mMeta.mGroupId; + const auto& msgId = post.mMeta.mMsgId; + + if(groupId.isNull() || msgId.isNull()) + { + RS_ERR("Got post with invalid id ", post); + print_stacktrace(); + return std::errc::invalid_argument; + } + + // Set up a TermGenerator that we'll use in indexing. + Xapian::TermGenerator termgenerator; + //termgenerator.set_stemmer(Xapian::Stem("en")); + + // We make a document and tell the term generator to use this. + Xapian::Document doc; + termgenerator.set_document(doc); + + // Index each field with a suitable prefix. + termgenerator.index_text(post.mMeta.mMsgName, 1, "S"); + termgenerator.index_text( + DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" ); + + // Avoid indexing RetroShare-gui HTML tags + const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg); + termgenerator.index_text(cleanMsg, 1, "XD" ); + + // Index fields without prefixes for general search. + termgenerator.index_text(post.mMeta.mMsgName); + + termgenerator.increase_termpos(); + termgenerator.index_text(cleanMsg); + // store the RS link so we are able to retrive it on matching search + const std::string rsLink(postIndexId(groupId, msgId)); + doc.add_value(URL_VALUENO, rsLink); + + // Store some fields for display purposes. + doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg); + + // We use the identifier to ensure each object ends up in the + // database only once no matter how many times we run the + // indexer. + const std::string idTerm("Q" + rsLink); + doc.add_boolean_term(idTerm); + + mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db) + { db.replace_document(idTerm, doc); } ); + + + return std::error_condition(); +} + +std::error_condition DeepForumsIndex::removeForumPostFromIndex( + RsGxsGroupId grpId, RsGxsMessageId msgId ) +{ + // "Q" prefix is a Xapian convention for unique id term. + std::string idTerm("Q" + postIndexId(grpId, msgId)); + mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db) + { db.delete_document(idTerm); } ); + + return std::error_condition(); +} + +/*static*/ std::string DeepForumsIndex::dbDefaultPath() +{ return RsAccounts::AccountDirectory() + "/deep_forum_index_xapian_db"; } diff --git a/libretroshare/src/deep_search/forumsindex.hpp b/libretroshare/src/deep_search/forumsindex.hpp new file mode 100644 index 000000000..2955ce323 --- /dev/null +++ b/libretroshare/src/deep_search/forumsindex.hpp @@ -0,0 +1,81 @@ +/******************************************************************************* + * RetroShare full text indexing and search implementation based on Xapian * + * * + * Copyright (C) 2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Affero General Public License version 3 as * + * published by the Free Software Foundation. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Affero General Public License for more details. * + * * + * You should have received a copy of the GNU Affero General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ +#pragma once + +#include +#include +#include + +#include "util/rstime.h" +#include "retroshare/rsgxsforums.h" +#include "retroshare/rsevents.h" +#include "deep_search/commonutils.hpp" + +struct DeepForumsSearchResult +{ + std::string mUrl; + double mWeight; + std::string mSnippet; +}; + +struct DeepForumsIndex +{ + explicit DeepForumsIndex(const std::string& dbPath) : + mDbPath(dbPath), mWriteQueue(dbPath) {} + + /** + * @brief Search indexed GXS groups and messages + * @param[in] maxResults maximum number of acceptable search results, 0 for + * no limits + * @return search results count + */ + std::error_condition search( const std::string& queryStr, + std::vector& results, + uint32_t maxResults = 100 ); + + std::error_condition indexForumGroup(const RsGxsForumGroup& chan); + + std::error_condition removeForumFromIndex(const RsGxsGroupId& grpId); + + std::error_condition indexForumPost(const RsGxsForumMsg& post); + + std::error_condition removeForumPostFromIndex( + RsGxsGroupId grpId, RsGxsMessageId msgId ); + + static std::string dbDefaultPath(); + +private: + static std::string forumIndexId(const RsGxsGroupId& grpId); + static std::string postIndexId( + const RsGxsGroupId& grpId, const RsGxsMessageId& msgId ); + + enum : Xapian::valueno + { + /// Used to store retroshare url of indexed documents + URL_VALUENO, + + /// @see Xapian::BAD_VALUENO + BAD_VALUENO = Xapian::BAD_VALUENO + }; + + const std::string mDbPath; + + DeepSearch::StubbornWriteOpQueue mWriteQueue; +}; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 8038b63f4..e664b580c 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5420,6 +5420,12 @@ std::error_condition RsGxsNetService::distantSearchRequest( rs_owner_ptr searchData, uint32_t dataSize, RsServiceType serviceType, TurtleRequestId& requestId ) { + if(!mGxsNetTunnel) + { + free(searchData); + return std::errc::function_not_supported; + } + return mGxsNetTunnel->turtleSearchRequest( searchData, dataSize, serviceType, requestId ); } diff --git a/libretroshare/src/gxs/rsnxsobserver.h b/libretroshare/src/gxs/rsnxsobserver.h index 5725df0f0..9d81e7656 100644 --- a/libretroshare/src/gxs/rsnxsobserver.h +++ b/libretroshare/src/gxs/rsnxsobserver.h @@ -104,8 +104,10 @@ public: * requests there. * @param[in] requestData search query * @param[in] requestSize search query size - * @param[out] resultData results data - * @param[out] resultSize results data size + * @param[out] resultData results data storage for a pointer to search + * result reply data or nullptr if no mathing results where found + * @param[out] resultSize storage for results data size or 0 if no matching + * results where found * @return Error details or success, NOT_OVERRIDDEN_BY_OBSERVER is * returned to inform the caller that this method was not overridden by the * observer so do not use it for other meanings. */ @@ -113,6 +115,8 @@ public: rs_view_ptr requestData, uint32_t requestSize, rs_owner_ptr& resultData, uint32_t& resultSize ) { + /* Avoid unused paramethers warning this way so doxygen can still parse + * paramethers documentation */ (void) requestData; (void) requestSize; (void) resultData; (void) resultSize; return RsNxsObserverErrorNum::NOT_OVERRIDDEN_BY_OBSERVER; diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index bd09950f4..a91e341a7 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -427,7 +427,7 @@ public: * @param[in] contentIds ids of requested contents, if empty summaries of * all messages are reqeusted * @param[out] summaries storage for summaries - * @return false if something failed, true otherwhise + * @return success or error details if something failed */ virtual std::error_condition getContentSummaries( const RsGxsGroupId& channelId, diff --git a/libretroshare/src/retroshare/rsgxsforums.h b/libretroshare/src/retroshare/rsgxsforums.h index 1b35f2db9..561fe646d 100644 --- a/libretroshare/src/retroshare/rsgxsforums.h +++ b/libretroshare/src/retroshare/rsgxsforums.h @@ -4,8 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012-2014 Robert Fernie * - * Copyright (C) 2018-2020 Gioacchino Mazzurco * - * Copyright (C) 2019-2020 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -118,7 +118,10 @@ enum class RsForumEventCode: uint8_t SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed PINNED_POSTS_CHANGED = 0x0b, /// some posts where pinned or un-pinned DELETED_FORUM = 0x0c, /// forum was deleted by cleaning - DELETED_POSTS = 0x0d /// Posts deleted by cleaning + DELETED_POST = 0x0d, /// Post deleted (usually by cleaning) + + /// Distant search result received + DISTANT_SEARCH_RESULT = 0x0e }; struct RsGxsForumEvent: RsEvent @@ -149,6 +152,29 @@ struct RsGxsForumEvent: RsEvent ~RsGxsForumEvent() override; }; +/** This event is fired once distant search results are received */ +struct RsGxsForumsDistantSearchEvent: RsEvent +{ + RsGxsForumsDistantSearchEvent(): + RsEvent(RsEventType::GXS_CHANNELS), + mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {} + + RsForumEventCode mForumEventCode; + TurtleRequestId mSearchId; + std::vector mSearchResults; + + ///* @see RsEvent @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RsEvent::serial_process(j, ctx); + + RS_SERIAL_PROCESS(mForumEventCode); + RS_SERIAL_PROCESS(mSearchId); + RS_SERIAL_PROCESS(mSearchResults); + } +}; + class RsGxsForums: public RsGxsIfaceHelper { public: @@ -385,6 +411,50 @@ public: const RsGxsGroupId& forumId, const RsGxsMessageId& postId, bool keepForever ) = 0; + /** + * @brief Get forum content summaries + * @jsonapi{development} + * @param[in] forumId id of the forum of which the content is requested + * @param[in] contentIds ids of requested contents, if empty summaries of + * all messages are reqeusted + * @param[out] summaries storage for summaries + * @return success or error details if something failed + */ + virtual std::error_condition getContentSummaries( + const RsGxsGroupId& forumId, + const std::set& contentIds, + std::vector& summaries ) = 0; + + /** + * @brief Search the whole reachable network for matching forums and + * posts + * @jsonapi{development} + * An @see RsGxsForumsDistantSearchEvent is emitted when matching results + * arrives from the network + * @param[in] matchString string to search into the forum and posts + * @param[out] searchId storage for search id, useful to track search events + * and retrieve search results + * @return success or error details + */ + virtual std::error_condition distantSearchRequest( + const std::string& matchString, TurtleRequestId& searchId ) = 0; + + /** + * @brief Search the local index for matching forums and posts + * @jsonapi{development} + * @param[in] matchString string to search into the index + * @param[out] searchResults storage for searchr esults + * @return success or error details + */ + virtual std::error_condition localSearch( + const std::string& matchString, + std::vector& searchResults ) = 0; + + + //////////////////////////////////////////////////////////////////////////// + /* Following functions are deprecated and should not be considered a stable + * to use API */ + /** * @brief Create forum. Blocking API. * @jsonapi{development} diff --git a/libretroshare/src/rsitems/rsgxsforumitems.h b/libretroshare/src/rsitems/rsgxsforumitems.h index 30c6c4e08..b1f5fb629 100644 --- a/libretroshare/src/rsitems/rsgxsforumitems.h +++ b/libretroshare/src/rsitems/rsgxsforumitems.h @@ -3,7 +3,9 @@ * * * libretroshare: retroshare core library * * * - * Copyright 2012-2012 by Robert Fernie * + * Copyright (C) 2012 Robert Fernie * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -19,8 +21,7 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#ifndef RS_GXS_FORUM_ITEMS_H -#define RS_GXS_FORUM_ITEMS_H +#pragma once #include @@ -31,7 +32,18 @@ #include "retroshare/rsgxsforums.h" +enum class RsGxsForumsItems : uint8_t +{ + GROUP_ITEM = 0x02, + MESSAGE_ITEM = 0x03, + SEARCH_REQUEST = 0x04, + SEARCH_REPLY = 0x05, +}; + +RS_DEPRECATED_FOR(RsGxsForumsItems) const uint8_t RS_PKT_SUBTYPE_GXSFORUM_GROUP_ITEM = 0x02; + +RS_DEPRECATED_FOR(RsGxsForumsItems) const uint8_t RS_PKT_SUBTYPE_GXSFORUM_MESSAGE_ITEM = 0x03; class RsGxsForumGroupItem : public RsGxsGrpItem @@ -61,6 +73,48 @@ public: RsGxsForumMsg mMsg; }; +struct RsGxsForumsSearchRequest : RsSerializable +{ + RsGxsForumsSearchRequest() : mType(RsGxsForumsItems::SEARCH_REQUEST) {} + + /// Just for easier back and forward compatibility + RsGxsForumsItems mType; + + /// Store search match string + std::string mQuery; + + /// @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RS_SERIAL_PROCESS(mType); + RS_SERIAL_PROCESS(mQuery); + } + + ~RsGxsForumsSearchRequest() override = default; +}; + +struct RsGxsForumsSearchReply : RsSerializable +{ + RsGxsForumsSearchReply() : mType(RsGxsForumsItems::SEARCH_REPLY) {} + + /// Just for easier back and forward compatibility + RsGxsForumsItems mType; + + /// Results storage + std::vector mResults; + + /// @see RsSerializable + void serial_process( RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ) override + { + RS_SERIAL_PROCESS(mType); + RS_SERIAL_PROCESS(mResults); + } + + ~RsGxsForumsSearchReply() override = default; +}; + class RsGxsForumSerialiser : public RsServiceSerializer { public: @@ -69,5 +123,3 @@ public: virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ; }; - -#endif /* RS_GXS_FORUM_ITEMS_H */ diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index ad39f46a1..7433ab151 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -2,7 +2,8 @@ * libretroshare/src/retroshare: rsinit.cc * * * * Copyright (C) 2004-2014 Robert Fernie * - * Copyright (C) 2016-2019 Gioacchino Mazzurco * + * Copyright (C) 2016-2021 Gioacchino Mazzurco * + * Copyright (C) 2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -1336,22 +1337,26 @@ int RsServer::StartupRetroShare() mWiki->setNetworkExchangeService(wiki_ns) ; #endif - /**** Forum GXS service ****/ + /************************* Forum GXS service ******************************/ - RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db", - RS_SERVICE_GXS_TYPE_FORUMS, NULL, rsInitConfig->gxs_passwd); + RsGeneralDataService* gxsforums_ds = new RsDataService( + currGxsDir + "/", "gxsforums_db", RS_SERVICE_GXS_TYPE_FORUMS, + nullptr, rsInitConfig->gxs_passwd ); + p3GxsForums* mGxsForums = new p3GxsForums( + gxsforums_ds, nullptr, mGxsIdService ); - p3GxsForums *mGxsForums = new p3GxsForums(gxsforums_ds, NULL, mGxsIdService); + RsGxsNetTunnelService* gxsForumsTunnelService = nullptr; +#ifdef RS_DEEP_FORUMS_INDEX + gxsForumsTunnelService = mGxsNetTunnel; +#endif - // create GXS photo service - RsGxsNetService* gxsforums_ns = new RsGxsNetService( - RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, - mGxsForums, mGxsForums->getServiceInfo(), - mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils);//,mGxsNetTunnel,true,true,true); + RsGxsNetService* gxsforums_ns = new RsGxsNetService( + RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums, + mGxsForums->getServiceInfo(), mReputations, mGxsCircles, + mGxsIdService, pgpAuxUtils, gxsForumsTunnelService ); + mGxsForums->setNetworkExchangeService(gxsforums_ns); - mGxsForums->setNetworkExchangeService(gxsforums_ns) ; /**** Channel GXS service ****/ @@ -1598,7 +1603,10 @@ int RsServer::StartupRetroShare() /**************************************************************************/ // Turtle search for GXS services - mGxsNetTunnel->registerSearchableService(gxschannels_ns) ; + mGxsNetTunnel->registerSearchableService(gxschannels_ns); +#ifdef RS_DEEP_FORUMS_INDEX + mGxsNetTunnel->registerSearchableService(gxsforums_ns); +#endif /**************************************************************************/ diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index cd7cbb5fe..2f85afe59 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -4,8 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012-2014 Robert Fernie * - * Copyright (C) 2018-2020 Gioacchino Mazzurco * - * Copyright (C) 2019-2020 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -59,7 +59,11 @@ p3GxsForums::p3GxsForums( RsGeneralDataService *gds, RsGenExchange( gds, nes, new RsGxsForumSerialiser(), RS_SERVICE_GXS_TYPE_FORUMS, gixs, forumsAuthenPolicy()), RsGxsForums(static_cast(*this)), mGenToken(0), - mGenActive(false), mGenCount(0), mKnownForumsMutex("GXS forums known forums timestamp cache") + mGenActive(false), mGenCount(0), + mKnownForumsMutex("GXS forums known forums timestamp cache") +#ifdef RS_DEEP_FORUMS_INDEX + , mDeepIndex(DeepForumsIndex::dbDefaultPath()) +#endif { // Test Data disabled in Repo. //RsTickEvent::schedule_in(FORUM_TESTEVENT_DUMMYDATA, DUMMYDATA_PERIOD); @@ -190,31 +194,61 @@ RsSerialiser* p3GxsForums::setupSerialiser() return rss; } -void p3GxsForums::notifyChanges(std::vector &changes) +void p3GxsForums::notifyChanges(std::vector& changes) { RS_DBG2(changes.size(), " changes to notify"); - std::vector::iterator it; - for(it = changes.begin(); it != changes.end(); ++it) + for(RsGxsNotify* gxsChange: changes) { - RsGxsNotify* gxsChange = *it; + // Let the compiler delete the change for us + std::unique_ptr gxsChangeDeleter(gxsChange); + switch(gxsChange->getType()) { case RsGxsNotify::TYPE_RECEIVED_NEW: // [[fallthrough]] case RsGxsNotify::TYPE_PUBLISHED: { - RsGxsMsgChange* msgChange = dynamic_cast(*it); - RsGxsGroupChange* groupChange = dynamic_cast(*it); + auto msgChange = dynamic_cast(gxsChange); - if(msgChange) /* Message received*/ + if(msgChange) /* Message received */ { - auto ev = std::make_shared(); - ev->mForumMsgId = msgChange->mMsgId; - ev->mForumGroupId = msgChange->mGroupId; - ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE; - rsEvents->postEvent(ev); + uint8_t msgSubtype = msgChange->mNewMsgItem->PacketSubType(); + switch(static_cast(msgSubtype)) + { + case RsGxsForumsItems::MESSAGE_ITEM: + { + auto newForumMessageItem = + dynamic_cast( + msgChange->mNewMsgItem ); + + if(!newForumMessageItem) + { + RS_ERR("Received message change with mNewMsgItem type " + "mismatching or null"); + print_stacktrace(); + return; + } + +#ifdef RS_DEEP_FORUMS_INDEX + RsGxsForumMsg tmpPost = newForumMessageItem->mMsg; + tmpPost.mMeta = newForumMessageItem->meta; + mDeepIndex.indexForumPost(tmpPost); +#endif + auto ev = std::make_shared(); + ev->mForumMsgId = msgChange->mMsgId; + ev->mForumGroupId = msgChange->mGroupId; + ev->mForumEventCode = RsForumEventCode::NEW_MESSAGE; + rsEvents->postEvent(ev); + break; + } + default: + RS_WARN("Got unknown gxs message subtype: ", msgSubtype); + break; + } } - else if(groupChange) /* Group received */ + + auto groupChange = dynamic_cast(gxsChange); + if(groupChange) /* Group received */ { bool unknown; { @@ -232,9 +266,25 @@ void p3GxsForums::notifyChanges(std::vector &changes) ev->mForumEventCode = RsForumEventCode::NEW_FORUM; rsEvents->postEvent(ev); } - else - RS_DBG1( " Not notifying already known forum ", - gxsChange->mGroupId ); + +#ifdef RS_DEEP_FORUMS_INDEX + uint8_t itemType = groupChange->mNewGroupItem->PacketSubType(); + switch(static_cast(itemType)) + { + case RsGxsForumsItems::GROUP_ITEM: + { + auto newForumGroupItem = + static_cast( + groupChange->mNewGroupItem ); + mDeepIndex.indexForumGroup(newForumGroupItem->mGroup); + break; + } + default: + RS_WARN("Got unknown gxs group subtype: ", itemType); + break; + } +#endif // def RS_DEEP_FORUMS_INDEX + } break; } @@ -256,25 +306,31 @@ void p3GxsForums::notifyChanges(std::vector &changes) } case RsGxsNotify::TYPE_MESSAGE_DELETED: { - RsGxsMsgDeletedChange* delChange = - dynamic_cast(gxsChange); - + auto delChange = dynamic_cast(gxsChange); if(!delChange) { RS_ERR( "Got mismatching notification type: ", gxsChange->getType() ); print_stacktrace(); - goto cleanup; + break; } +#ifdef RS_DEEP_FORUMS_INDEX + mDeepIndex.removeForumPostFromIndex( + delChange->mGroupId, delChange->messageId); +#endif + auto ev = std::make_shared(); - ev->mForumEventCode = RsForumEventCode::DELETED_POSTS; + ev->mForumEventCode = RsForumEventCode::DELETED_POST; ev->mForumGroupId = delChange->mGroupId; ev->mForumMsgId = delChange->messageId; break; } case RsGxsNotify::TYPE_GROUP_DELETED: { +#ifdef RS_DEEP_FORUMS_INDEX + mDeepIndex.removeForumFromIndex(gxsChange->mGroupId); +#endif auto ev = std::make_shared(); ev->mForumGroupId = gxsChange->mGroupId; ev->mForumEventCode = RsForumEventCode::DELETED_FORUM; @@ -299,7 +355,7 @@ void p3GxsForums::notifyChanges(std::vector &changes) * analyse the old and new group in order to detect possible * notifications for clients */ - RsGxsGroupChange* grpChange = dynamic_cast(*it); + auto grpChange = dynamic_cast(gxsChange); RsGxsForumGroupItem* old_forum_grp_item = dynamic_cast(grpChange->mOldGroupItem); @@ -312,9 +368,13 @@ void p3GxsForums::notifyChanges(std::vector &changes) "mNewGroup not of type RsGxsForumGroupItem or NULL. " "This is inconsistent!"); print_stacktrace(); - goto cleanup; + break; } +#ifdef RS_DEEP_FORUMS_INDEX + mDeepIndex.indexForumGroup(new_forum_grp_item->mGroup); +#endif + /* First of all, we check if there is a difference between the old * and new list of moderators */ @@ -382,9 +442,6 @@ void p3GxsForums::notifyChanges(std::vector &changes) " Currently not handled." ); break; } - -cleanup: - delete *it; } } @@ -1348,6 +1405,254 @@ bool RsGxsForumGroup::canEditPosts(const RsGxsId& id) const id == mMeta.mAuthorId; } +std::error_condition p3GxsForums::getContentSummaries( + const RsGxsGroupId& forumId, + const std::set& contentIds, + std::vector& summaries ) +{ + uint32_t token; + RsTokReqOptions opts; + opts.mReqType = GXS_REQUEST_TYPE_MSG_META; + + GxsMsgReq msgReq; + msgReq[forumId] = contentIds; + + + if(!requestMsgInfo(token, opts, msgReq)) + { + RS_ERR("requestMsgInfo failed"); + return std::errc::invalid_argument; + } + + switch(waitToken(token, std::chrono::seconds(5))) + { + case RsTokenService::COMPLETE: + { + GxsMsgMetaMap metaMap; + if(!RsGenExchange::getMsgMeta(token, metaMap)) + return std::errc::result_out_of_range; + summaries = metaMap[forumId]; + return std::error_condition(); + } + case RsTokenService::PARTIAL: // [[fallthrough]]; + case RsTokenService::PENDING: + return std::errc::timed_out; + default: + return std::errc::not_supported; + } +} + +#ifdef RS_DEEP_FORUMS_INDEX +std::error_condition p3GxsForums::handleDistantSearchRequest( + rs_view_ptr requestData, uint32_t requestSize, + rs_owner_ptr& resultData, uint32_t& resultSize ) +{ + RS_DBG1(""); + + RsGxsForumsSearchRequest request; + { + RsGenericSerializer::SerializeContext ctx(requestData, requestSize); + RsGenericSerializer::SerializeJob j = + RsGenericSerializer::SerializeJob::DESERIALIZE; + RS_SERIAL_PROCESS(request); + } + + if(request.mType != RsGxsForumsItems::SEARCH_REQUEST) + { + // If more types are implemented we would put a switch on mType instead + RS_WARN( "Got search request with unkown type: ", + static_cast(request.mType) ); + return std::errc::bad_message; + } + + RsGxsForumsSearchReply reply; + auto mErr = prepareSearchResults(request.mQuery, true, reply.mResults); + if(mErr || reply.mResults.empty()) return mErr; + + { + RsGenericSerializer::SerializeContext ctx; + RsGenericSerializer::SerializeJob j = + RsGenericSerializer::SerializeJob::SIZE_ESTIMATE; + RS_SERIAL_PROCESS(reply); + resultSize = ctx.mOffset; + } + + resultData = rs_malloc(resultSize); + RsGenericSerializer::SerializeContext ctx(resultData, resultSize); + RsGenericSerializer::SerializeJob j = + RsGenericSerializer::SerializeJob::SERIALIZE; + RS_SERIAL_PROCESS(reply); + + return std::error_condition(); +} + +std::error_condition p3GxsForums::distantSearchRequest( + const std::string& matchString, TurtleRequestId& searchId ) +{ + RsGxsForumsSearchRequest request; + request.mQuery = matchString; + + uint32_t requestSize; + { + RsGenericSerializer::SerializeContext ctx; + RsGenericSerializer::SerializeJob j = + RsGenericSerializer::SerializeJob::SIZE_ESTIMATE; + RS_SERIAL_PROCESS(request); + requestSize = ctx.mOffset; + } + + std::error_condition ec; + auto requestData = rs_malloc(requestSize, &ec); + if(!requestData) return ec; + { + RsGenericSerializer::SerializeContext ctx(requestData, requestSize); + RsGenericSerializer::SerializeJob j = + RsGenericSerializer::SerializeJob::SERIALIZE; + RS_SERIAL_PROCESS(request); + } + + return netService()->distantSearchRequest( + requestData, requestSize, + static_cast(serviceType()), searchId ); +} + +std::error_condition p3GxsForums::localSearch( + const std::string& matchString, + std::vector& searchResults ) +{ return prepareSearchResults(matchString, false, searchResults); } + +std::error_condition p3GxsForums::prepareSearchResults( + const std::string& matchString, bool publicOnly, + std::vector& searchResults ) +{ + std::vector results; + auto mErr = mDeepIndex.search(matchString, results); + if(mErr) return mErr; + + searchResults.clear(); + for(auto uRes: results) + { + RsUrl resUrl(uRes.mUrl); + const auto forumIdStr = resUrl.getQueryV(RsGxsForums::FORUM_URL_ID_FIELD); + if(!forumIdStr) + { + RS_ERR( "Forum URL retrieved from deep index miss ID. ", + "Should never happen! ", uRes.mUrl ); + print_stacktrace(); + return std::errc::address_not_available; + } + + std::vector forumsInfo; + RsGxsGroupId forumId(*forumIdStr); + if(forumId.isNull()) + { + RS_ERR( "Forum ID retrieved from deep index is invalid. ", + "Should never happen! ", uRes.mUrl ); + print_stacktrace(); + return std::errc::bad_address; + } + + if( !getForumsInfo(std::list{forumId}, forumsInfo) || + forumsInfo.empty() ) + { + RS_ERR( "Forum just parsed from deep index link not found. " + "Should never happen! ", forumId, " ", uRes.mUrl ); + print_stacktrace(); + return std::errc::identifier_removed; + } + + RsGroupMetaData& fMeta(forumsInfo[0].mMeta); + + // Avoid leaking sensitive information to unkown peers + if( publicOnly && + !(fMeta.mGroupFlags & GXS_SERV::FLAG_PRIVACY_PUBLIC) ) continue; + + RsGxsSearchResult res; + res.mGroupId = forumId; + res.mGroupName = fMeta.mGroupName; + res.mAuthorId = fMeta.mAuthorId; + res.mPublishTs = fMeta.mPublishTs; + res.mSearchContext = uRes.mSnippet; + + auto postIdStr = + resUrl.getQueryV(RsGxsForums::FORUM_URL_MSG_ID_FIELD); + if(postIdStr) + { + RsGxsMessageId msgId(*postIdStr); + if(msgId.isNull()) + { + RS_ERR( "Post just parsed from deep index link is invalid. " + "Should never happen! ", postIdStr, " ", uRes.mUrl ); + print_stacktrace(); + return std::errc::bad_address; + } + + std::vector msgSummaries; + auto errc = getContentSummaries( + forumId, std::set{msgId}, msgSummaries); + if(errc) return errc; + + if(msgSummaries.size() != 1) + { + RS_ERR( "getContentSummaries returned: ", msgSummaries.size(), + "should never happen!" ); + return std::errc::result_out_of_range; + } + + RsMsgMetaData& msgMeta(msgSummaries[0]); + res.mMsgId = msgMeta.mMsgId; + res.mMsgName = msgMeta.mMsgName; + res.mAuthorId = msgMeta.mAuthorId; + } + + RS_DBG4(res); + searchResults.push_back(res); + } + + return std::error_condition(); +} + +std::error_condition p3GxsForums::receiveDistantSearchResult( + const TurtleRequestId requestId, + rs_owner_ptr& resultData, uint32_t& resultSize ) +{ + RsGxsForumsSearchReply reply; + { + RsGenericSerializer::SerializeContext ctx(resultData, resultSize); + RsGenericSerializer::SerializeJob j = + RsGenericSerializer::SerializeJob::DESERIALIZE; + RS_SERIAL_PROCESS(reply); + } + free(resultData); + + if(reply.mType != RsGxsForumsItems::SEARCH_REPLY) + { + // If more types are implemented we would put a switch on mType instead + RS_WARN( "Got search request with unkown type: ", + static_cast(reply.mType) ); + return std::errc::bad_message; + } + + auto event = std::make_shared(); + event->mSearchId = requestId; + event->mSearchResults = reply.mResults; + rsEvents->postEvent(event); + return std::error_condition(); +} + +#else // def RS_DEEP_FORUMS_INDEX + +std::error_condition p3GxsForums::distantSearchRequest( + const std::string&, TurtleRequestId& ) +{ return std::errc::function_not_supported; } + +std::error_condition p3GxsForums::localSearch( + const std::string&, + std::vector& ) +{ return std::errc::function_not_supported; } + +#endif // def RS_DEEP_FORUMS_INDEX + /*static*/ const std::string RsGxsForums::DEFAULT_FORUM_BASE_URL = "retroshare:///forums"; /*static*/ const std::string RsGxsForums::FORUM_URL_NAME_FIELD = diff --git a/libretroshare/src/services/p3gxsforums.h b/libretroshare/src/services/p3gxsforums.h index 97f04cc7b..b499e7b0d 100644 --- a/libretroshare/src/services/p3gxsforums.h +++ b/libretroshare/src/services/p3gxsforums.h @@ -4,8 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012-2014 Robert Fernie * - * Copyright (C) 2018-2020 Gioacchino Mazzurco * - * Copyright (C) 2019-2020 Asociación Civil Altermundi * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -32,6 +32,10 @@ #include "util/rstickevent.h" #include "util/rsdebug.h" +#ifdef RS_DEEP_FORUMS_INDEX +#include "deep_search/forumsindex.hpp" +#endif + class p3GxsForums: public RsGenExchange, public RsGxsForums, public p3Config, public RsTickEvent /* only needed for testing - remove after */ @@ -142,7 +146,34 @@ public: /// @see RsGxsForums std::error_condition setPostKeepForever( const RsGxsGroupId& forumId, const RsGxsMessageId& postId, - bool keepForever ) override; + bool keepForever ) override; + + /// @see RsGxsForums + std::error_condition getContentSummaries( + const RsGxsGroupId& forumId, + const std::set& contentIds, + std::vector& summaries ) override; + + /// @see RsGxsForums + std::error_condition distantSearchRequest( + const std::string& matchString, TurtleRequestId& searchId ) override; + + /// @see RsGxsForums + std::error_condition localSearch( + const std::string& matchString, + std::vector& searchResults ) override; + +#ifdef RS_DEEP_FORUMS_INDEX + /// @see RsNxsObserver + std::error_condition handleDistantSearchRequest( + rs_view_ptr requestData, uint32_t requestSize, + rs_owner_ptr& resultData, uint32_t& resultSize ) override; + + /// @see RsNxsObserver + std::error_condition receiveDistantSearchResult( + const TurtleRequestId requestId, + rs_owner_ptr& resultData, uint32_t& resultSize ) override; +#endif /// implementation of rsGxsGorums /// @@ -155,6 +186,17 @@ public: bool getMsgMetaData(const uint32_t &token, GxsMsgMetaMap& msg_metas) ; +protected: +#ifdef RS_DEEP_FORUMS_INDEX + /** Internal usage + * @param[in] publicOnly if true is passed only results pertaining to + * publicly shared forums are returned + */ + std::error_condition prepareSearchResults( + const std::string& matchString, bool publicOnly, + std::vector& searchResults ); +#endif //def RS_DEEP_FORUMS_INDEX + private: static uint32_t forumsAuthenPolicy(); @@ -189,4 +231,8 @@ bool generateGroup(uint32_t &token, std::string groupName); std::map mKnownForums ; RsMutex mKnownForumsMutex; + +#ifdef RS_DEEP_FORUMS_INDEX + DeepForumsIndex mDeepIndex; +#endif }; diff --git a/libretroshare/src/turtle/turtleclientservice.h b/libretroshare/src/turtle/turtleclientservice.h index 9cbe5763f..55525ae3c 100644 --- a/libretroshare/src/turtle/turtleclientservice.h +++ b/libretroshare/src/turtle/turtleclientservice.h @@ -19,23 +19,25 @@ * along with this program. If not, see . * * * *******************************************************************************/ - -// This class is the parent class for any service that will use the turtle router to distribute its packets. -// Typical representative clients include: -// -// p3ChatService: opens tunnels to distant peers for chatting -// ftServer: searches and open tunnels to distant sources for file transfer -// #pragma once #include #include -#include -#include + +#include "serialiser/rsserial.h" +#include "turtle/rsturtleitem.h" +#include "util/rsdebug.h" struct RsItem; class p3turtle ; +/** This class is the parent class for any service that will use the turtle + * router to distribute its packets. + * Typical representative clients include: + * p3ChatService: opens tunnels to distant peers for chatting + * ftServer: searches and open tunnels to distant sources for file + * transfer + */ class RsTurtleClientService { public: @@ -87,30 +89,35 @@ class RsTurtleClientService std::cerr << "!!!!!! Received Data from turtle router, but the client service is not handling it !!!!!!!!!!" << std::endl ; } - /*! - * \brief receiveSearchRequest - * This method is called by the turtle router to notify the client of a search request in the form generic data. The returned - * result contains the serialised generic result returned by the client. - * - * The turtle router keeps the memory ownership over search_request_data - * - * \param search_request_data generic serialized search data - * \param search_request_data_len length of the serialized search data - * \param search_result_data generic serialized search result data - * \param search_result_data_len length of the serialized search result data - * \param max_allowed_hits max number of hits allowed to be sent back and forwarded - * - * \return true if the search is successful. - */ - virtual bool receiveSearchRequest(unsigned char */*search_request_data*/, - uint32_t /*search_request_data_len*/, - unsigned char *& /*search_result_data*/, - uint32_t& /*search_result_data_len*/, - uint32_t& /* max_allows_hits */) - { - std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ; - return false; - } + /*! + * This method is called by the turtle router to notify the client of a + * search request in the form generic data. + * The returned result contains the serialised generic result returned by the + * client service. + * The turtle router keeps the memory ownership over search_request_data + * \param search_request_data generic serialized search data + * \param search_request_data_len length of the serialized search data + * \param search_result_data generic serialized search result data + * \param search_result_data_len length of the serialized search result data + * \param max_allowed_hits max number of hits allowed to be sent back and + * forwarded + * \return true if matching results are available, false otherwise. + */ + virtual bool receiveSearchRequest( + unsigned char *search_request_data, uint32_t search_request_data_len, + unsigned char *& search_result_data, uint32_t& search_result_data_len, + uint32_t& max_allows_hits ) + { + /* Suppress unused warning this way and not commenting the param names + * so doxygen match documentation against params */ + (void) search_request_data; (void) search_request_data_len; + (void) search_result_data; (void) search_result_data_len; + (void) max_allows_hits; + + RS_WARN( "Received search request from turtle router, but the client " + "is not handling it!" ); + return false; + } /*! * \brief receiveSearchResult diff --git a/retroshare.pri b/retroshare.pri index 8c55efd8f..d0439e5e2 100644 --- a/retroshare.pri +++ b/retroshare.pri @@ -141,6 +141,11 @@ rs_macos10.15:CONFIG -= rs_macos10.11 CONFIG *= no_rs_jsonapi rs_jsonapi:CONFIG -= no_rs_jsonapi +# To enable forums indexing append the following assignation to qmake command +# line "CONFIG+=rs_deep_forums_index" +CONFIG *= no_rs_deep_forums_index +rs_deep_forums_index:CONFIG -= no_rs_deep_forums_index + # To enable channel indexing append the following assignation to qmake command # line "CONFIG+=rs_deep_channels_index" CONFIG *= no_rs_deep_channels_index @@ -561,6 +566,7 @@ rs_webui { DEFINES *= RS_WEBUI } +rs_deep_forums_index:DEFINES *= RS_DEEP_FORUMS_INDEX rs_deep_channels_index:DEFINES *= RS_DEEP_CHANNEL_INDEX rs_deep_files_index:DEFINES *= RS_DEEP_FILES_INDEX From dcb2bee8cc0c7afa8d0a52e3d2e76e82b67b444a Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 25 Feb 2021 11:18:26 +0100 Subject: [PATCH 06/10] Epurate this branch from channels deep search changes --- .../src/deep_search/channelsindex.cpp | 173 ++++++++++-------- .../src/deep_search/channelsindex.hpp | 39 ++-- libretroshare/src/retroshare/rsgxschannels.h | 149 +++++---------- libretroshare/src/rsitems/rsgxschannelitems.h | 65 +------ libretroshare/src/services/p3gxschannels.h | 74 +++++--- .../src/gui/gxschannels/GxsChannelDialog.cpp | 4 +- 6 files changed, 218 insertions(+), 286 deletions(-) diff --git a/libretroshare/src/deep_search/channelsindex.cpp b/libretroshare/src/deep_search/channelsindex.cpp index 86a1e341a..cd1c374fc 100644 --- a/libretroshare/src/deep_search/channelsindex.cpp +++ b/libretroshare/src/deep_search/channelsindex.cpp @@ -1,8 +1,8 @@ /******************************************************************************* * RetroShare full text indexing and search implementation based on Xapian * * * - * Copyright (C) 2018-2021 Gioacchino Mazzurco * - * Copyright (C) 2019-2021 Asociación Civil Altermundi * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License version 3 as * @@ -20,23 +20,16 @@ #include "deep_search/channelsindex.hpp" #include "deep_search/commonutils.hpp" -#include "retroshare/rsinit.h" -#include "util/rsdebuglevel3.h" -/*static*/ std::string DeepChannelsIndex::dbDefaultPath() -{ return RsAccounts::AccountDirectory() + "/deep_channels_xapian_db"; } - -std::error_condition DeepChannelsIndex::search( +uint32_t DeepChannelsIndex::search( const std::string& queryStr, std::vector& results, uint32_t maxResults ) { - RS_DBG3(queryStr); - results.clear(); std::unique_ptr dbPtr( - DeepSearch::openReadOnlyDatabase(mDbPath) ); - if(!dbPtr) return std::errc::bad_file_descriptor; + DeepSearch::openReadOnlyDatabase(dbPath()) ); + if(!dbPtr) return 0; Xapian::Database& db(*dbPtr); @@ -70,13 +63,17 @@ std::error_condition DeepChannelsIndex::search( results.push_back(s); } - return std::error_condition(); + return static_cast(results.size()); } -std::error_condition DeepChannelsIndex::indexChannelGroup( - const RsGxsChannelGroup& chan ) +void DeepChannelsIndex::indexChannelGroup(const RsGxsChannelGroup& chan) { - RS_DBG4(chan); + std::unique_ptr dbPtr( + DeepSearch::openWritableDatabase( + dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); + if(!dbPtr) return; + + Xapian::WritableDatabase& db(*dbPtr); // Set up a TermGenerator that we'll use in indexing. Xapian::TermGenerator termgenerator; @@ -97,8 +94,21 @@ std::error_condition DeepChannelsIndex::indexChannelGroup( termgenerator.increase_termpos(); termgenerator.index_text(chan.mDescription); + RsUrl chanUrl; chanUrl + .setScheme("retroshare").setPath("/channel") + .setQueryKV("id", chan.mMeta.mGroupId.toStdString()); + const std::string idTerm("Q" + chanUrl.toString()); + + chanUrl.setQueryKV("publishTs", std::to_string(chan.mMeta.mPublishTs)); + chanUrl.setQueryKV("name", chan.mMeta.mGroupName); + if(!chan.mMeta.mAuthorId.isNull()) + chanUrl.setQueryKV("authorId", chan.mMeta.mAuthorId.toStdString()); + if(chan.mMeta.mSignFlags) + chanUrl.setQueryKV( "signFlags", + std::to_string(chan.mMeta.mSignFlags) ); + std::string rsLink(chanUrl.toString()); + // store the RS link so we are able to retrive it on matching search - const std::string rsLink(channelIndexId(chan.mMeta.mGroupId)); doc.add_value(URL_VALUENO, rsLink); // Store some fields for display purposes. @@ -107,32 +117,35 @@ std::error_condition DeepChannelsIndex::indexChannelGroup( // We use the identifier to ensure each object ends up in the // database only once no matter how many times we run the // indexer. "Q" prefix is a Xapian convention for unique id term. - const std::string idTerm("Q" + rsLink); doc.add_boolean_term(idTerm); - - mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db) - { db.replace_document(idTerm, doc); } ); - - return std::error_condition(); + db.replace_document(idTerm, doc); } -std::error_condition DeepChannelsIndex::removeChannelFromIndex( - const RsGxsGroupId& grpId ) +void DeepChannelsIndex::removeChannelFromIndex(RsGxsGroupId grpId) { - RS_DBG3(grpId); - // "Q" prefix is a Xapian convention for unique id term. - const std::string idTerm("Q" + channelIndexId(grpId)); - mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db) - { db.delete_document(idTerm); } ); + RsUrl chanUrl; chanUrl + .setScheme("retroshare").setPath("/channel") + .setQueryKV("id", grpId.toStdString()); + std::string idTerm("Q" + chanUrl.toString()); - return std::error_condition(); + std::unique_ptr dbPtr( + DeepSearch::openWritableDatabase( + dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); + if(!dbPtr) return; + + Xapian::WritableDatabase& db(*dbPtr); + db.delete_document(idTerm); } -std::error_condition DeepChannelsIndex::indexChannelPost( - const RsGxsChannelPost& post ) +void DeepChannelsIndex::indexChannelPost(const RsGxsChannelPost& post) { - RS_DBG4(post); + std::unique_ptr dbPtr( + DeepSearch::openWritableDatabase( + dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); + if(!dbPtr) return; + + Xapian::WritableDatabase& db(*dbPtr); // Set up a TermGenerator that we'll use in indexing. Xapian::TermGenerator termgenerator; @@ -147,16 +160,21 @@ std::error_condition DeepChannelsIndex::indexChannelPost( termgenerator.index_text( DeepSearch::timetToXapianDate(post.mMeta.mPublishTs), 1, "D" ); - // Avoid indexing RetroShare-gui HTML tags - const std::string cleanMsg = DeepSearch::simpleTextHtmlExtract(post.mMsg); - termgenerator.index_text( - DeepSearch::simpleTextHtmlExtract(post.mMsg), 1, "XD" ); + // TODO: we should strip out HTML tags instead of skipping indexing + // Avoid indexing HTML + bool isPlainMsg = + post.mMsg[0] != '<' || post.mMsg[post.mMsg.size() - 1] != '>'; + + if(isPlainMsg) + termgenerator.index_text(post.mMsg, 1, "XD"); // Index fields without prefixes for general search. termgenerator.index_text(post.mMeta.mMsgName); - - termgenerator.increase_termpos(); - termgenerator.index_text(cleanMsg); + if(isPlainMsg) + { + termgenerator.increase_termpos(); + termgenerator.index_text(post.mMsg); + } for(const RsGxsFile& attachment : post.mFiles) { @@ -166,50 +184,47 @@ std::error_condition DeepChannelsIndex::indexChannelPost( termgenerator.index_text(attachment.mName); } - // store the RS link so we are able to retrive it on matching search - const std::string rsLink(postIndexId(post.mMeta.mGroupId, post.mMeta.mMsgId)); - doc.add_value(URL_VALUENO, rsLink); - - // Store some fields for display purposes. - doc.set_data(post.mMeta.mMsgName + "\n" + cleanMsg); - // We use the identifier to ensure each object ends up in the // database only once no matter how many times we run the // indexer. - const std::string idTerm("Q" + rsLink); + RsUrl postUrl; postUrl + .setScheme("retroshare").setPath("/channel") + .setQueryKV("id", post.mMeta.mGroupId.toStdString()) + .setQueryKV("msgid", post.mMeta.mMsgId.toStdString()); + std::string idTerm("Q" + postUrl.toString()); + + postUrl.setQueryKV("publishTs", std::to_string(post.mMeta.mPublishTs)); + postUrl.setQueryKV("name", post.mMeta.mMsgName); + postUrl.setQueryKV("authorId", post.mMeta.mAuthorId.toStdString()); + std::string rsLink(postUrl.toString()); + + // store the RS link so we are able to retrive it on matching search + doc.add_value(URL_VALUENO, rsLink); + + // Store some fields for display purposes. + if(isPlainMsg) + doc.set_data(post.mMeta.mMsgName + "\n" + post.mMsg); + else doc.set_data(post.mMeta.mMsgName); + doc.add_boolean_term(idTerm); - - mWriteQueue.push( [idTerm, doc](Xapian::WritableDatabase& db) - { db.replace_document(idTerm, doc); } ); - - return std::error_condition(); + db.replace_document(idTerm, doc); } -std::error_condition DeepChannelsIndex::removeChannelPostFromIndex( - const RsGxsGroupId& grpId, const RsGxsMessageId& msgId ) -{ - RS_DBG3(grpId, msgId); - - std::string idTerm("Q" + postIndexId(grpId, msgId)); - mWriteQueue.push( [idTerm](Xapian::WritableDatabase& db) - { db.delete_document(idTerm); } ); - - return std::error_condition(); -} - -/*static*/ std::string DeepChannelsIndex::channelIndexId(RsGxsGroupId grpId) -{ - RsUrl channelIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL); - channelIndexId.setQueryKV( - RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString() ); - return channelIndexId.toString(); -} - -/*static*/ std::string DeepChannelsIndex::postIndexId( +void DeepChannelsIndex::removeChannelPostFromIndex( RsGxsGroupId grpId, RsGxsMessageId msgId ) { - RsUrl postIndexId(RsGxsChannels::DEFAULT_CHANNEL_BASE_URL); - postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_ID_FIELD, grpId.toStdString()); - postIndexId.setQueryKV(RsGxsChannels::CHANNEL_URL_MSG_ID_FIELD, msgId.toStdString()); - return postIndexId.toString(); + RsUrl postUrl; postUrl + .setScheme("retroshare").setPath("/channel") + .setQueryKV("id", grpId.toStdString()) + .setQueryKV("msgid", msgId.toStdString()); + // "Q" prefix is a Xapian convention for unique id term. + std::string idTerm("Q" + postUrl.toString()); + + std::unique_ptr dbPtr( + DeepSearch::openWritableDatabase( + dbPath(), Xapian::DB_CREATE_OR_OPEN ) ); + if(!dbPtr) return; + + Xapian::WritableDatabase& db(*dbPtr); + db.delete_document(idTerm); } diff --git a/libretroshare/src/deep_search/channelsindex.hpp b/libretroshare/src/deep_search/channelsindex.hpp index b767ce2b4..0a49629d9 100644 --- a/libretroshare/src/deep_search/channelsindex.hpp +++ b/libretroshare/src/deep_search/channelsindex.hpp @@ -1,8 +1,8 @@ /******************************************************************************* * RetroShare full text indexing and search implementation based on Xapian * * * - * Copyright (C) 2018-2021 Gioacchino Mazzurco * - * Copyright (C) 2019-2021 Asociación Civil Altermundi * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2019 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Affero General Public License version 3 as * @@ -24,8 +24,8 @@ #include "util/rstime.h" #include "retroshare/rsgxschannels.h" +#include "retroshare/rsinit.h" #include "util/rsurl.h" -#include "deep_search/commonutils.hpp" struct DeepChannelsSearchResult { @@ -36,34 +36,28 @@ struct DeepChannelsSearchResult struct DeepChannelsIndex { - explicit DeepChannelsIndex(const std::string& dbPath) : - mDbPath(dbPath), mWriteQueue(dbPath) {} - /** * @brief Search indexed GXS groups and messages * @param[in] maxResults maximum number of acceptable search results, 0 for * no limits * @return search results count */ - std::error_condition search( - const std::string& queryStr, - std::vector& results, - uint32_t maxResults = 100 ); + static uint32_t search( const std::string& queryStr, + std::vector& results, + uint32_t maxResults = 100 ); - std::error_condition indexChannelGroup(const RsGxsChannelGroup& chan); + static void indexChannelGroup(const RsGxsChannelGroup& chan); - std::error_condition removeChannelFromIndex(const RsGxsGroupId& grpId); + static void removeChannelFromIndex(RsGxsGroupId grpId); - std::error_condition indexChannelPost(const RsGxsChannelPost& post); + static void indexChannelPost(const RsGxsChannelPost& post); - std::error_condition removeChannelPostFromIndex( - const RsGxsGroupId& grpId, const RsGxsMessageId& msgId ); + static void removeChannelPostFromIndex( + RsGxsGroupId grpId, RsGxsMessageId msgId ); - static std::string dbDefaultPath(); + static uint32_t indexFile(const std::string& path); private: - static std::string channelIndexId(RsGxsGroupId grpId); - static std::string postIndexId(RsGxsGroupId grpId, RsGxsMessageId msgId); enum : Xapian::valueno { @@ -74,7 +68,10 @@ private: BAD_VALUENO = Xapian::BAD_VALUENO }; - const std::string mDbPath; - - DeepSearch::StubbornWriteOpQueue mWriteQueue; + static const std::string& dbPath() + { + static const std::string dbDir = + RsAccounts::AccountDirectory() + "/deep_channels_xapian_db"; + return dbDir; + } }; diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index a91e341a7..421417bdc 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -4,8 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012 Robert Fernie * - * Copyright (C) 2018-2021 Gioacchino Mazzurco * - * Copyright (C) 2019-2021 Asociación Civil Altermundi * + * Copyright (C) 2018-2020 Gioacchino Mazzurco * + * Copyright (C) 2019-2020 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -46,11 +46,6 @@ class RsGxsChannels; extern RsGxsChannels* rsGxsChannels; -/* TODO: At this abstraction level that turtle is used for distant searches - * should be an hidden implementation detail. As of today a bit of those - * implementation details leaks around because we use TurtleRequestId in this - * interface */ - struct RsGxsChannelGroup : RsSerializable, RsGxsGenericGroupData { RsGxsChannelGroup() : mAutoDownload(false) {} @@ -113,38 +108,29 @@ struct RsGxsChannelPost : RsSerializable, RsGxsGenericMsgData enum class RsChannelEventCode: uint8_t { UNKNOWN = 0x00, - NEW_CHANNEL = 0x01, /// emitted when new channel is received - UPDATED_CHANNEL = 0x02, /// emitted when existing channel is updated - NEW_MESSAGE = 0x03, /// new message reeived in a particular channel (group and msg id) - UPDATED_MESSAGE = 0x04, /// existing message has been updated in a particular channel - RECEIVED_PUBLISH_KEY = 0x05, /// publish key for this channel has been received - SUBSCRIBE_STATUS_CHANGED = 0x06, /// subscription for channel mChannelGroupId changed. - READ_STATUS_CHANGED = 0x07, /// existing message has been read or set to unread - - /** Result for the given group id available for the given turtle request id - * @deprecated kept for retrocompatibility with old search system new code - * should use @see DISTANT_SEARCH_RESULT instead */ - RECEIVED_TURTLE_SEARCH_RESULT = 0x08, - - STATISTICS_CHANGED = 0x09, /// stats (nb of supplier friends, how many msgs they have etc) has changed - SYNC_PARAMETERS_UPDATED = 0x0a, /// sync and storage times have changed - NEW_COMMENT = 0x0b, /// new comment arrived/published. mChannelThreadId gives the ID of the commented message - NEW_VOTE = 0x0c, /// new vote arrived/published. mChannelThreadId gives the ID of the votes message comment - DELETED_CHANNEL = 0x0d, /// channel was deleted by auto-cleaning system - DELETED_POST = 0x0e, /// Post deleted (usually by cleaning) - DISTANT_SEARCH_RESULT = 0x0f /// Distant search result received + NEW_CHANNEL = 0x01, // emitted when new channel is received + UPDATED_CHANNEL = 0x02, // emitted when existing channel is updated + NEW_MESSAGE = 0x03, // new message reeived in a particular channel (group and msg id) + UPDATED_MESSAGE = 0x04, // existing message has been updated in a particular channel + RECEIVED_PUBLISH_KEY = 0x05, // publish key for this channel has been received + SUBSCRIBE_STATUS_CHANGED = 0x06, // subscription for channel mChannelGroupId changed. + READ_STATUS_CHANGED = 0x07, // existing message has been read or set to unread + RECEIVED_DISTANT_SEARCH_RESULT = 0x08, // result for the given group id available for the given turtle request id + STATISTICS_CHANGED = 0x09, // stats (nb of supplier friends, how many msgs they have etc) has changed + SYNC_PARAMETERS_UPDATED = 0x0a, // sync and storage times have changed + NEW_COMMENT = 0x0b, // new comment arrived/published. mChannelThreadId gives the ID of the commented message + NEW_VOTE = 0x0c, // new vote arrived/published. mChannelThreadId gives the ID of the votes message comment + DELETED_CHANNEL = 0x0d, // channel was deleted by auto-cleaning system }; struct RsGxsChannelEvent: RsEvent { - RsGxsChannelEvent(): - RsEvent(RsEventType::GXS_CHANNELS), - mChannelEventCode(RsChannelEventCode::UNKNOWN) {} + RsGxsChannelEvent(): RsEvent(RsEventType::GXS_CHANNELS), mChannelEventCode(RsChannelEventCode::UNKNOWN) {} RsChannelEventCode mChannelEventCode; RsGxsGroupId mChannelGroupId; RsGxsMessageId mChannelMsgId; - RsGxsMessageId mChannelThreadId; + RsGxsMessageId mChannelThreadId; ///* @see RsEvent @see RsSerializable void serial_process( RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) override @@ -157,14 +143,13 @@ struct RsGxsChannelEvent: RsEvent } }; -/** This event is used to factor multiple search results notifications in a - * single event.*/ -struct RS_DEPRECATED_FOR(RsGxsChannelDistantSearchResultEvent) -RsGxsChannelSearchResultEvent: RsEvent +// This event is used to factor multiple search results notifications in a single event. + +struct RsGxsChannelSearchResultEvent: RsEvent { RsGxsChannelSearchResultEvent(): RsEvent(RsEventType::GXS_CHANNELS), - mChannelEventCode(RsChannelEventCode::RECEIVED_TURTLE_SEARCH_RESULT) {} + mChannelEventCode(RsChannelEventCode::RECEIVED_DISTANT_SEARCH_RESULT) {} RsChannelEventCode mChannelEventCode; std::map > mSearchResultsMap; @@ -179,29 +164,6 @@ RsGxsChannelSearchResultEvent: RsEvent } }; -/** This event is fired once distant search results are received */ -struct RsGxsChannelDistantSearchResultEvent: RsEvent -{ - RsGxsChannelDistantSearchResultEvent(): - RsEvent(RsEventType::GXS_CHANNELS), - mChannelEventCode(RsChannelEventCode::DISTANT_SEARCH_RESULT) {} - - RsChannelEventCode mChannelEventCode; - TurtleRequestId mSearchId; - std::vector mSearchResults; - - ///* @see RsEvent @see RsSerializable - void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx ) override - { - RsEvent::serial_process(j, ctx); - - RS_SERIAL_PROCESS(mChannelEventCode); - RS_SERIAL_PROCESS(mSearchId); - RS_SERIAL_PROCESS(mSearchResults); - } -}; - class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService { public: @@ -424,15 +386,11 @@ public: * @brief Get channel content summaries * @jsonapi{development} * @param[in] channelId id of the channel of which the content is requested - * @param[in] contentIds ids of requested contents, if empty summaries of - * all messages are reqeusted * @param[out] summaries storage for summaries - * @return success or error details if something failed + * @return false if something failed, true otherwhise */ - virtual std::error_condition getContentSummaries( - const RsGxsGroupId& channelId, - const std::set& contentIds, - std::vector& summaries ) = 0; + virtual bool getContentSummaries( const RsGxsGroupId& channelId, + std::vector& summaries ) = 0; /** * @brief Toggle post read status. Blocking API. @@ -464,23 +422,22 @@ public: virtual bool subscribeToChannel( const RsGxsGroupId& channelId, bool subscribe ) = 0; - /** - * @brief Retrieve statistics about the channel service + /** + * \brief Retrieve statistics about the channel service * @jsonapi{development} - * @param[out] stat storage for statistics - * @return true on success false otherwise - */ - virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0; + * \param[out] stat Statistics structure + * \return + */ + virtual bool getChannelServiceStatistics(GxsServiceStatistic& stat) =0; - /** - * @brief Retrieve statistics about the given channel + /** + * \brief Retrieve statistics about the given channel * @jsonapi{development} - * @param[in] channelId Id of the channel group - * @param[out] stat storage for statistics - * @return true on success false otherwise - */ - virtual bool getChannelStatistics( - const RsGxsGroupId& channelId, GxsGroupStatistic& stat ) =0; + * \param[in] channelId Id of the channel group + * \param[out] stat Statistics structure + * \return + */ + virtual bool getChannelStatistics(const RsGxsGroupId& channelId,GxsGroupStatistic& stat) =0; /// default base URL used for channels links @see exportChannelLink static const std::string DEFAULT_CHANNEL_BASE_URL; @@ -539,18 +496,14 @@ public: std::string& errMsg = RS_DEFAULT_STORAGE_PARAM(std::string) ) = 0; /** - * @brief Search the whole reachable network for matching channels and - * contents + * @brief Search the turtle reachable network for matching channels * @jsonapi{development} - * An @see RsGxsChannelSearchResultEvent is emitted when matching results - * arrives from the network + * An @see RsGxsChannelSearchResultEvent is emitted when matching channels + * arrives from the network * @param[in] matchString string to search into the channels - * @param[out] searchId storage for search id, useful to track search events - * and retrieve search results - * @return success or error details + * @return search id */ - virtual std::error_condition distantSearchRequest( - const std::string& matchString, TurtleRequestId& searchId ) = 0; + virtual TurtleRequestId turtleSearchRequest(const std::string& matchString)=0; /** * @brief Retrieve available search results @@ -580,18 +533,16 @@ public: * @param[out] distantGroup storage for group data * @return false on error, true otherwise */ - virtual bool getDistantSearchResultGroupData( - const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0; + virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& groupId, RsGxsChannelGroup& distantGroup ) = 0; - /** - * @brief Get the status of ongoing search - * @return unknown (probably not even searched), known as a search result, - * data request ongoing and data available - */ - virtual DistantSearchGroupStatus getDistantSearchStatus( - const RsGxsGroupId& group_id ) =0; + /** + * @brief getDistantSearchStatus + * Returns the status of ongoing search: unknown (probably not even searched), known as a search result, + * data request ongoing and data available + */ + virtual DistantSearchGroupStatus getDistantSearchStatus(const RsGxsGroupId& group_id) =0; - /** + /** * @brief Clear accumulated search results * @jsonapi{development} * @param[in] reqId search id diff --git a/libretroshare/src/rsitems/rsgxschannelitems.h b/libretroshare/src/rsitems/rsgxschannelitems.h index 598400966..fa52c8cfc 100644 --- a/libretroshare/src/rsitems/rsgxschannelitems.h +++ b/libretroshare/src/rsitems/rsgxschannelitems.h @@ -3,9 +3,7 @@ * * * libretroshare: retroshare core library * * * - * Copyright (C) 2012 Robert Fernie * - * Copyright (C) 2021 Gioacchino Mazzurco * - * Copyright (C) 2021 Asociación Civil Altermundi * + * Copyright 2012-2012 by Robert Fernie * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -21,7 +19,8 @@ * along with this program. If not, see . * * * *******************************************************************************/ -#pragma once +#ifndef RS_GXS_CHANNEL_ITEMS_H +#define RS_GXS_CHANNEL_ITEMS_H #include @@ -31,23 +30,14 @@ #include "serialiser/rstlvfileitem.h" #include "serialiser/rstlvimage.h" -#include "serialiser/rsserializable.h" + #include "retroshare/rsgxschannels.h" + #include "serialiser/rsserializer.h" + #include "util/rsdir.h" -enum class RsGxsChannelItems : uint8_t -{ - GROUP_ITEM = 0x02, - POST_ITEM = 0x03, - SEARCH_REQUEST = 0x04, - SEARCH_REPLY = 0x05, -}; - -RS_DEPRECATED_FOR(RsGxsChannelItems) const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_GROUP_ITEM = 0x02; - -RS_DEPRECATED_FOR(RsGxsChannelItems) const uint8_t RS_PKT_SUBTYPE_GXSCHANNEL_POST_ITEM = 0x03; class RsGxsChannelGroupItem : public RsGxsGrpItem @@ -89,47 +79,6 @@ public: RsTlvImage mThumbnail; }; -struct RsGxsChannelsSearchRequest : RsSerializable -{ - RsGxsChannelsSearchRequest() : mType(RsGxsChannelItems::SEARCH_REQUEST) {} - - /// Just for easier back and forward compatibility - RsGxsChannelItems mType; - - /// Store search match string - std::string mQuery; - - /// @see RsSerializable - void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx ) override - { - RS_SERIAL_PROCESS(mType); - RS_SERIAL_PROCESS(mQuery); - } - - ~RsGxsChannelsSearchRequest() override = default; -}; - -struct RsGxsChannelsSearchReply : RsSerializable -{ - RsGxsChannelsSearchReply() : mType(RsGxsChannelItems::SEARCH_REPLY) {} - - /// Just for easier back and forward compatibility - RsGxsChannelItems mType; - - /// Results storage - std::vector mResults; - - /// @see RsSerializable - void serial_process( RsGenericSerializer::SerializeJob j, - RsGenericSerializer::SerializeContext& ctx ) override - { - RS_SERIAL_PROCESS(mType); - RS_SERIAL_PROCESS(mResults); - } - - ~RsGxsChannelsSearchReply() override = default; -}; class RsGxsChannelSerialiser : public RsGxsCommentSerialiser { @@ -140,3 +89,5 @@ public: virtual RsItem *create_item(uint16_t service_id,uint8_t item_subtype) const ; }; + +#endif /* RS_GXS_CHANNEL_ITEMS_H */ diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index 58af0b22d..b5fe1426f 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -4,8 +4,7 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012 Robert Fernie * - * Copyright (C) 2018-2021 Gioacchino Mazzurco * - * Copyright (C) 2019-2021 Asociación Civil Altermundi * + * Copyright (C) 2018-2019 Gioacchino Mazzurco * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -23,8 +22,6 @@ *******************************************************************************/ #pragma once -#include -#include #include "retroshare/rsgxschannels.h" #include "services/p3gxscommon.h" @@ -33,7 +30,9 @@ #include "util/rsmemory.h" #include "util/rsdebug.h" #include "util/rstickevent.h" -#include "deep_search/channelsindex.hpp" + +#include +#include // This class is only a helper to parse the channel group service string. @@ -57,11 +56,6 @@ class p3GxsChannels: public RsGenExchange, public RsGxsChannels, public: p3GxsChannels( RsGeneralDataService* gds, RsNetworkExchangeService* nes, RsGixs* gixs ); - - /// @see RsGxsChannels - std::error_condition distantSearchRequest( - const std::string& matchString, TurtleRequestId& searchId ) override; - virtual RsServiceInfo getServiceInfo() override; virtual void service_tick() override; @@ -75,7 +69,7 @@ protected: virtual bool loadList(std::list& loadList) override; // @see p3Config::loadList(std::list&) virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id) override; - + virtual TurtleRequestId turtleSearchRequest(const std::string& match_string) override; virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) override; virtual bool clearDistantSearchResults(TurtleRequestId req) override; virtual bool getDistantSearchResultGroupData(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) override; @@ -118,6 +112,24 @@ virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled) virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory) override; virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory) override; +#ifdef TO_REMOVE + /// @see RsGxsChannels::turtleSearchRequest + virtual bool turtleSearchRequest(const std::string& matchString, + const std::function& multiCallback, + rstime_t maxWait = 300 ) override; + + /// @see RsGxsChannels::turtleChannelRequest + virtual bool turtleChannelRequest( + const RsGxsGroupId& channelId, + const std::function& multiCallback, + rstime_t maxWait = 300 ) override; + + /// @see RsGxsChannels::localSearchRequest + virtual bool localSearchRequest(const std::string& matchString, + const std::function& multiCallback, + rstime_t maxWait = 30 ) override; +#endif + /** * Receive results from turtle search @see RsGenExchange @see RsNxsObserver * @see RsGxsNetService::receiveTurtleSearchResults @@ -203,10 +215,9 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override; const std::set &contentIds, std::vector &comments) override; - /// @see RsGxsChannels - std::error_condition getContentSummaries( + /// Implementation of @see RsGxsChannels::getContentSummaries + bool getContentSummaries( const RsGxsGroupId& channelId, - const std::set& contentIds, std::vector& summaries ) override; /// Implementation of @see RsGxsChannels::getChannelStatistics @@ -286,17 +297,6 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override; virtual bool shareChannelKeys( const RsGxsGroupId& channelId, const std::set& peers ) override; -#ifdef RS_DEEP_CHANNEL_INDEX - /// @see RsNxsObserver - std::error_condition handleDistantSearchRequest( - rs_view_ptr requestData, uint32_t requestSize, - rs_owner_ptr& resultData, uint32_t& resultSize ) override; - - std::error_condition receiveDistantSearchResult( - const TurtleRequestId requestId, - rs_owner_ptr& resultData, uint32_t& resultSize ) override; -#endif - /// Implementation of @see RsGxsChannels::createChannel RS_DEPRECATED_FOR(createChannelV2) bool createChannel(RsGxsChannelGroup& channel) override; @@ -313,6 +313,7 @@ virtual bool ExtraFileRemove(const RsFileHash &hash) override; RS_DEPRECATED_FOR(createVoteV2) bool createVote(RsGxsVote& vote) override; + protected: // Overloaded from GxsTokenQueue for Request callbacks. virtual void handleResponse(uint32_t token, uint32_t req_type @@ -328,6 +329,7 @@ static uint32_t channelsAuthenPolicy(); void request_SpecificSubscribedGroups(const std::list &groups); void load_SubscribedGroups(const uint32_t &token); + void request_SpecificUnprocessedPosts(std::list > &ids); void request_GroupUnprocessedPosts(const std::list &grouplist); void load_unprocessedPosts(uint32_t token); @@ -388,8 +390,26 @@ bool generateGroup(uint32_t &token, std::string groupName); rstime_t mLastDistantSearchNotificationTS; std::map > mSearchResultsToNotify; +#ifdef TO_REMOVE + /** Store search callbacks with timeout*/ + std::map< + TurtleRequestId, + std::pair< + std::function, + std::chrono::system_clock::time_point > + > mSearchCallbacksMap; + RsMutex mSearchCallbacksMapMutex; -#ifdef RS_DEEP_CHANNEL_INDEX - DeepChannelsIndex mDeepIndex; + /** Store distant channels requests callbacks with timeout*/ + std::map< + TurtleRequestId, + std::pair< + std::function, + std::chrono::system_clock::time_point > + > mDistantChannelsCallbacksMap; + RsMutex mDistantChannelsCallbacksMapMutex; + + /// Cleanup mSearchCallbacksMap and mDistantChannelsCallbacksMap + void cleanTimedOutCallbacks(); #endif }; diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index baf715a96..e18a161a3 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -411,9 +411,7 @@ void GxsChannelDialog::clearDistantSearchResults(TurtleRequestId id) TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string) { - TurtleRequestId searchId; - rsGxsChannels->distantSearchRequest(search_string.toStdString(), searchId); - return searchId; + return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ; } bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map& group_infos) From bf8ddf498ec0c85654a5c81f5f5cd6bcb01ba62b Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 23 Mar 2021 23:04:10 +0100 Subject: [PATCH 07/10] Fix Android compilation --- libretroshare/src/deep_search/commonutils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/deep_search/commonutils.cpp b/libretroshare/src/deep_search/commonutils.cpp index c3b9c5342..693bf9924 100644 --- a/libretroshare/src/deep_search/commonutils.cpp +++ b/libretroshare/src/deep_search/commonutils.cpp @@ -117,7 +117,7 @@ std::error_condition StubbornWriteOpQueue::flush( if(maxRemaining > 0) { std::chrono::milliseconds interval( - std::max(50l, maxRemaining*1000/5) ); + std::max(rstime_t(50), maxRemaining*1000/5) ); RS_DBG3( "Cannot acquire database write lock, retrying in:", interval.count(), "ms" ); RsThread::async([this, acceptDelay, callTS, interval]() From 2196505d19707245bf017c73cc5a149168120d8b Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 24 Mar 2021 18:55:41 +0100 Subject: [PATCH 08/10] RsGxsForumsDistantSearchEvent fix event type --- libretroshare/src/retroshare/rsgxsforums.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/retroshare/rsgxsforums.h b/libretroshare/src/retroshare/rsgxsforums.h index 561fe646d..4967d46c5 100644 --- a/libretroshare/src/retroshare/rsgxsforums.h +++ b/libretroshare/src/retroshare/rsgxsforums.h @@ -156,7 +156,7 @@ struct RsGxsForumEvent: RsEvent struct RsGxsForumsDistantSearchEvent: RsEvent { RsGxsForumsDistantSearchEvent(): - RsEvent(RsEventType::GXS_CHANNELS), + RsEvent(RsEventType::GXS_FORUMS), mForumEventCode(RsForumEventCode::DISTANT_SEARCH_RESULT) {} RsForumEventCode mForumEventCode; From 0b5874017472d616a2ca62cb7f9698647f48545a Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Wed, 29 Sep 2021 22:25:04 +0200 Subject: [PATCH 09/10] Properly avoid private forum leak in deep search mGroupFlags & GXS_SERV::FLAG_PRIVACY_PUBLIC is always true for forums even if they are circle restricted, use circle flags to check if it is really public --- libretroshare/src/retroshare/rsgxscircles.h | 5 +++-- libretroshare/src/services/p3gxsforums.cc | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/retroshare/rsgxscircles.h b/libretroshare/src/retroshare/rsgxscircles.h index 32bc8e547..40916f234 100644 --- a/libretroshare/src/retroshare/rsgxscircles.h +++ b/libretroshare/src/retroshare/rsgxscircles.h @@ -4,7 +4,8 @@ * libretroshare: retroshare core library * * * * Copyright (C) 2012-2014 Robert Fernie * - * Copyright (C) 2018-2019 Gioacchino Mazzurco * + * Copyright (C) 2018-2021 Gioacchino Mazzurco * + * Copyright (C) 2019-2021 Asociación Civil Altermundi * * * * This program is free software: you can redistribute it and/or modify * * it under the terms of the GNU Lesser General Public License as * @@ -46,7 +47,7 @@ extern RsGxsCircles* rsGxsCircles; enum class RsGxsCircleType : uint32_t // 32 bit overkill, just for retrocompat { UNKNOWN = 0, /// Used to detect uninizialized values. - PUBLIC = 1, /// Public distribution, based on GxsIds + PUBLIC = 1, /// Public distribution EXTERNAL = 2, /// Restricted to an external circle, based on GxsIds NODES_GROUP = 3, /// Restricted to a group of friend nodes, the administrator of the circle behave as a hub for them diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index 2f85afe59..6cb3ae5c6 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -653,6 +653,8 @@ bool p3GxsForums::createForumV2( forum.mMeta.mSignFlags = GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ | GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED; + /* This flag have always this value even for circle restricted forums due to + * how GXS distribute/verify groups */ forum.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC; forum.mMeta.mCircleId.clear(); @@ -1565,7 +1567,8 @@ std::error_condition p3GxsForums::prepareSearchResults( // Avoid leaking sensitive information to unkown peers if( publicOnly && - !(fMeta.mGroupFlags & GXS_SERV::FLAG_PRIVACY_PUBLIC) ) continue; + ( static_cast(fMeta.mCircleType) != + RsGxsCircleType::PUBLIC ) ) continue; RsGxsSearchResult res; res.mGroupId = forumId; From 62810d32e13694a64263e8ff86454b05dc00cac7 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Tue, 26 Oct 2021 15:39:47 +0200 Subject: [PATCH 10/10] deep_search: fix compilation with old Xapian versions --- libretroshare/src/deep_search/commonutils.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/libretroshare/src/deep_search/commonutils.cpp b/libretroshare/src/deep_search/commonutils.cpp index 693bf9924..e8e1e4b59 100644 --- a/libretroshare/src/deep_search/commonutils.cpp +++ b/libretroshare/src/deep_search/commonutils.cpp @@ -26,6 +26,14 @@ #include "util/rsthreads.h" #include "util/rsdebuglevel0.h" +#ifndef XAPIAN_AT_LEAST +/// Added in Xapian 1.4.2. +#define XAPIAN_AT_LEAST(A,B,C) \ + (XAPIAN_MAJOR_VERSION > (A) || \ + (XAPIAN_MAJOR_VERSION == (A) && \ + (XAPIAN_MINOR_VERSION > (B) || \ + (XAPIAN_MINOR_VERSION == (B) && XAPIAN_REVISION >= (C))))) +#endif namespace DeepSearch { @@ -34,8 +42,17 @@ std::unique_ptr openReadOnlyDatabase( { try { +#if XAPIAN_AT_LEAST(1,3,2) std::unique_ptr dbPtr( new Xapian::Database(path, flags) ); +#else + std::unique_ptr dbPtr(new Xapian::Database(path)); + if(flags) + { + RS_WARN( "Xapian DB flags: ", flags, " ignored due to old Xapian " + "library version: ", XAPIAN_VERSION, " < 1.3.2" ); + } +#endif return dbPtr; } catch(Xapian::DatabaseOpeningError& e)