From 37c515bcb5dff500d5795a9cbe3ca08a5187b710 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 15 Oct 2018 20:57:21 +0200 Subject: [PATCH 1/4] patch from sss to fix deadlocks on forum fill thread. To be tested --- .../gui/gxsforums/GxsForumThreadWidget.cpp | 1 - .../src/gui/gxsforums/GxsForumsFillThread.cpp | 75 ++++++++++++++++++- 2 files changed, 71 insertions(+), 5 deletions(-) diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 2b72f3f39..e050c7ce2 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -1438,7 +1438,6 @@ void GxsForumThreadWidget::insertThreads() GxsForumsFillThread *thread = mFillThread; mFillThread = NULL; thread->stop(); - delete(thread); mStateHelper->setLoading(mTokenTypeInsertThreads, false); } diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumsFillThread.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumsFillThread.cpp index a0228f671..dca70d165 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumsFillThread.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumsFillThread.cpp @@ -74,7 +74,6 @@ void GxsForumsFillThread::stop() disconnect(); mStopped = true; QApplication::processEvents(); - wait(); } void GxsForumsFillThread::calculateExpand(const RsGxsForumMsg &msg, QTreeWidgetItem *item) @@ -125,7 +124,10 @@ void GxsForumsFillThread::run() } if (requestStatus == RsTokenService::FAILED) + { + deleteLater(); return; + } } // also get the forum meta data. @@ -154,7 +156,10 @@ void GxsForumsFillThread::run() } if (requestStatus == RsTokenService::FAILED) + { + deleteLater(); return; + } } if (wasStopped()) @@ -166,6 +171,7 @@ void GxsForumsFillThread::run() /* cancel request */ service->cancelRequest(msg_token); service->cancelRequest(grp_token); + deleteLater(); return; } @@ -174,7 +180,10 @@ void GxsForumsFillThread::run() std::vector forum_groups; if (!rsGxsForums->getGroupData(grp_token, forum_groups) || forum_groups.size() != 1) + { + deleteLater(); return; + } RsGxsForumGroup forum_group = *forum_groups.begin(); @@ -197,7 +206,10 @@ void GxsForumsFillThread::run() std::vector msgs_array; if (!rsGxsForums->getMsgData(msg_token, msgs_array)) + { + deleteLater(); return; + } // now put everything into a map in order to make search log(n) @@ -240,6 +252,12 @@ void GxsForumsFillThread::run() std::list msg_stack ; for ( std::map::iterator msgIt = msgs.begin(); msgIt != msgs.end();++msgIt) + { + if(wasStopped()) + { + deleteLater(); + return; + } if(!msgIt->second.mMeta.mOrigMsgId.isNull() && msgIt->second.mMeta.mOrigMsgId != msgIt->second.mMeta.mMsgId) { #ifdef DEBUG_FORUMS @@ -271,14 +289,26 @@ void GxsForumsFillThread::run() mPostVersions[msgIt->second.mMeta.mOrigMsgId].push_back(QPair(msgIt->second.mMeta.mPublishTs,msgIt->second.mMeta.mMsgId)) ; } + } // The following code assembles all new versions of a given post into the same array, indexed by the oldest version of the post. for(QMap > >::iterator it(mPostVersions.begin());it!=mPostVersions.end();++it) { + if(wasStopped()) + { + deleteLater(); + return; + } QVector >& v(*it) ; for(int32_t i=0;i::iterator msgIt = msgs.begin(); msgIt != msgs.end();++msgIt) { + if (wasStopped()) + { + deleteLater(); + return; + } if(mFlatView || msgIt->second.mMeta.mParentId.isNull()) { /* add all threads */ if (wasStopped()) + { + deleteLater(); return; + } const RsGxsForumMsg& msg = msgIt->second; @@ -422,7 +478,13 @@ void GxsForumsFillThread::run() while (!threadStack.empty()) { - std::pair threadPair = threadStack.front(); + if (wasStopped()) + { + deleteLater(); + return; + } + + std::pair threadPair = threadStack.front(); threadStack.pop_front(); std::map >::iterator it = kids_array.find(threadPair.first) ; @@ -433,11 +495,14 @@ void GxsForumsFillThread::run() if(it == kids_array.end()) continue ; - if (wasStopped()) - return; for(std::list::const_iterator it2(it->second.begin());it2!=it->second.end();++it2) { + if(wasStopped()) + { + deleteLater(); + return; + } // We iterate through the top level thread items, and look for which message has the current item as parent. // When found, the item is put in the thread list itself, as a potential new parent. @@ -485,6 +550,8 @@ void GxsForumsFillThread::run() std::cerr << "GxsForumsFillThread::run() stopped: " << (wasStopped() ? "yes" : "no") << std::endl; #endif + if(wasStopped()) + deleteLater(); } From 945775e9f52f21758bbe38f5e861265688b3593f Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Thu, 18 Oct 2018 04:21:23 +0200 Subject: [PATCH 2/4] Fix deep search compiling on 32bit time_t targets --- libretroshare/src/deep_search/deep_search.h | 3 ++- libretroshare/src/gxs/rsgxsnetservice.cc | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/deep_search/deep_search.h b/libretroshare/src/deep_search/deep_search.h index e9cb84b8b..5d2361342 100644 --- a/libretroshare/src/deep_search/deep_search.h +++ b/libretroshare/src/deep_search/deep_search.h @@ -264,7 +264,8 @@ private: static std::string timetToXapianDate(const rstime_t& time) { char date[] = "YYYYMMDD\0"; - std::strftime(date, 9, "%Y%m%d", std::gmtime(&time)); + time_t tTime = static_cast(time); + std::strftime(date, 9, "%Y%m%d", std::gmtime(&tTime)); return date; } }; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 02bdb1b9b..cb73e06a7 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5140,11 +5140,13 @@ TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_st return mGxsNetTunnel->turtleSearchRequest(match_string,this) ; } +#ifndef RS_DEEP_SEARCH static bool termSearch(const std::string& src, const std::string& substring) { /* always ignore case */ return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() ); } +#endif // ndef RS_DEEP_SEARCH bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map& group_infos) { From 9e2c4ce49e845a9d6505fe6b5bd5db8af22b7156 Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Fri, 19 Oct 2018 15:10:15 +0200 Subject: [PATCH 3/4] JSON API authorize token in createLocation Solve uncoherent 401 behaviour at first use without login --- libretroshare/src/jsonapi/jsonapi.cpp | 48 +++++++++++++++++++++++++++ libretroshare/src/retroshare/rsinit.h | 10 +++--- libretroshare/src/rsserver/rsinit.cc | 2 +- 3 files changed, 54 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/jsonapi/jsonapi.cpp b/libretroshare/src/jsonapi/jsonapi.cpp index 0a293de93..ac398f92f 100644 --- a/libretroshare/src/jsonapi/jsonapi.cpp +++ b/libretroshare/src/jsonapi/jsonapi.cpp @@ -118,6 +118,54 @@ JsonApiServer::JsonApiServer(uint16_t port, const std::string& bindAddress, mNewAccessRequestCallback(newAccessRequestCallback), configMutex("JsonApiServer config") { + registerHandler("/rsLoginHelper/createLocation", + [this](const std::shared_ptr session) + { + size_t reqSize = session->get_request()->get_header("Content-Length", 0); + session->fetch( reqSize, [this]( + const std::shared_ptr session, + const rb::Bytes& body ) + { + INITIALIZE_API_CALL_JSON_CONTEXT; + + RsLoginHelper::Location location; + std::string password; + std::string errorMessage; + bool makeHidden = false; + bool makeAutoTor = false; + + // deserialize input parameters from JSON + { + RsGenericSerializer::SerializeContext& ctx(cReq); + RsGenericSerializer::SerializeJob j(RsGenericSerializer::FROM_JSON); + RS_SERIAL_PROCESS(location); + RS_SERIAL_PROCESS(password); + RS_SERIAL_PROCESS(makeHidden); + RS_SERIAL_PROCESS(makeAutoTor); + } + + // call retroshare C++ API + bool retval = rsLoginHelper->createLocation( + location, password, errorMessage, makeHidden, + makeAutoTor ); + + if(retval) + authorizeToken(location.mLocationId.toStdString()+":"+password); + + // serialize out parameters and return value to JSON + { + RsGenericSerializer::SerializeContext& ctx(cAns); + RsGenericSerializer::SerializeJob j(RsGenericSerializer::TO_JSON); + RS_SERIAL_PROCESS(location); + RS_SERIAL_PROCESS(errorMessage); + RS_SERIAL_PROCESS(retval); + } + + // return them to the API caller + DEFAULT_API_CALL_JSON_RETURN(rb::OK); + } ); + }, false); + registerHandler("/rsLoginHelper/attemptLogin", [this](const std::shared_ptr session) { diff --git a/libretroshare/src/retroshare/rsinit.h b/libretroshare/src/retroshare/rsinit.h index 5dfade3f8..f81bfaddf 100644 --- a/libretroshare/src/retroshare/rsinit.h +++ b/libretroshare/src/retroshare/rsinit.h @@ -292,20 +292,20 @@ struct RsLoginHelper /** * @brief Creates a new RetroShare location, and log in once is created - * @jsonapi{development,unauthenticated} + * @jsonapi{development,manualwrapper} * @param[inout] location provide input information to generate the location * and storage to output the data of the generated location * @param[in] password to protect and unlock the associated PGP key + * @param[out] errorMessage if some error occurred human readable error + * message * @param[in] makeHidden pass true to create an hidden location. UNTESTED! * @param[in] makeAutoTor pass true to create an automatically configured * Tor hidden location. UNTESTED! - * @param[out] errorMessage if some error occurred human readable error - * message * @return true if success, false otherwise */ bool createLocation( RsLoginHelper::Location& location, - const std::string& password, bool makeHidden, - bool makeAutoTor, std::string& errorMessage ); + const std::string& password, std::string& errorMessage, + bool makeHidden = false, bool makeAutoTor = false ); /** * @brief Check if RetroShare is already logged in, this usually return true diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index fe2f4c991..e3f1f141e 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1952,7 +1952,7 @@ void RsLoginHelper::getLocations(std::vector& store) bool RsLoginHelper::createLocation( RsLoginHelper::Location& l, const std::string& password, - bool makeHidden, bool makeAutoTor, std::string& errorMessage ) + std::string& errorMessage, bool makeHidden, bool makeAutoTor ) { if(isLoggedIn()) return (errorMessage="Already Running", false); From 2d497d730338f7687f135665e67e2f235317c33f Mon Sep 17 00:00:00 2001 From: Gioacchino Mazzurco Date: Sat, 20 Oct 2018 14:07:39 +0200 Subject: [PATCH 4/4] Fix compilation with deep search and rm dead code --- libretroshare/src/services/p3gxschannels.cc | 95 ++------------------- 1 file changed, 7 insertions(+), 88 deletions(-) diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index da490ee1e..3897cbb2a 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -1045,119 +1045,38 @@ bool p3GxsChannels::getChannelsContent( bool p3GxsChannels::createChannel(RsGxsChannelGroup& channel) { uint32_t token; - time_t beginCreation = time(nullptr); - if( !createGroup(token, channel) || waitToken(token) != RsTokenService::COMPLETE ) + if( !createGroup(token, channel) + || waitToken(token) != RsTokenService::COMPLETE ) return false; - if(RsGenExchange::getPublishedGroupMeta(token,channel.mMeta)) + if(RsGenExchange::getPublishedGroupMeta(token, channel.mMeta)) { #ifdef RS_DEEP_SEARCH - if(found) DeepSearch::indexChannelGroup(channel); + DeepSearch::indexChannelGroup(channel); #endif // RS_DEEP_SEARCH return true; } - else - return false; -// time_t endCreation = time(nullptr); -// -// -// std::list channels; -// if(!getChannelsSummaries(channels)) return false; -// -// /* This is ugly but after digging and doing many tries of doing it the right -// * way ending always into too big refactor chain reaction, I think this is -// * not that bad, moreover seems the last created group tend to end up near -// * the beginning of the list so it is fast founding it. -// * The shortcoming of this is that if groups with same data are created in -// * a burst (more then once in a second) is that the id of another similar -// * group can be returned, but this is a pointy case. -// * Order of conditions in the `if` matter for performances */ -// bool found = false; -// for(const RsGroupMetaData& chan : channels) -// { -// if( IS_GROUP_ADMIN(chan.mSubscribeFlags) -// && IS_GROUP_SUBSCRIBED(chan.mSubscribeFlags) -// && chan.mPublishTs >= beginCreation -// && chan.mPublishTs <= endCreation -// && chan.mGroupFlags == channel.mMeta.mGroupFlags -// && chan.mSignFlags == channel.mMeta.mSignFlags -// && chan.mCircleType == channel.mMeta.mCircleType -// && chan.mAuthorId == channel.mMeta.mAuthorId -// && chan.mCircleId == channel.mMeta.mCircleId -// && chan.mServiceString == channel.mMeta.mServiceString -// && chan.mGroupName == channel.mMeta.mGroupName ) -// { -// channel.mMeta = chan; -// found = true; -// break; -// } -// } + return false; } bool p3GxsChannels::createPost(RsGxsChannelPost& post) { uint32_t token; - time_t beginCreation = time(nullptr); if( !createPost(token, post) || waitToken(token) != RsTokenService::COMPLETE ) return false; - time_t endCreation = time(nullptr); if(RsGenExchange::getPublishedMsgMeta(token,post.mMeta)) { #ifdef RS_DEEP_SEARCH - if(found) DeepSearch::indexChannelGroup(post); + DeepSearch::indexChannelPost(post); #endif // RS_DEEP_SEARCH return true; } - else - return false; -// std::list chanIds; chanIds.push_back(post.mMeta.mGroupId); -// std::vector posts; -// std::vector comments; -// if(!getChannelsContent(chanIds, posts, comments)) return false; -// -// /* This is ugly but after digging and doing many tries of doing it the right -// * way ending always into too big refactor chain reaction, I think this is -// * not that bad. -// * The shortcoming of this is that if posts with same data are created in -// * a burst (more then once in a second) is that the id of another similar -// * post could be returned, but this is a pointy case. -// * Order of conditions in the `if` matter for performances */ -// bool found = false; -// for(const RsGxsChannelPost& itPost : posts) -// { -// std::cout << __PRETTY_FUNCTION__ << " " << beginCreation << " " -// << itPost.mMeta.mPublishTs << " " << endCreation << " " -// << itPost.mMeta.mMsgId << std::endl; -// -// if( itPost.mMeta.mPublishTs >= beginCreation -// && itPost.mMeta.mPublishTs <= endCreation -// && itPost.mMeta.mMsgFlags == post.mMeta.mMsgFlags -// && itPost.mMeta.mGroupId == post.mMeta.mGroupId -// && itPost.mMeta.mThreadId == post.mMeta.mThreadId -// && itPost.mMeta.mParentId == post.mMeta.mParentId -// && itPost.mMeta.mAuthorId == post.mMeta.mAuthorId -// && itPost.mMeta.mMsgName == post.mMeta.mMsgName -// && itPost.mFiles.size() == post.mFiles.size() -// && itPost.mMeta.mServiceString == post.mMeta.mServiceString -// && itPost.mOlderVersions == post.mOlderVersions -// && itPost.mMsg == post.mMsg ) -// { -// post = itPost; -// found = true; -// break; -// } -// } -// -//#ifdef RS_DEEP_SEARCH -// if(found) DeepSearch::indexChannelPost(post); -//#endif // RS_DEEP_SEARCH -// -// return found; + return false; }