From ba500284c7f2b980599641f7369d5a0fea7128ea Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 30 Jun 2016 14:16:45 -0400 Subject: [PATCH 01/18] added GXS distribution flag to defavor posts signed by key form unknown RS nodes, and updated UI in forums to use it --- libretroshare/src/gxs/rsgenexchange.cc | 11 ++- libretroshare/src/retroshare/rsgxsflags.h | 2 + retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp | 74 ++++++++------ retroshare-gui/src/gui/gxs/GxsGroupDialog.h | 5 +- retroshare-gui/src/gui/gxs/GxsGroupDialog.ui | 98 +++++++------------ .../src/gui/gxsforums/CreateGxsForumMsg.cpp | 2 +- .../gui/gxsforums/GxsForumThreadWidget.cpp | 3 +- 7 files changed, 96 insertions(+), 99 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 615c96e3e..e4c83d5db 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -881,8 +881,15 @@ int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, const uin { // now check reputation of the message author - float reputation_threshold = ( (signFlag & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) && !(details.mFlags & RS_IDENTITY_FLAGS_PGP_LINKED)) ? (RsReputations::REPUTATION_THRESHOLD_ANTI_SPAM): (RsReputations::REPUTATION_THRESHOLD_DEFAULT) ; - + float reputation_threshold = RsReputations::REPUTATION_THRESHOLD_DEFAULT; + + if( (signFlag & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG_KNOWN) && !(details.mFlags & RS_IDENTITY_FLAGS_PGP_KNOWN)) + reputation_threshold = RsReputations::REPUTATION_THRESHOLD_ANTI_SPAM; + else if( (signFlag & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) && !(details.mFlags & RS_IDENTITY_FLAGS_PGP_LINKED)) + reputation_threshold = RsReputations::REPUTATION_THRESHOLD_ANTI_SPAM; + else + reputation_threshold = RsReputations::REPUTATION_THRESHOLD_DEFAULT; + if(details.mReputation.mOverallReputationScore < reputation_threshold) { #ifdef GEN_EXCH_DEBUG diff --git a/libretroshare/src/retroshare/rsgxsflags.h b/libretroshare/src/retroshare/rsgxsflags.h index c9b92c6f1..6233fc1fd 100644 --- a/libretroshare/src/retroshare/rsgxsflags.h +++ b/libretroshare/src/retroshare/rsgxsflags.h @@ -34,6 +34,7 @@ namespace GXS_SERV { static const uint32_t FLAG_AUTHOR_AUTHENTICATION_REQUIRED = 0x00000200; static const uint32_t FLAG_AUTHOR_AUTHENTICATION_IFNOPUBSIGN = 0x00000400; static const uint32_t FLAG_AUTHOR_AUTHENTICATION_TRACK_MESSAGES = 0x00000800; + static const uint32_t FLAG_AUTHOR_AUTHENTICATION_GPG_KNOWN = 0x00001000; static const uint32_t FLAG_GROUP_SIGN_PUBLISH_MASK = 0x000000ff; static const uint32_t FLAG_GROUP_SIGN_PUBLISH_ENCRYPTED = 0x00000001; @@ -105,6 +106,7 @@ namespace GXS_SERV { #define IS_MSG_UNPROCESSED(status) (status & GXS_SERV::GXS_MSG_STATUS_UNPROCESSED) #define IS_GROUP_PGP_AUTHED(signFlags) (signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) +#define IS_GROUP_PGP_KNOWN_AUTHED(signFlags) (signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG_KNOWN) #define IS_GROUP_MESSAGE_TRACKING(signFlags) (signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_TRACK_MESSAGES) #define IS_GROUP_ADMIN(subscribeFlags) (subscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_ADMIN) diff --git a/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp index f519cb4bf..8c451e3da 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp @@ -302,7 +302,13 @@ void GxsGroupDialog::setupDefaults() } } ui.antiSpam_trackMessages->setChecked((bool)(mDefaultsFlags & GXS_GROUP_DEFAULTS_ANTISPAM_TRACK)); - ui.antiSpam_signedIds->setChecked((bool)(mDefaultsFlags & GXS_GROUP_DEFAULTS_ANTISPAM_FAVOR_PGP)); + + if( (mDefaultsFlags & GXS_GROUP_DEFAULTS_ANTISPAM_FAVOR_PGP) && (mDefaultsFlags & GXS_GROUP_DEFAULTS_ANTISPAM_FAVOR_PGP_KNOWN)) + ui.antiSpam_perms_CB->setCurrentIndex(2) ; + else if(mDefaultsFlags & GXS_GROUP_DEFAULTS_ANTISPAM_FAVOR_PGP) + ui.antiSpam_perms_CB->setCurrentIndex(1) ; + else + ui.antiSpam_perms_CB->setCurrentIndex(0) ; QString antispam_string ; if(mDefaultsFlags & GXS_GROUP_DEFAULTS_ANTISPAM_TRACK) antispam_string += tr("Message tracking") ; @@ -613,32 +619,37 @@ void GxsGroupDialog::createGroup() uint32_t GxsGroupDialog::getGroupSignFlags() { - /* grab from the ui options -> */ - uint32_t signFlags = 0; - if (ui.publish_encrypt->isChecked()) { - signFlags |= GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_ENCRYPTED; - } else if (ui.publish_required->isChecked()) { - signFlags |= GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_ALLSIGNED; - } else if (ui.publish_threads->isChecked()) { - signFlags |= GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_THREADHEAD; - } else { // publish_open (default). - signFlags |= GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ; - } + /* grab from the ui options -> */ + uint32_t signFlags = 0; + if (ui.publish_encrypt->isChecked()) { + signFlags |= GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_ENCRYPTED; + } else if (ui.publish_required->isChecked()) { + signFlags |= GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_ALLSIGNED; + } else if (ui.publish_threads->isChecked()) { + signFlags |= GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_THREADHEAD; + } else { // publish_open (default). + signFlags |= GXS_SERV::FLAG_GROUP_SIGN_PUBLISH_NONEREQ; + } - if (ui.personal_required->isChecked()) - signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED; - - if (ui.personal_ifnopub->isChecked()) - signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_IFNOPUBSIGN; - - // Author Signature. - if (ui.antiSpam_signedIds->isChecked()) - signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG; - - if (ui.antiSpam_trackMessages->isChecked()) - signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_TRACK_MESSAGES; - - return signFlags; + if (ui.personal_required->isChecked()) + signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_REQUIRED; + + if (ui.personal_ifnopub->isChecked()) + signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_IFNOPUBSIGN; + + // Author Signature. + switch(ui.antiSpam_perms_CB->currentIndex()) + { + case 0: break ; + case 2: signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG_KNOWN; // no break below, since we want *both* flags in this case. + case 1: signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG; + break ; + } + + if (ui.antiSpam_trackMessages->isChecked()) + signFlags |= GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_TRACK_MESSAGES; + + return signFlags; } void GxsGroupDialog::setGroupSignFlags(uint32_t signFlags) @@ -660,11 +671,18 @@ void GxsGroupDialog::setGroupSignFlags(uint32_t signFlags) ui.personal_ifnopub->setChecked(true); ui.antiSpam_trackMessages ->setChecked((bool)(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_TRACK_MESSAGES) ); - ui.antiSpam_signedIds ->setChecked((bool)(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) ); + if( (signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG_KNOWN) && (signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG)) + ui.antiSpam_perms_CB->setCurrentIndex(2) ; + else if(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) + ui.antiSpam_perms_CB->setCurrentIndex(1) ; + else + ui.antiSpam_perms_CB->setCurrentIndex(0) ; + QString antispam_string ; if(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_TRACK_MESSAGES) antispam_string += tr("Message tracking") ; - if(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) antispam_string += (antispam_string.isNull()?"":" and ")+tr("PGP signature required") ; + if(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG_KNOWN) antispam_string += (antispam_string.isNull()?"":" and ")+tr("PGP signature from known ID required") ; + if(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) antispam_string += (antispam_string.isNull()?"":" and ")+tr("PGP signature required") ; ui.antiSpamValueLabel->setText(antispam_string) ; //ui.antiSpam_trackMessages_2->setChecked((bool)(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_TRACK_MESSAGES) ); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupDialog.h index 7f94c32db..b71a39952 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupDialog.h @@ -90,8 +90,9 @@ public: #define GXS_GROUP_DEFAULTS_COMMENTS_YES 0x00001000 #define GXS_GROUP_DEFAULTS_COMMENTS_NO 0x00002000 -#define GXS_GROUP_DEFAULTS_ANTISPAM_FAVOR_PGP 0x00100000 -#define GXS_GROUP_DEFAULTS_ANTISPAM_TRACK 0x00200000 +#define GXS_GROUP_DEFAULTS_ANTISPAM_FAVOR_PGP 0x00100000 +#define GXS_GROUP_DEFAULTS_ANTISPAM_TRACK 0x00200000 +#define GXS_GROUP_DEFAULTS_ANTISPAM_FAVOR_PGP_KNOWN 0x00400000 /*! * The aim of this dialog is to be convenient to encapsulate group diff --git a/retroshare-gui/src/gui/gxs/GxsGroupDialog.ui b/retroshare-gui/src/gui/gxs/GxsGroupDialog.ui index eb5228c81..790897d7c 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupDialog.ui +++ b/retroshare-gui/src/gui/gxs/GxsGroupDialog.ui @@ -6,32 +6,23 @@ 0 0 - 828 - 612 + 935 + 785 Create New - - 0 - - - 0 - - - 0 - - - 0 - 6 0 + + 0 + @@ -57,16 +48,7 @@ 0 - - 4 - - - 4 - - - 4 - - + 4 @@ -130,16 +112,7 @@ 9 - - 3 - - - 3 - - - 3 - - + 3 @@ -261,16 +234,7 @@ - - 0 - - - 0 - - - 0 - - + 0 @@ -580,35 +544,39 @@ Spam-protection - - 6 - - - 9 - - - 4 - - - 9 - - - 4 - - - - <html><head/><body><p>This makes the media increase the reputation threshold to 0.4 for anonymous ids, while keeping it to 0.0 for PGP-linked ids. Therefore, anonymous ids can still post, if their local reputation score is above that threshold.</p></body></html> - + - Favor PGP-signed ids + Posts permissions: + + + + <html><head/><body><p>This combo box allows you to choose how posts are handled depending on the node the poster belongs to.</p><p><span style=" font-weight:600;">All allowed</span>: all posts are treated equally.</p><p><span style=" font-weight:600;">Defavor posts from unsigned IDs</span>: anonymous IDs will require a reputation of 0.4 to be received/forwarded.</p><p><span style=" font-weight:600;">Defavor posts from unsigned IDs and IDs from unknown nodes</span>: anonymous IDs and IDs signed by unknown Retroshare nodes will require a reputation of 0.4 to be received/forwarded.</p></body></html> + + + + All allowed + + + + + Defavor unsigned IDs + + + + + Defavor unsigned IDs and IDs from unknown nodes + + + + - <html><head/><body><p align="justify">This feature allows Retroshare to locally keep a record of who forwarded each message to you, for the last 10 days. Although useless if alone (and already available whatsoever) this information can be used by a group of collaborative friends to easily locate the source of spams. To be used with care, since it significantly decreases the anonymity of message posts.</p></body></html> + <html><head/><body><p align="justify">This feature allows Retroshare to locally keep a record of who forwarded each message to you, for the last 10 days. Although useless if alone (and already available whatsoever) this information can be used by a group of teamed-up friends to easily locate a source of spams. To be used with care, since it significantly decreases the anonymity of message posts.</p></body></html> Keep track of posts diff --git a/retroshare-gui/src/gui/gxsforums/CreateGxsForumMsg.cpp b/retroshare-gui/src/gui/gxsforums/CreateGxsForumMsg.cpp index 6898ce228..f73aeef91 100644 --- a/retroshare-gui/src/gui/gxsforums/CreateGxsForumMsg.cpp +++ b/retroshare-gui/src/gui/gxsforums/CreateGxsForumMsg.cpp @@ -212,7 +212,7 @@ void CreateGxsForumMsg::loadFormInformation() //std::cerr << "CreateGxsForumMsg::loadMsgInformation() using signFlags=" << std::hex << mForumMeta.mSignFlags << std::dec << std::endl; - if(mForumMeta.mSignFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) + if( (mForumMeta.mSignFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) || (mForumMeta.mSignFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG_KNOWN)) ui.idChooser->setFlags(IDCHOOSER_ID_REQUIRED | IDCHOOSER_NON_ANONYMOUS) ; else ui.idChooser->setFlags(IDCHOOSER_ID_REQUIRED) ; diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 82ed1e6dd..84dda7516 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -791,7 +791,8 @@ void GxsForumThreadWidget::insertGroupData() tw->ui->forumName->setText(QString::fromUtf8(group.mMeta.mGroupName.c_str())); QString anti_spam_features1 ; - if(IS_GROUP_PGP_AUTHED(tw->mSignFlags)) anti_spam_features1 = tr("Anonymous IDs reputation threshold set to 0.4"); + if(IS_GROUP_PGP_KNOWN_AUTHED(tw->mSignFlags)) anti_spam_features1 = tr("Anonymous/unknown node IDs reputation threshold set to 0.4"); + else if(IS_GROUP_PGP_AUTHED(tw->mSignFlags)) anti_spam_features1 = tr("Anonymous IDs reputation threshold set to 0.4"); QString anti_spam_features2 ; if(IS_GROUP_MESSAGE_TRACKING(tw->mSignFlags)) anti_spam_features2 = tr("Message routing info kept for 10 days"); From ef954514bef4a9c93ec953fcc6452e889fe36b7f Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 28 Jun 2016 21:16:31 -0400 Subject: [PATCH 02/18] updated changelog --- build_scripts/Debian+Ubuntu/changelog | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build_scripts/Debian+Ubuntu/changelog b/build_scripts/Debian+Ubuntu/changelog index 31a9a39fa..22357aac9 100644 --- a/build_scripts/Debian+Ubuntu/changelog +++ b/build_scripts/Debian+Ubuntu/changelog @@ -1,5 +1,11 @@ retroshare06 (0.6.0-1.XXXXXX~YYYYYY) YYYYYY; urgency=low + 9f7ef8b csoler Tue, 28 Jun 2016 20:59:56 -0400 fixed possible cross deadlocks between RsGxsGenExchange and RsGxsNetService + 00bdc50 csoler Tue, 28 Jun 2016 17:56:37 -0400 updated changelog + bb82b6e csoler Tue, 28 Jun 2016 14:35:59 -0400 Merge pull request #429 from PhenomRetroShare/Fix_StandardQSS + 456751a Phenom Tue, 28 Jun 2016 20:23:32 +0200 Fix ProgressBar Chunk with Standard.qss + 019ed12 csoler Tue, 28 Jun 2016 12:37:04 -0400 Merge pull request #428 from sehraf/pr-fix-udp + ea07a13 sehraf Tue, 28 Jun 2016 17:37:22 +0200 fix TcpPacket::readPacket c15c979 csoler Mon, 27 Jun 2016 17:17:28 +0100 Merge pull request #426 from sehraf/pr-disable-dht-stunner ff86678 csoler Sun, 26 Jun 2016 23:14:01 -0400 clear error queue before calling SSL_read()/SSL_write(), to avoid remaining errors to corrupt the next connection attempts and cause disconnections 31089d7 csoler Sat, 25 Jun 2016 19:12:35 +0100 added auto-clean of pending data in pqistreamer after connection is closed, to avoid confusing the peer after re-connecting (probably not the cause of the SSL error) From 2dfe116a31a8982bbcf53856aa5e0057d1922d64 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 30 Jun 2016 14:21:54 -0400 Subject: [PATCH 03/18] added compilation flag for profiling in GUI --- retroshare-gui/src/retroshare-gui.pro | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index 69278042a..e4f1fa73c 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -14,6 +14,13 @@ DEFINES += TARGET=\\\"$TARGET\\\" #CONFIG += debug #DEFINES *= SIGFPE_DEBUG +profiling { + QMAKE_CXXFLAGS -= -fomit-frame-pointer + QMAKE_CXXFLAGS *= -pg -g -fno-omit-frame-pointer + QMAKE_LFLAGS *= -pg +} + + #QMAKE_CFLAGS += -fmudflap #LIBS *= /usr/lib/gcc/x86_64-linux-gnu/4.4/libmudflap.a /usr/lib/gcc/x86_64-linux-gnu/4.4/libmudflapth.a @@ -69,7 +76,7 @@ linux-* { PKGCONFIG *= x11 xscrnsaver - LIBS *= -rdynamic + LIBS *= -rdynamic DEFINES *= HAVE_XSS # for idle time, libx screensaver extensions DEFINES *= HAS_GNOME_KEYRING } From 0206c652df5dba5b3758c7958379be8098ace56c Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 30 Jun 2016 15:06:41 -0400 Subject: [PATCH 04/18] added missing else in display of forum anti-spam message --- retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp index 8c451e3da..049566aad 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp @@ -682,6 +682,7 @@ void GxsGroupDialog::setGroupSignFlags(uint32_t signFlags) QString antispam_string ; if(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_TRACK_MESSAGES) antispam_string += tr("Message tracking") ; if(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG_KNOWN) antispam_string += (antispam_string.isNull()?"":" and ")+tr("PGP signature from known ID required") ; + else if(signFlags & GXS_SERV::FLAG_AUTHOR_AUTHENTICATION_GPG) antispam_string += (antispam_string.isNull()?"":" and ")+tr("PGP signature required") ; ui.antiSpamValueLabel->setText(antispam_string) ; From da4c563223300032386533fb7806fd367b51f683 Mon Sep 17 00:00:00 2001 From: Gio Date: Thu, 30 Jun 2016 22:03:22 +0200 Subject: [PATCH 05/18] Fix typo in libresapi documentation --- libresapi/src/api/ApiServer.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libresapi/src/api/ApiServer.cpp b/libresapi/src/api/ApiServer.cpp index 6db36f96f..34d1cfc5c 100644 --- a/libresapi/src/api/ApiServer.cpp +++ b/libresapi/src/api/ApiServer.cpp @@ -150,10 +150,10 @@ want generic resource caching mechanism - dynamic handlers contain result data, and thus get deleted after a while it is even possible to implement a resource-changed check at the highest level -this allows to compute everything on the server side and only send chanes to the client +this allows to compute everything on the server side and only send changes to the client the different resource providers don't have to implement a resource changed check then a top level change detector will poll them -of course this does not wokr with a deep resource tree with millions of nodes +of course this does not work with a deep resource tree with millions of nodes for this we have the dynamic handlers, they are created on demand and know how to listen for changes which affect them From 86d8f01033debc7fd6af10cbf9626a19f3532961 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 1 Jul 2016 23:38:10 -0400 Subject: [PATCH 06/18] added a local cache in rsdataservice to avoid calling sqlcipher on GrpMetaData --- libretroshare/src/gxs/rsdataservice.cc | 139 ++++++++++++++++++------- libretroshare/src/gxs/rsdataservice.h | 7 ++ 2 files changed, 107 insertions(+), 39 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index c0fe58f4d..f8bc384de 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -27,6 +27,7 @@ /***** * #define RS_DATA_SERVICE_DEBUG 1 * #define RS_DATA_SERVICE_DEBUG_TIME 1 + * #define RS_DATA_SERVICE_DEBUG_CACHE 1 ****/ #include @@ -123,6 +124,7 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d : RsGeneralDataService(), mDbMutex("RsDataService"), mServiceDir(serviceDir), mDbName(dbName), mDbPath(mServiceDir + "/" + dbName), mServType(serviceType), mDb(NULL) { bool isNewDatabase = !RsDirUtil::fileExists(mDbPath); + mGrpMetaDataCache_ContainsAllDatabase = false ; mDb = new RetroDb(mDbPath, RetroDb::OPEN_READWRITE_CREATE, key); @@ -1231,6 +1233,9 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes if (c) { locked_retrieveMsgMeta(c, metaSet); +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << "Retrieving (all) Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; +#endif } }else{ @@ -1245,6 +1250,9 @@ int RsDataService::retrieveGxsMsgMetaData(const GxsMsgReq& reqIds, GxsMsgMetaRes if (c) { locked_retrieveMsgMeta(c, metaSet); +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << "Retrieving Msg metadata grpId=" << grpId << ", " << std::dec << metaSet.size() << " messages" << std::endl; +#endif } } } @@ -1296,34 +1304,51 @@ int RsDataService::retrieveGxsGrpMetaData(std::map::const_iterator it(mGrpMetaDataCache.begin());it!=mGrpMetaDataCache.end();++it) + grp[it->first] = new RsGxsGrpMetaData(it->second); + } + else + { #ifdef RS_DATA_SERVICE_DEBUG - std::cerr << "RsDataService::retrieveGxsGrpMetaData() retrieving all"; - std::cerr << std::endl; + std::cerr << "RsDataService::retrieveGxsGrpMetaData() retrieving all" << std::endl; #endif - RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "", ""); + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "", ""); - if(c) - { - bool valid = c->moveToFirst(); + if(c) + { + bool valid = c->moveToFirst(); - while(valid) - { - RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); - if(g) - { - grp[g->mGroupId] = g; - } - valid = c->moveToNext(); + while(valid) + { + RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); + if(g) + { + grp[g->mGroupId] = g; + mGrpMetaDataCache[g->mGroupId] = *g ; +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << (void *)this << ": Retrieving (all) Grp metadata grpId=" << g->mGroupId << std::endl; +#endif + } + valid = c->moveToNext(); #ifdef RS_DATA_SERVICE_DEBUG_TIME - ++resultCount; + ++resultCount; #endif - } - delete c; - } + } + delete c; + } + + mGrpMetaDataCache_ContainsAllDatabase = true ; + } }else { @@ -1331,30 +1356,56 @@ int RsDataService::retrieveGxsGrpMetaData(std::mapfirst; - RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "grpId='" + grpId.toStdString() + "'", ""); - - if(c) + std::map::const_iterator itt = mGrpMetaDataCache.find(mit->first) ; + if(itt != mGrpMetaDataCache.end()) { - bool valid = c->moveToFirst(); +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << "Retrieving Grp metadata grpId=" << mit->first << " from cache!" << std::endl; +#endif + grp[mit->first] = new RsGxsGrpMetaData(itt->second) ; + } + else + { +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << "Retrieving Grp metadata grpId=" << mit->first ; +#endif - while(valid) - { - RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); + const RsGxsGroupId& grpId = mit->first; + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "grpId='" + grpId.toStdString() + "'", ""); - if(g) - { - grp[g->mGroupId] = g; - } - valid = c->moveToNext(); + if(c) + { + bool valid = c->moveToFirst(); + +//#ifdef RS_DATA_SERVICE_DEBUG_CACHE + if(!valid) + std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl; +//#endif + while(valid) + { + RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); + + if(g) + { + grp[g->mGroupId] = g; + mGrpMetaDataCache[g->mGroupId] = *g ; +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + std::cerr << ". Got it. Updating cache." << std::endl; +#endif + } + valid = c->moveToNext(); #ifdef RS_DATA_SERVICE_DEBUG_TIME - ++resultCount; + ++resultCount; #endif - } - delete c; - } - + } + delete c; + } +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + else + std::cerr << ". not found!" << std::endl; +#endif + } } } @@ -1391,13 +1442,23 @@ int RsDataService::resetDataStore() int RsDataService::updateGroupMetaData(GrpLocMetaData &meta) { - RsStackMutex stack(mDbMutex); + std::cerr << (void*)this << ": Updating Grp Meta data: grpId = " << meta.grpId << std::endl; + + RsStackMutex stack(mDbMutex); RsGxsGroupId& grpId = meta.grpId; + + std::cerr << (void*)this << ": erasing old entry from cache." << std::endl; + + mGrpMetaDataCache_ContainsAllDatabase = false ; + mGrpMetaDataCache.erase(meta.grpId) ; + return mDb->sqlUpdate(GRP_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", meta.val) ? 1 : 0; } int RsDataService::updateMessageMetaData(MsgLocMetaData &metaData) { + std::cerr << (void*)this << ": Updating Msg Meta data: grpId = " << metaData.msgId.first << " msgId = " << metaData.msgId.second << std::endl; + RsStackMutex stack(mDbMutex); RsGxsGroupId& grpId = metaData.msgId.first; RsGxsMessageId& msgId = metaData.msgId.second; diff --git a/libretroshare/src/gxs/rsdataservice.h b/libretroshare/src/gxs/rsdataservice.h index b15f08c27..451d630cc 100644 --- a/libretroshare/src/gxs/rsdataservice.h +++ b/libretroshare/src/gxs/rsdataservice.h @@ -340,6 +340,13 @@ private: uint16_t mServType; RetroDb* mDb; + + // used to store metadata instead of reading it from the database. + // The boolean variable below is also used to force re-reading when + // the entre list of grp metadata is requested (which happens quite often) + + std::map mGrpMetaDataCache ; + bool mGrpMetaDataCache_ContainsAllDatabase ; }; #endif // RSDATASERVICE_H From 720c6fd91512901c2f34ab4c56430a6f79e4edbb Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 2 Jul 2016 22:41:12 -0400 Subject: [PATCH 07/18] fixed bug causing TS update of some unused identities --- libretroshare/src/services/p3idservice.cc | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 6536d384d..4d59be143 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -258,10 +258,7 @@ time_t p3IdService::locked_getLastUsageTS(const RsGxsId& gxs_id) std::map::const_iterator it = mKeysTS.find(gxs_id) ; if(it == mKeysTS.end()) - { - slowIndicateConfigChanged() ; - return mKeysTS[gxs_id] = time(NULL) ; - } + return 0 ; else return it->second ; } @@ -345,7 +342,7 @@ public: if(it == mLastUsageTS.end()) { - std::cerr << "No Ts for this ID" << std::endl; + std::cerr << "No Ts for this ID => kept" << std::endl; return true ; } @@ -413,10 +410,7 @@ void p3IdService::cleanUnusedKeys() { RS_STACK_MUTEX(mIdMtx) ; - std::map::iterator tmp = mKeysTS.find(*it) ; - - if(mKeysTS.end() != tmp) - mKeysTS.erase(tmp) ; + mKeysTS.erase(*it) ; // mPublicKeyCache.erase(*it) ; no need to do it now. It's done in p3IdService::deleteGroup() } From e611b2bb05905308edf4c81582bb2b8ebe9de8fb Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 3 Jul 2016 18:06:01 -0400 Subject: [PATCH 08/18] moved isIdentitybanned from p3GxsReputations to p3IdService, so as to avoid calling back p3IdService from p3GxsReputations, which simplifies the code --- libretroshare/src/chat/distributedchat.cc | 5 +- libretroshare/src/grouter/p3grouter.cc | 3 +- libretroshare/src/gxs/rsgxsnetservice.cc | 10 +-- libretroshare/src/gxs/rsgxsutil.cc | 5 +- libretroshare/src/retroshare/rsidentity.h | 1 + libretroshare/src/retroshare/rsreputations.h | 4 +- libretroshare/src/services/p3gxsreputation.cc | 75 ++++++++++++++----- libretroshare/src/services/p3gxsreputation.h | 4 +- libretroshare/src/services/p3idservice.cc | 18 ++++- libretroshare/src/services/p3idservice.h | 1 + .../src/gui/Identity/IdDetailsDialog.cpp | 3 +- retroshare-gui/src/gui/Identity/IdDialog.cpp | 18 ++--- retroshare-gui/src/gui/gxs/GxsIdDetails.cpp | 6 +- .../src/gui/gxs/GxsIdTreeWidgetItem.cpp | 6 +- .../gui/gxsforums/GxsForumThreadWidget.cpp | 4 +- 15 files changed, 100 insertions(+), 63 deletions(-) diff --git a/libretroshare/src/chat/distributedchat.cc b/libretroshare/src/chat/distributedchat.cc index 88460f1f7..2ed75fd1b 100644 --- a/libretroshare/src/chat/distributedchat.cc +++ b/libretroshare/src/chat/distributedchat.cc @@ -34,7 +34,6 @@ #include "pqi/p3historymgr.h" #include "retroshare/rspeers.h" #include "retroshare/rsiface.h" -#include "retroshare/rsreputations.h" #include "retroshare/rsidentity.h" #include "rsserver/p3face.h" #include "gxs/rsgixs.h" @@ -139,7 +138,7 @@ bool DistributedChatService::handleRecvChatLobbyMsgItem(RsChatMsgItem *ci) return false ; } - if(rsReputations->isIdentityBanned(cli->signature.keyId)) + if(rsIdentity->isBanned(cli->signature.keyId)) { std::cerr << "(WW) Received lobby msg/item from banned identity " << cli->signature.keyId << ". Dropping it." << std::endl; return false ; @@ -648,7 +647,7 @@ void DistributedChatService::handleRecvChatLobbyEventItem(RsChatLobbyEventItem * #endif time_t now = time(NULL) ; - if(rsReputations->isIdentityBanned(item->signature.keyId)) + if(rsIdentity->isBanned(item->signature.keyId)) { std::cerr << "(WW) Received lobby msg/item from banned identity " << item->signature.keyId << ". Dropping it." << std::endl; return ; diff --git a/libretroshare/src/grouter/p3grouter.cc b/libretroshare/src/grouter/p3grouter.cc index 9b6f9a8de..1d698881c 100644 --- a/libretroshare/src/grouter/p3grouter.cc +++ b/libretroshare/src/grouter/p3grouter.cc @@ -191,7 +191,6 @@ #include "turtle/p3turtle.h" #include "gxs/rsgixs.h" #include "retroshare/rspeers.h" -#include "retroshare/rsreputations.h" #include "p3grouter.h" #include "grouteritems.h" @@ -1994,7 +1993,7 @@ bool p3GRouter::verifySignedDataItem(RsGRouterAbstractMsgItem *item,uint32_t& er { try { - if(rsReputations->isIdentityBanned(item->signature.keyId)) + if(rsIdentity->isBanned(item->signature.keyId)) { std::cerr << "(WW) received global router message from banned identity " << item->signature.keyId << ". Rejecting the message." << std::endl; return false ; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 8df2bea19..477e53613 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -204,7 +204,6 @@ #include "rsgxsnetservice.h" #include "gxssecurity.h" #include "retroshare/rsconfig.h" -#include "retroshare/rsreputations.h" #include "retroshare/rsgxsflags.h" #include "retroshare/rsgxscircles.h" #include "pgp/pgpauxutils.h" @@ -2928,9 +2927,8 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) #endif continue; } - // FIXTESTS global variable rsReputations not available in unittests! - if(rsReputations == 0){ std::cerr << "rsReputations==0, accepting all messages!" << std::endl; } - if(rsReputations && rsReputations->isIdentityBanned(syncItem->authorId)) + + if(rsIdentity && rsIdentity->isBanned(syncItem->authorId)) { #ifdef NXS_NET_DEBUG_1 GXSNETDEBUG_PG(item->PeerId(),grpId) << ", Identity " << syncItem->authorId << " is banned. Not requesting message!" << std::endl; @@ -3174,8 +3172,8 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) latestVersion = grpSyncItem->publishTs > metaIter->second->mPublishTs; } // FIXTESTS global variable rsReputations not available in unittests! - if(rsReputations == 0){ std::cerr << "rsReputations==0, accepting all groups!" << std::endl; } - if(!grpSyncItem->authorId.isNull() && rsReputations && rsReputations->isIdentityBanned(grpSyncItem->authorId)) + + if(!grpSyncItem->authorId.isNull() && rsIdentity && rsIdentity->isBanned(grpSyncItem->authorId)) { #ifdef NXS_NET_DEBUG_0 GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << " Identity " << grpSyncItem->authorId << " is banned. Not syncing group." << std::endl; diff --git a/libretroshare/src/gxs/rsgxsutil.cc b/libretroshare/src/gxs/rsgxsutil.cc index 0187e9cb4..22e2e8661 100644 --- a/libretroshare/src/gxs/rsgxsutil.cc +++ b/libretroshare/src/gxs/rsgxsutil.cc @@ -28,7 +28,6 @@ #include "rsgxsutil.h" #include "retroshare/rsgxsflags.h" #include "retroshare/rspeers.h" -#include "retroshare/rsreputations.h" #include "pqi/pqihash.h" #include "gxs/rsgixs.h" @@ -165,7 +164,7 @@ bool RsGxsIntegrityCheck::check() std::cerr << "TimeStamping group authors' key ID " << grp->metaData->mAuthorId << " in group ID " << grp->grpId << std::endl; #endif - if(rsReputations!=NULL && !rsReputations->isIdentityBanned(grp->metaData->mAuthorId)) + if(rsIdentity!=NULL && !rsIdentity->isBanned(grp->metaData->mAuthorId)) used_gxs_ids.insert(grp->metaData->mAuthorId) ; } } @@ -247,7 +246,7 @@ bool RsGxsIntegrityCheck::check() #ifdef GXSUTIL_DEBUG std::cerr << "TimeStamping message authors' key ID " << msg->metaData->mAuthorId << " in message " << msg->msgId << ", group ID " << msg->grpId<< std::endl; #endif - if(rsReputations!=NULL && !rsReputations->isIdentityBanned(msg->metaData->mAuthorId)) + if(rsIdentity!=NULL && !rsIdentity->isBanned(msg->metaData->mAuthorId)) used_gxs_ids.insert(msg->metaData->mAuthorId) ; } diff --git a/libretroshare/src/retroshare/rsidentity.h b/libretroshare/src/retroshare/rsidentity.h index e57d43162..b36133d0b 100644 --- a/libretroshare/src/retroshare/rsidentity.h +++ b/libretroshare/src/retroshare/rsidentity.h @@ -253,6 +253,7 @@ virtual bool getRecognTagRequest(const RsGxsId &id, const std::string &comment, virtual bool setAsRegularContact(const RsGxsId& id,bool is_a_contact) = 0 ; virtual bool isARegularContact(const RsGxsId& id) = 0 ; + virtual bool isBanned(const RsGxsId& id) =0; // Specific RsIdentity Functions.... /* Specific Service Data */ diff --git a/libretroshare/src/retroshare/rsreputations.h b/libretroshare/src/retroshare/rsreputations.h index 6445efd09..f620f3d5e 100644 --- a/libretroshare/src/retroshare/rsreputations.h +++ b/libretroshare/src/retroshare/rsreputations.h @@ -50,14 +50,14 @@ public: }; virtual bool setOwnOpinion(const RsGxsId& key_id, const Opinion& op) =0; - virtual bool getReputationInfo(const RsGxsId& id,ReputationInfo& info) =0 ; + virtual bool getReputationInfo(const RsGxsId& id,const RsPgpId& owner_id,ReputationInfo& info) =0 ; virtual void setNodeAutoBanThreshold(uint32_t n) =0; virtual uint32_t nodeAutoBanThreshold() =0; // This one is a proxy designed to allow fast checking of a GXS id. // it basically returns true if assessment is not ASSESSMENT_OK - virtual bool isIdentityBanned(const RsGxsId& id) =0; + virtual bool isIdentityBanned(const RsGxsId& id,const RsPgpId& owner_node) =0; }; // To access reputations from anywhere diff --git a/libretroshare/src/services/p3gxsreputation.cc b/libretroshare/src/services/p3gxsreputation.cc index f806b4748..ce3b43ca5 100644 --- a/libretroshare/src/services/p3gxsreputation.cc +++ b/libretroshare/src/services/p3gxsreputation.cc @@ -39,6 +39,7 @@ /**** * #define DEBUG_REPUTATION 1 ****/ +#define DEBUG_REPUTATION 1 /************ IMPLEMENTATION NOTES ********************************* * @@ -333,11 +334,35 @@ void p3GxsReputation::updateIdentityFlags() void p3GxsReputation::cleanup() { // remove opinions from friends that havn't been seen online for more than the specified delay - + #ifdef DEBUG_REPUTATION - std::cerr << "p3GxsReputation::cleanup() " << std::endl; #endif - std::cerr << __PRETTY_FUNCTION__ << ": not implemented. TODO!" << std::endl; + std::cerr << "p3GxsReputation::cleanup() " << std::endl; + + // remove optionions about identities that do not exist anymore. That will in particular avoid asking p3idservice about deleted + // identities, which would cause an excess of hits to the database. + + bool updated = false ; + + RsStackMutex stack(mReputationMtx); /****** LOCKED MUTEX *******/ + + for(std::map::iterator it(mReputations.begin());it!=mReputations.end();) + if(it->second.mOpinions.empty() && it->second.mOwnOpinion == RsReputations::OPINION_NEUTRAL) + { + std::map::iterator tmp(it) ; + ++tmp ; + mReputations.erase(it) ; + it = tmp ; +#ifdef DEBUG_REPUTATION + std::cerr << " ID " << it->first << ": own is neutral and no opinions from friends => remove entry" << std::endl; +#endif + updated = true ; + } + else + ++it ; + + if(updated) + IndicateConfigChanged() ; } void p3GxsReputation::updateActiveFriends() @@ -668,7 +693,7 @@ bool p3GxsReputation::updateLatestUpdate(RsPeerId peerid,time_t latest_update) * Opinion ****/ -bool p3GxsReputation::getReputationInfo(const RsGxsId& gxsid, RsReputations::ReputationInfo& info) +bool p3GxsReputation::getReputationInfo(const RsGxsId& gxsid, const RsPgpId& owner_id, RsReputations::ReputationInfo& info) { if(gxsid.isNull()) return false ; @@ -678,25 +703,34 @@ bool p3GxsReputation::getReputationInfo(const RsGxsId& gxsid, RsReputations::Rep #ifdef DEBUG_REPUTATION std::cerr << "getReputationInfo() for " << gxsid << std::endl; #endif - Reputation& rep(mReputations[gxsid]) ; + std::map::const_iterator it = mReputations.find(gxsid) ; - info.mOwnOpinion = RsReputations::Opinion(rep.mOwnOpinion) ; - info.mOverallReputationScore = rep.mReputation ; - info.mFriendAverage = rep.mFriendAverage ; - - if( (rep.mIdentityFlags & REPUTATION_IDENTITY_FLAG_PGP_LINKED) && (mBannedPgpIds.find(rep.mOwnerNode) != mBannedPgpIds.end())) + if(it == mReputations.end()) { - info.mAssessment = RsReputations::ASSESSMENT_BAD ; -#ifdef DEBUG_REPUTATION - std::cerr << "p3GxsReputations: identity " << gxsid << " is banned because owner node ID " << rep.mOwnerNode << " is banned." << std::endl; -#endif - return true; + info.mOwnOpinion = RsReputations::OPINION_NEUTRAL ; + info.mOverallReputationScore = RsReputations::REPUTATION_THRESHOLD_DEFAULT ; + info.mFriendAverage = REPUTATION_THRESHOLD_DEFAULT ; } - - if(info.mOverallReputationScore > REPUTATION_ASSESSMENT_THRESHOLD_X1) - info.mAssessment = RsReputations::ASSESSMENT_OK ; else + { + const Reputation& rep(it->second) ; + + info.mOwnOpinion = RsReputations::Opinion(rep.mOwnOpinion) ; + info.mOverallReputationScore = rep.mReputation ; + info.mFriendAverage = rep.mFriendAverage ; + } + + if(!owner_id.isNull() && (mBannedPgpIds.find(owner_id) != mBannedPgpIds.end())) + { +#ifdef DEBUG_REPUTATION + std::cerr << "p3GxsReputations: identity " << gxsid << " is banned because owner node ID " << owner_id << " is banned." << std::endl; +#endif info.mAssessment = RsReputations::ASSESSMENT_BAD ; + } + else if(info.mOverallReputationScore <= REPUTATION_ASSESSMENT_THRESHOLD_X1) + info.mAssessment = RsReputations::ASSESSMENT_BAD ; + else + info.mAssessment = RsReputations::ASSESSMENT_OK ; #ifdef DEBUG_REPUTATION std::cerr << " information present. OwnOp = " << info.mOwnOpinion << ", overall score=" << info.mAssessment << std::endl; @@ -705,11 +739,12 @@ bool p3GxsReputation::getReputationInfo(const RsGxsId& gxsid, RsReputations::Rep return true ; } -bool p3GxsReputation::isIdentityBanned(const RsGxsId &id) +bool p3GxsReputation::isIdentityBanned(const RsGxsId &id,const RsPgpId& owner_node) { RsReputations::ReputationInfo info ; - getReputationInfo(id,info) ; + if(!getReputationInfo(id,owner_node,info)) + return false ; #ifdef DEBUG_REPUTATION std::cerr << "isIdentityBanned(): returning " << (info.mAssessment == RsReputations::ASSESSMENT_BAD) << " for GXS id " << id << std::endl; diff --git a/libretroshare/src/services/p3gxsreputation.h b/libretroshare/src/services/p3gxsreputation.h index ae49517e5..8dca8cd5a 100644 --- a/libretroshare/src/services/p3gxsreputation.h +++ b/libretroshare/src/services/p3gxsreputation.h @@ -97,8 +97,8 @@ class p3GxsReputation: public p3Service, public p3Config, public RsReputations / /***** Interface for RsReputations *****/ virtual bool setOwnOpinion(const RsGxsId& key_id, const Opinion& op) ; - virtual bool getReputationInfo(const RsGxsId& id,ReputationInfo& info) ; - virtual bool isIdentityBanned(const RsGxsId& id) ; + virtual bool getReputationInfo(const RsGxsId& id, const RsPgpId &owner_id, ReputationInfo& info) ; + virtual bool isIdentityBanned(const RsGxsId& id, const RsPgpId &owner_node) ; virtual void setNodeAutoBanThreshold(uint32_t n) ; virtual uint32_t nodeAutoBanThreshold() ; diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 4d59be143..1abecd8f5 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -264,7 +264,7 @@ time_t p3IdService::locked_getLastUsageTS(const RsGxsId& gxs_id) } void p3IdService::timeStampKey(const RsGxsId& gxs_id) { - if(rsReputations->isIdentityBanned(gxs_id)) + if(isBanned(gxs_id)) { std::cerr << "(II) p3IdService:timeStampKey(): refusing to time stamp key " << gxs_id << " because it is banned." << std::endl; return; @@ -324,7 +324,7 @@ public: time_t now = time(NULL); const RsGxsId& gxs_id = entry.details.mId ; - bool is_id_banned = rsReputations->isIdentityBanned(gxs_id) ; + bool is_id_banned = rsReputations->isIdentityBanned(gxs_id,entry.details.mPgpId) ; bool is_own_id = (bool)(entry.details.mFlags & RS_IDENTITY_FLAGS_IS_OWN_ID) ; bool is_known_id = (bool)(entry.details.mFlags & RS_IDENTITY_FLAGS_PGP_KNOWN) ; bool is_signed_id = (bool)(entry.details.mFlags & RS_IDENTITY_FLAGS_PGP_LINKED) ; @@ -503,7 +503,7 @@ bool p3IdService:: getNickname(const RsGxsId &id, std::string &nickname) } #endif -bool p3IdService:: getIdDetails(const RsGxsId &id, RsIdentityDetails &details) +bool p3IdService::getIdDetails(const RsGxsId &id, RsIdentityDetails &details) { #ifdef DEBUG_IDS std::cerr << "p3IdService::getIdDetails(" << id << ")"; @@ -524,7 +524,7 @@ bool p3IdService:: getIdDetails(const RsGxsId &id, RsIdentityDetails &details) if(details.mNickname.length() > RSID_MAXIMUM_NICKNAME_SIZE*4) details.mNickname = "[too long a name]" ; - rsReputations->getReputationInfo(id,details.mReputation) ; + rsReputations->getReputationInfo(id,details.mPgpId,details.mReputation) ; return true; } @@ -536,6 +536,16 @@ bool p3IdService:: getIdDetails(const RsGxsId &id, RsIdentityDetails &details) return false; } +bool p3IdService::isBanned(const RsGxsId &id) +{ + RsIdentityDetails det ; + getIdDetails(id,det) ; + +#ifdef DEBUG_REPUTATION + std::cerr << "isIdentityBanned(): returning " << (det.mReputation.mAssessment == RsReputations::ASSESSMENT_BAD) << " for GXS id " << id << std::endl; +#endif + return det.mReputation.mAssessment == RsReputations::ASSESSMENT_BAD ; +} bool p3IdService::isOwnId(const RsGxsId& id) { diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index 316fad106..3c39e0a2d 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -274,6 +274,7 @@ virtual bool getRecognTagRequest(const RsGxsId &id, const std::string &comment, virtual bool setAsRegularContact(const RsGxsId& id,bool is_a_contact) ; virtual bool isARegularContact(const RsGxsId& id) ; + virtual bool isBanned(const RsGxsId& id) ; /**************** RsGixs Implementation ***************/ diff --git a/retroshare-gui/src/gui/Identity/IdDetailsDialog.cpp b/retroshare-gui/src/gui/Identity/IdDetailsDialog.cpp index 57e83c1b4..dcc3c78fb 100644 --- a/retroshare-gui/src/gui/Identity/IdDetailsDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDetailsDialog.cpp @@ -26,7 +26,6 @@ #include "gui/common/UIStateHelper.h" #include -#include // Data Requests. #define IDDETAILSDIALOG_IDDETAILS 1 @@ -261,7 +260,7 @@ void IdDetailsDialog::insertIdDetails(uint32_t token) #endif RsReputations::ReputationInfo info ; - rsReputations->getReputationInfo(RsGxsId(data.mMeta.mGroupId),info) ; + rsReputations->getReputationInfo(RsGxsId(data.mMeta.mGroupId),data.mPgpId,info) ; ui->neighborNodesOpinion_TF->setText(QString::number(info.mOverallReputationScore-1.0f)); diff --git a/retroshare-gui/src/gui/Identity/IdDialog.cpp b/retroshare-gui/src/gui/Identity/IdDialog.cpp index 583c83375..137f9d23a 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDialog.cpp @@ -41,7 +41,6 @@ #include "util/QtVersion.h" #include -#include #include "retroshare/rsgxsflags.h" #include "retroshare/rsmsgs.h" #include @@ -1360,8 +1359,8 @@ bool IdDialog::fillIdListItem(const RsGxsIdGroup& data, QTreeWidgetItem *&item, if (!item) item = new TreeWidgetItem(); - RsReputations::ReputationInfo info ; - rsReputations->getReputationInfo(RsGxsId(data.mMeta.mGroupId),info) ; + RsIdentityDetails idd ; + rsIdentity->getIdDetails(RsGxsId(data.mMeta.mGroupId),idd) ; item->setText(RSID_COL_NICKNAME, QString::fromUtf8(data.mMeta.mGroupName.c_str()).left(RSID_MAXIMUM_NICKNAME_SIZE)); item->setText(RSID_COL_KEYID, QString::fromStdString(data.mMeta.mGroupId.toStdString())); @@ -1372,13 +1371,10 @@ bool IdDialog::fillIdListItem(const RsGxsIdGroup& data, QTreeWidgetItem *&item, item->setData(RSID_COL_KEYID, Qt::UserRole,QVariant(item_flags)) ; item->setTextAlignment(RSID_COL_VOTES, Qt::AlignRight); - item->setData(RSID_COL_VOTES,Qt::DisplayRole, QString::number(info.mOverallReputationScore - 1.0f,'f',3)); + item->setData(RSID_COL_VOTES,Qt::DisplayRole, QString::number(idd.mReputation.mOverallReputationScore - 1.0f,'f',3)); if(isOwnId) { - RsIdentityDetails idd ; - rsIdentity->getIdDetails(RsGxsId(data.mMeta.mGroupId),idd) ; - QFont font = item->font(RSID_COL_NICKNAME) ; font.setBold(true) ; @@ -1740,7 +1736,7 @@ void IdDialog::insertIdDetails(uint32_t token) #endif RsReputations::ReputationInfo info ; - rsReputations->getReputationInfo(RsGxsId(data.mMeta.mGroupId),info) ; + rsReputations->getReputationInfo(RsGxsId(data.mMeta.mGroupId),data.mPgpId,info) ; ui->neighborNodesOpinion_TF->setText(QString::number(info.mFriendAverage - 1.0f)); @@ -2021,10 +2017,10 @@ void IdDialog::IdListCustomPopupMenu( QPoint ) #endif RsGxsId keyId((*it)->text(RSID_COL_KEYID).toStdString()); - RsReputations::ReputationInfo info ; - rsReputations->getReputationInfo(keyId,info) ; + RsIdentityDetails det ; + rsIdentity->getIdDetails(keyId,det) ; - switch(info.mOwnOpinion) + switch(det.mReputation.mOwnOpinion) { case RsReputations::OPINION_NEGATIVE: ++n_negative_reputations ; break ; diff --git a/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp b/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp index 7458e0406..c1205e6b3 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp @@ -905,7 +905,7 @@ bool GxsIdDetails::MakeIdDesc(const RsGxsId &id, bool doIcons, QString &str, QLi QString GxsIdDetails::getName(const RsIdentityDetails &details) { - if(rsReputations->isIdentityBanned(details.mId)) + if(rsIdentity->isBanned(details.mId)) return tr("[Banned]") ; QString name = QString::fromUtf8(details.mNickname.c_str()).left(RSID_MAXIMUM_NICKNAME_SIZE); @@ -924,7 +924,7 @@ QString GxsIdDetails::getComment(const RsIdentityDetails &details) QString comment; QString nickname ; - bool banned = rsReputations->isIdentityBanned(details.mId) ; + bool banned = rsIdentity->isBanned(details.mId) ; if(details.mNickname.empty()) nickname = tr("[Unknown]") ; @@ -963,7 +963,7 @@ void GxsIdDetails::getIcons(const RsIdentityDetails &details, QList &icon { QPixmap pix ; - if(rsReputations->isIdentityBanned(details.mId)) + if(rsIdentity->isBanned(details.mId)) { icons.clear() ; icons.push_back(QIcon(IMAGE_BANNED)) ; diff --git a/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp b/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp index d002e9541..998a98a5e 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp @@ -117,14 +117,14 @@ void GxsIdRSTreeWidgetItem::setId(const RsGxsId &id, int column, bool retryWhenF void GxsIdRSTreeWidgetItem::updateBannedState() { - if(mBannedState != rsReputations->isIdentityBanned(mId)) + if(mBannedState != rsIdentity->isBanned(mId)) forceUpdate() ; } void GxsIdRSTreeWidgetItem::forceUpdate() { mIdFound = false; - mBannedState = rsReputations->isIdentityBanned(mId) ; + mBannedState = rsIdentity->isBanned(mId) ; startProcess(); } @@ -170,7 +170,7 @@ QVariant GxsIdRSTreeWidgetItem::data(int column, int role) const if(mId.isNull()) return RSTreeWidgetItem::data(column, role); - else if(rsReputations->isIdentityBanned(mId)) + else if(rsIdentity->isBanned(mId)) pix = QImage(BANNED_IMAGE) ; else if (mAvatar.mSize == 0 || !pix.loadFromData(mAvatar.mData, mAvatar.mSize, "PNG")) pix = GxsIdDetails::makeDefaultIcon(mId); diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 82ed1e6dd..19ef8c89a 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -985,7 +985,7 @@ QTreeWidgetItem *GxsForumThreadWidget::convertMsgToThreadWidget(const RsGxsForum // is flagged with a bad reputation - bool redacted = rsReputations->isIdentityBanned(msg.mMeta.mAuthorId) ; + bool redacted = rsIdentity->isBanned(msg.mMeta.mAuthorId) ; GxsIdRSTreeWidgetItem *item = new GxsIdRSTreeWidgetItem(mThreadCompareRole,GxsIdDetails::ICON_TYPE_ALL || (redacted?(GxsIdDetails::ICON_TYPE_REDACTED):0)); item->moveToThread(ui->threadTreeWidget->thread()); @@ -1408,7 +1408,7 @@ void GxsForumThreadWidget::insertMessageData(const RsGxsForumMsg &msg) return; } - bool redacted = rsReputations->isIdentityBanned(msg.mMeta.mAuthorId) ; + bool redacted = rsIdentity->isBanned(msg.mMeta.mAuthorId) ; mStateHelper->setActive(mTokenTypeMessageData, true); From 692293e030fab2b7ef0c545d25440f6c97eb2af2 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 3 Jul 2016 18:12:50 -0400 Subject: [PATCH 09/18] added red color to banned identities in IdDialog --- retroshare-gui/src/gui/Identity/IdDialog.cpp | 7 +++++++ retroshare-gui/src/gui/gxs/GxsIdDetails.cpp | 1 - retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp | 1 - retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp | 1 - 4 files changed, 7 insertions(+), 3 deletions(-) diff --git a/retroshare-gui/src/gui/Identity/IdDialog.cpp b/retroshare-gui/src/gui/Identity/IdDialog.cpp index 137f9d23a..1a8d20fde 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDialog.cpp @@ -1368,6 +1368,13 @@ bool IdDialog::fillIdListItem(const RsGxsIdGroup& data, QTreeWidgetItem *&item, //time_t now = time(NULL) ; //item->setText(RSID_COL_LASTUSED, getHumanReadableDuration(now - data.mLastUsageTS)) ; + + if(idd.mReputation.mAssessment == RsReputations::ASSESSMENT_BAD) + { + item->setForeground(RSID_COL_NICKNAME,QBrush(Qt::red)); + item->setForeground(RSID_COL_KEYID,QBrush(Qt::red)); + item->setForeground(RSID_COL_IDTYPE,QBrush(Qt::red)); + } item->setData(RSID_COL_KEYID, Qt::UserRole,QVariant(item_flags)) ; item->setTextAlignment(RSID_COL_VOTES, Qt::AlignRight); diff --git a/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp b/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp index c1205e6b3..5c7d00dd7 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdDetails.cpp @@ -29,7 +29,6 @@ #include #include "GxsIdDetails.h" #include "retroshare-gui/RsAutoUpdatePage.h" -#include "retroshare/rsreputations.h" #include diff --git a/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp b/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp index 998a98a5e..0177f892f 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp @@ -25,7 +25,6 @@ #include "GxsIdTreeWidgetItem.h" #include "GxsIdDetails.h" #include "util/HandleRichText.h" -#include "retroshare/rsreputations.h" #define BANNED_IMAGE ":/icons/yellow_biohazard64.png" diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 19ef8c89a..1e98e3283 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -44,7 +44,6 @@ #include #include -#include #include // These should be in retroshare/ folder. #include "retroshare/rsgxsflags.h" From bf28ea7ac72a2abc32e5af952b14112e7d93d50f Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 3 Jul 2016 18:49:02 -0400 Subject: [PATCH 10/18] removed debug info in udpstack.cc --- libbitdht/src/udp/udpstack.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libbitdht/src/udp/udpstack.cc b/libbitdht/src/udp/udpstack.cc index 381fc182d..a66f4c6e2 100644 --- a/libbitdht/src/udp/udpstack.cc +++ b/libbitdht/src/udp/udpstack.cc @@ -87,8 +87,10 @@ bool UdpStack::getLocalAddress(struct sockaddr_in &local) bool UdpStack::resetAddress(struct sockaddr_in &local) { - std::cerr << "UdpStack::resetAddress(" << local << ")"; +#ifdef DEBUG_UDP_RECV + std::cerr << "UdpStack::resetAddress(" << local << ")"; std::cerr << std::endl; +#endif laddr = local; return udpLayer->reset(local); From c98416dc285e9b36b33157719fdd6ec584cdd7c1 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 3 Jul 2016 18:59:30 -0400 Subject: [PATCH 11/18] added auto-cleaning of reputation info for outdated/removed identities --- libretroshare/src/retroshare/rsidentity.h | 59 +++--- libretroshare/src/services/p3gxsreputation.cc | 69 ++++--- libretroshare/src/services/p3idservice.cc | 6 + libretroshare/src/services/p3idservice.h | 193 +++++++++--------- 4 files changed, 174 insertions(+), 153 deletions(-) diff --git a/libretroshare/src/retroshare/rsidentity.h b/libretroshare/src/retroshare/rsidentity.h index b36133d0b..bf6a80724 100644 --- a/libretroshare/src/retroshare/rsidentity.h +++ b/libretroshare/src/retroshare/rsidentity.h @@ -221,47 +221,48 @@ public: RsIdentity(RsGxsIface *gxs): RsGxsIfaceHelper(gxs) { return; } virtual ~RsIdentity() { return; } -/********************************************************************************************/ -/********************************************************************************************/ - - // For Other Services.... - // It should be impossible for them to get a message which we don't have the identity. - // Its a major error if we don't have the identity. + /********************************************************************************************/ + /********************************************************************************************/ - // We cache all identities, and provide alternative (instantaneous) - // functions to extract info, rather than the standard Token system. + // For Other Services.... + // It should be impossible for them to get a message which we don't have the identity. + // Its a major error if we don't have the identity. -//virtual bool getNickname(const RsGxsId &id, std::string &nickname) = 0; -virtual bool getIdDetails(const RsGxsId &id, RsIdentityDetails &details) = 0; + // We cache all identities, and provide alternative (instantaneous) + // functions to extract info, rather than the standard Token system. -// Fills up list of all own ids. Returns false if ids are not yet loaded. -virtual bool getOwnIds(std::list &ownIds) = 0; -virtual bool isOwnId(const RsGxsId& id) = 0; + //virtual bool getNickname(const RsGxsId &id, std::string &nickname) = 0; + virtual bool getIdDetails(const RsGxsId &id, RsIdentityDetails &details) = 0; - // -virtual bool submitOpinion(uint32_t& token, const RsGxsId &id, - bool absOpinion, int score) = 0; -virtual bool createIdentity(uint32_t& token, RsIdentityParameters ¶ms) = 0; + // Fills up list of all own ids. Returns false if ids are not yet loaded. + virtual bool getOwnIds(std::list &ownIds) = 0; + virtual bool isOwnId(const RsGxsId& id) = 0; -virtual bool updateIdentity(uint32_t& token, RsGxsIdGroup &group) = 0; -virtual bool deleteIdentity(uint32_t& token, RsGxsIdGroup &group) = 0; + // + virtual bool submitOpinion(uint32_t& token, const RsGxsId &id, + bool absOpinion, int score) = 0; + virtual bool createIdentity(uint32_t& token, RsIdentityParameters ¶ms) = 0; -virtual bool parseRecognTag(const RsGxsId &id, const std::string &nickname, - const std::string &tag, RsRecognTagDetails &details) = 0; -virtual bool getRecognTagRequest(const RsGxsId &id, const std::string &comment, - uint16_t tag_class, uint16_t tag_type, std::string &tag) = 0; + virtual bool updateIdentity(uint32_t& token, RsGxsIdGroup &group) = 0; + virtual bool deleteIdentity(uint32_t& token, RsGxsIdGroup &group) = 0; + + virtual bool parseRecognTag(const RsGxsId &id, const std::string &nickname, + const std::string &tag, RsRecognTagDetails &details) = 0; + virtual bool getRecognTagRequest(const RsGxsId &id, const std::string &comment, + uint16_t tag_class, uint16_t tag_type, std::string &tag) = 0; virtual bool setAsRegularContact(const RsGxsId& id,bool is_a_contact) = 0 ; virtual bool isARegularContact(const RsGxsId& id) = 0 ; virtual bool isBanned(const RsGxsId& id) =0; - - // Specific RsIdentity Functions.... - /* Specific Service Data */ - /* We expose these initially for testing / GUI purposes. + virtual time_t getLastUsageTS(const RsGxsId &id) =0; + + // Specific RsIdentity Functions.... + /* Specific Service Data */ + /* We expose these initially for testing / GUI purposes. */ -virtual bool getGroupData(const uint32_t &token, std::vector &groups) = 0; -//virtual bool getMsgData(const uint32_t &token, std::vector &opinions) = 0; + virtual bool getGroupData(const uint32_t &token, std::vector &groups) = 0; + //virtual bool getMsgData(const uint32_t &token, std::vector &opinions) = 0; }; diff --git a/libretroshare/src/services/p3gxsreputation.cc b/libretroshare/src/services/p3gxsreputation.cc index ce3b43ca5..f40beb145 100644 --- a/libretroshare/src/services/p3gxsreputation.cc +++ b/libretroshare/src/services/p3gxsreputation.cc @@ -117,13 +117,13 @@ * 10 | 1.0 | 0 | 0.25 | 1.0 * * To check: - * [ ] Opinions are saved/loaded accross restart - * [ ] Opinions are transmitted to friends - * [ ] Opinions are transmitted to friends when updated + * [X] Opinions are saved/loaded accross restart + * [X] Opinions are transmitted to friends + * [X] Opinions are transmitted to friends when updated * * To do: - * [ ] Add debug info - * [ ] Test the whole thing + * [X] Add debug info + * [X] Test the whole thing * [X] Implement a system to allow not storing info when we don't have it */ @@ -139,6 +139,7 @@ static const float REPUTATION_ASSESSMENT_THRESHOLD_X1 = 0.5f ; // reputat static const uint32_t PGP_AUTO_BAN_THRESHOLD_DEFAULT = 2 ; // above this, auto ban any GXS id signed by this node static const uint32_t IDENTITY_FLAGS_UPDATE_DELAY = 100 ; // static const uint32_t BANNED_NODES_UPDATE_DELAY = 313 ; // update approx every 5 mins. Chosen to not be a multiple of IDENTITY_FLAGS_UPDATE_DELAY +static const uint32_t REPUTATION_INFO_KEEP_DELAY = 86400*35; // remove old reputation info 5 days after last usage limit, in case the ID would come back.. p3GxsReputation::p3GxsReputation(p3LinkMgr *lm) :p3Service(), p3Config(), @@ -343,26 +344,38 @@ void p3GxsReputation::cleanup() // identities, which would cause an excess of hits to the database. bool updated = false ; + time_t now = time(NULL) ; RsStackMutex stack(mReputationMtx); /****** LOCKED MUTEX *******/ for(std::map::iterator it(mReputations.begin());it!=mReputations.end();) - if(it->second.mOpinions.empty() && it->second.mOwnOpinion == RsReputations::OPINION_NEUTRAL) - { - std::map::iterator tmp(it) ; - ++tmp ; - mReputations.erase(it) ; - it = tmp ; + if(it->second.mOpinions.empty() && it->second.mOwnOpinion == RsReputations::OPINION_NEUTRAL) + { + std::map::iterator tmp(it) ; + ++tmp ; + mReputations.erase(it) ; + it = tmp ; #ifdef DEBUG_REPUTATION - std::cerr << " ID " << it->first << ": own is neutral and no opinions from friends => remove entry" << std::endl; + std::cerr << " ID " << it->first << ": own is neutral and no opinions from friends => remove entry" << std::endl; #endif - updated = true ; - } - else - ++it ; + updated = true ; + } + else if(rsIdentity->getLastUsageTS(it->first) + REPUTATION_INFO_KEEP_DELAY < now) + { +#ifdef DEBUG_REPUTATION + std::cerr << " Identity " << it->first << " has a last usage TS of " << now - rsIdentity->getLastUsageTS(it->first) << " secs ago: deleting it." << std::endl; +#endif + std::map::iterator tmp(it) ; + ++tmp ; + mReputations.erase(it) ; + it = tmp ; + updated = true ; + } + else + ++it ; if(updated) - IndicateConfigChanged() ; + IndicateConfigChanged() ; } void p3GxsReputation::updateActiveFriends() @@ -709,28 +722,28 @@ bool p3GxsReputation::getReputationInfo(const RsGxsId& gxsid, const RsPgpId& own { info.mOwnOpinion = RsReputations::OPINION_NEUTRAL ; info.mOverallReputationScore = RsReputations::REPUTATION_THRESHOLD_DEFAULT ; - info.mFriendAverage = REPUTATION_THRESHOLD_DEFAULT ; + info.mFriendAverage = REPUTATION_THRESHOLD_DEFAULT ; } else { - const Reputation& rep(it->second) ; + const Reputation& rep(it->second) ; - info.mOwnOpinion = RsReputations::Opinion(rep.mOwnOpinion) ; - info.mOverallReputationScore = rep.mReputation ; - info.mFriendAverage = rep.mFriendAverage ; + info.mOwnOpinion = RsReputations::Opinion(rep.mOwnOpinion) ; + info.mOverallReputationScore = rep.mReputation ; + info.mFriendAverage = rep.mFriendAverage ; } - + if(!owner_id.isNull() && (mBannedPgpIds.find(owner_id) != mBannedPgpIds.end())) { #ifdef DEBUG_REPUTATION - std::cerr << "p3GxsReputations: identity " << gxsid << " is banned because owner node ID " << owner_id << " is banned." << std::endl; + std::cerr << "p3GxsReputations: identity " << gxsid << " is banned because owner node ID " << owner_id << " is banned." << std::endl; #endif - info.mAssessment = RsReputations::ASSESSMENT_BAD ; + info.mAssessment = RsReputations::ASSESSMENT_BAD ; } else if(info.mOverallReputationScore <= REPUTATION_ASSESSMENT_THRESHOLD_X1) - info.mAssessment = RsReputations::ASSESSMENT_BAD ; + info.mAssessment = RsReputations::ASSESSMENT_BAD ; else - info.mAssessment = RsReputations::ASSESSMENT_OK ; + info.mAssessment = RsReputations::ASSESSMENT_OK ; #ifdef DEBUG_REPUTATION std::cerr << " information present. OwnOp = " << info.mOwnOpinion << ", overall score=" << info.mAssessment << std::endl; @@ -745,7 +758,7 @@ bool p3GxsReputation::isIdentityBanned(const RsGxsId &id,const RsPgpId& owner_no if(!getReputationInfo(id,owner_node,info)) return false ; - + #ifdef DEBUG_REPUTATION std::cerr << "isIdentityBanned(): returning " << (info.mAssessment == RsReputations::ASSESSMENT_BAD) << " for GXS id " << id << std::endl; #endif diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 1abecd8f5..f521f38be 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -503,6 +503,12 @@ bool p3IdService:: getNickname(const RsGxsId &id, std::string &nickname) } #endif +time_t p3IdService::getLastUsageTS(const RsGxsId &id) +{ + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + return locked_getLastUsageTS(id) ; +} + bool p3IdService::getIdDetails(const RsGxsId &id, RsIdentityDetails &details) { #ifdef DEBUG_IDS diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index 3c39e0a2d..8efc3aea8 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -218,34 +218,34 @@ private: class p3IdService: public RsGxsIdExchange, public RsIdentity, public GxsTokenQueue, public RsTickEvent, public p3Config { - public: +public: p3IdService(RsGeneralDataService* gds, RsNetworkExchangeService* nes, PgpAuxUtils *pgpUtils); -virtual RsServiceInfo getServiceInfo(); -static uint32_t idAuthenPolicy(); + virtual RsServiceInfo getServiceInfo(); + static uint32_t idAuthenPolicy(); virtual void service_tick(); // needed for background processing. - /*! - * Design hack, id service must be constructed first as it - * is need for construction of subsequent net services - */ - void setNes(RsNetworkExchangeService* nes); + /*! + * Design hack, id service must be constructed first as it + * is need for construction of subsequent net services + */ + void setNes(RsNetworkExchangeService* nes); /* General Interface is provided by RsIdentity / RsGxsIfaceImpl. */ /* Data Specific Interface */ // These are exposed via RsIdentity. -virtual bool getGroupData(const uint32_t &token, std::vector &groups); -//virtual bool getMsgData(const uint32_t &token, std::vector &opinions); + virtual bool getGroupData(const uint32_t &token, std::vector &groups); + //virtual bool getMsgData(const uint32_t &token, std::vector &opinions); // These are local - and not exposed via RsIdentity. -virtual bool createGroup(uint32_t& token, RsGxsIdGroup &group); -virtual bool updateGroup(uint32_t& token, RsGxsIdGroup &group); -virtual bool deleteGroup(uint32_t& token, RsGxsIdGroup &group); -//virtual bool createMsg(uint32_t& token, RsGxsIdOpinion &opinion); + virtual bool createGroup(uint32_t& token, RsGxsIdGroup &group); + virtual bool updateGroup(uint32_t& token, RsGxsIdGroup &group); + virtual bool deleteGroup(uint32_t& token, RsGxsIdGroup &group); + //virtual bool createMsg(uint32_t& token, RsGxsIdOpinion &opinion); /**************** RsIdentity External Interface. * Notes: @@ -256,87 +256,88 @@ virtual bool deleteGroup(uint32_t& token, RsGxsIdGroup &group); * Also need to handle Cache updates / invalidation from internal changes. * */ -//virtual bool getNickname(const RsGxsId &id, std::string &nickname); -virtual bool getIdDetails(const RsGxsId &id, RsIdentityDetails &details); + //virtual bool getNickname(const RsGxsId &id, std::string &nickname); + virtual bool getIdDetails(const RsGxsId &id, RsIdentityDetails &details); - // -virtual bool submitOpinion(uint32_t& token, const RsGxsId &id, - bool absOpinion, int score); -virtual bool createIdentity(uint32_t& token, RsIdentityParameters ¶ms); + // + virtual bool submitOpinion(uint32_t& token, const RsGxsId &id, + bool absOpinion, int score); + virtual bool createIdentity(uint32_t& token, RsIdentityParameters ¶ms); -virtual bool updateIdentity(uint32_t& token, RsGxsIdGroup &group); -virtual bool deleteIdentity(uint32_t& token, RsGxsIdGroup &group); + virtual bool updateIdentity(uint32_t& token, RsGxsIdGroup &group); + virtual bool deleteIdentity(uint32_t& token, RsGxsIdGroup &group); -virtual bool parseRecognTag(const RsGxsId &id, const std::string &nickname, - const std::string &tag, RsRecognTagDetails &details); -virtual bool getRecognTagRequest(const RsGxsId &id, const std::string &comment, - uint16_t tag_class, uint16_t tag_type, std::string &tag); + virtual bool parseRecognTag(const RsGxsId &id, const std::string &nickname, + const std::string &tag, RsRecognTagDetails &details); + virtual bool getRecognTagRequest(const RsGxsId &id, const std::string &comment, + uint16_t tag_class, uint16_t tag_type, std::string &tag); -virtual bool setAsRegularContact(const RsGxsId& id,bool is_a_contact) ; - virtual bool isARegularContact(const RsGxsId& id) ; - virtual bool isBanned(const RsGxsId& id) ; - - /**************** RsGixs Implementation ***************/ + virtual bool setAsRegularContact(const RsGxsId& id,bool is_a_contact) ; + virtual bool isARegularContact(const RsGxsId& id) ; + virtual bool isBanned(const RsGxsId& id) ; + virtual time_t getLastUsageTS(const RsGxsId &id) ; - virtual bool getOwnIds(std::list &ownIds); + /**************** RsGixs Implementation ***************/ - //virtual bool getPublicKey(const RsGxsId &id, RsTlvSecurityKey &key) ; - //virtual void networkRequestPublicKey(const RsGxsId& key_id,const std::list& peer_ids) ; + virtual bool getOwnIds(std::list &ownIds); - virtual bool isOwnId(const RsGxsId& key_id) ; + //virtual bool getPublicKey(const RsGxsId &id, RsTlvSecurityKey &key) ; + //virtual void networkRequestPublicKey(const RsGxsId& key_id,const std::list& peer_ids) ; - virtual bool signData(const uint8_t *data,uint32_t data_size,const RsGxsId& signer_id,RsTlvKeySignature& signature,uint32_t& signing_error) ; - virtual bool validateData(const uint8_t *data,uint32_t data_size,const RsTlvKeySignature& signature,bool force_load,uint32_t& signing_error) ; + virtual bool isOwnId(const RsGxsId& key_id) ; - virtual bool encryptData(const uint8_t *decrypted_data,uint32_t decrypted_data_size,uint8_t *& encrypted_data,uint32_t& encrypted_data_size,const RsGxsId& encryption_key_id,bool force_load,uint32_t& encryption_error) ; - virtual bool decryptData(const uint8_t *encrypted_data,uint32_t encrypted_data_size,uint8_t *& decrypted_data,uint32_t& decrypted_data_size,const RsGxsId& encryption_key_id,uint32_t& encryption_error) ; + virtual bool signData(const uint8_t *data,uint32_t data_size,const RsGxsId& signer_id,RsTlvKeySignature& signature,uint32_t& signing_error) ; + virtual bool validateData(const uint8_t *data,uint32_t data_size,const RsTlvKeySignature& signature,bool force_load,uint32_t& signing_error) ; - virtual bool haveKey(const RsGxsId &id); - virtual bool havePrivateKey(const RsGxsId &id); + virtual bool encryptData(const uint8_t *decrypted_data,uint32_t decrypted_data_size,uint8_t *& encrypted_data,uint32_t& encrypted_data_size,const RsGxsId& encryption_key_id,bool force_load,uint32_t& encryption_error) ; + virtual bool decryptData(const uint8_t *encrypted_data,uint32_t encrypted_data_size,uint8_t *& decrypted_data,uint32_t& decrypted_data_size,const RsGxsId& encryption_key_id,uint32_t& encryption_error) ; - virtual bool getKey(const RsGxsId &id, RsTlvPublicRSAKey &key); - virtual bool getPrivateKey(const RsGxsId &id, RsTlvPrivateRSAKey &key); + virtual bool haveKey(const RsGxsId &id); + virtual bool havePrivateKey(const RsGxsId &id); - virtual bool requestKey(const RsGxsId &id, const std::list &peers); - virtual bool requestPrivateKey(const RsGxsId &id); + virtual bool getKey(const RsGxsId &id, RsTlvPublicRSAKey &key); + virtual bool getPrivateKey(const RsGxsId &id, RsTlvPrivateRSAKey &key); + + virtual bool requestKey(const RsGxsId &id, const std::list &peers); + virtual bool requestPrivateKey(const RsGxsId &id); - /**************** RsGixsReputation Implementation ****************/ + /**************** RsGixsReputation Implementation ****************/ - // get Reputation. -virtual bool haveReputation(const RsGxsId &id); -virtual bool loadReputation(const RsGxsId &id, const std::list& peers); -virtual bool getReputation(const RsGxsId &id, GixsReputation &rep); + // get Reputation. + virtual bool haveReputation(const RsGxsId &id); + virtual bool loadReputation(const RsGxsId &id, const std::list& peers); + virtual bool getReputation(const RsGxsId &id, GixsReputation &rep); - protected: - /** Notifications **/ -virtual void notifyChanges(std::vector& changes); +protected: + /** Notifications **/ + virtual void notifyChanges(std::vector& changes); /** Overloaded to add PgpIdHash to Group Definition **/ -virtual ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet); + virtual ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet); - // Overloaded from GxsTokenQueue for Request callbacks. -virtual void handleResponse(uint32_t token, uint32_t req_type); + // Overloaded from GxsTokenQueue for Request callbacks. + virtual void handleResponse(uint32_t token, uint32_t req_type); - // Overloaded from RsTickEvent. -virtual void handle_event(uint32_t event_type, const std::string &elabel); + // Overloaded from RsTickEvent. + virtual void handle_event(uint32_t event_type, const std::string &elabel); - //===================================================// - // p3Config methods // - //===================================================// + //===================================================// + // p3Config methods // + //===================================================// - // Load/save the routing info, the pending items in transit, and the config variables. - // - virtual bool loadList(std::list& items) ; - virtual bool saveList(bool& cleanup,std::list& items) ; + // Load/save the routing info, the pending items in transit, and the config variables. + // + virtual bool loadList(std::list& items) ; + virtual bool saveList(bool& cleanup,std::list& items) ; - virtual RsSerialiser *setupSerialiser() ; + virtual RsSerialiser *setupSerialiser() ; - private: +private: -/************************************************************************ + /************************************************************************ * This is the Cache for minimising calls to the DataStore. * */ @@ -349,7 +350,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); bool cache_store(const RsGxsIdGroupItem *item); bool cache_update_if_cached(const RsGxsId &id, std::string serviceString); - bool isPendingNetworkRequest(const RsGxsId& gxsId); + bool isPendingNetworkRequest(const RsGxsId& gxsId); void requestIdsFromNet(); // Mutex protected. @@ -360,7 +361,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); // Switching to RsMemCache for Key Caching. RsMemCache mKeyCache; -/************************************************************************ + /************************************************************************ * Refreshing own Ids. * */ @@ -369,7 +370,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); std::list mOwnIds; -/************************************************************************ + /************************************************************************ * Test fns for Caching. * */ @@ -377,7 +378,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); bool cachetest_getlist(); bool cachetest_handlerequest(uint32_t token); -/************************************************************************ + /************************************************************************ * for processing background tasks that use the serviceString. * - must be mutually exclusive to avoid clashes. */ @@ -387,7 +388,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); bool mBgSchedule_Active; uint32_t mBgSchedule_Mode; -/************************************************************************ + /************************************************************************ * pgphash processing. * */ @@ -395,7 +396,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); bool pgphash_handlerequest(uint32_t token); bool pgphash_process(); - bool checkId(const RsGxsIdGroup &grp, RsPgpId &pgp_id, bool &error); + bool checkId(const RsGxsIdGroup &grp, RsPgpId &pgp_id, bool &error); void getPgpIdList(); /* MUTEX PROTECTED DATA (mIdMtx - maybe should use a 2nd?) */ @@ -403,7 +404,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); std::map mPgpFingerprintMap; std::list mGroupsToProcess; -/************************************************************************ + /************************************************************************ * recogn processing. * */ @@ -421,7 +422,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); void loadRecognKeys(); -/************************************************************************ + /************************************************************************ * opinion processing. * */ @@ -449,24 +450,24 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); std::map mRecognSignKeys; std::map mRecognOldSignKeys; -/************************************************************************ + /************************************************************************ * Below is the background task for processing opinions => reputations * */ -virtual void generateDummyData(); + virtual void generateDummyData(); void generateDummy_OwnIds(); void generateDummy_FriendPGP(); void generateDummy_UnknownPGP(); void generateDummy_UnknownPseudo(); - void cleanUnusedKeys() ; - void slowIndicateConfigChanged() ; + void cleanUnusedKeys() ; + void slowIndicateConfigChanged() ; - virtual void timeStampKey(const RsGxsId& id) ; - time_t locked_getLastUsageTS(const RsGxsId& gxs_id); + virtual void timeStampKey(const RsGxsId& id) ; + time_t locked_getLastUsageTS(const RsGxsId& gxs_id); -std::string genRandomId(int len = 20); + std::string genRandomId(int len = 20); #if 0 bool reputation_start(); @@ -479,7 +480,7 @@ std::string genRandomId(int len = 20); bool background_processNewMessages(); bool background_FullCalcRequest(); bool background_processFullCalc(); - + bool background_cleanup(); #endif @@ -489,40 +490,40 @@ std::string genRandomId(int len = 20); /***** below here is locked *****/ bool mLastBgCheck; bool mBgProcessing; - + uint32_t mBgToken; uint32_t mBgPhase; - + std::map mBgGroupMap; std::list mBgFullCalcGroups; #endif -/************************************************************************ + /************************************************************************ * Other Data that is protected by the Mutex. */ - private: +private: std::map > mIdsPendingCache; std::map > mGroupNotPresent; std::map > mIdsNotPresent; std::map mKeysTS ; - - // keep a list of regular contacts. This is useful to sort IDs, and allow some services to priviledged ids only. - std::set mContacts; + + // keep a list of regular contacts. This is useful to sort IDs, and allow some services to priviledged ids only. + std::set mContacts; RsNetworkExchangeService* mNes; /************************** - * AuxUtils provides interface to Security Function (e.g. GPGAuth(), notify etc.) + * AuxUtils provides interface to Security Function (e.g. GPGAuth(), notify etc.) * without depending directly on all these classes. */ - PgpAuxUtils *mPgpUtils; + PgpAuxUtils *mPgpUtils; - time_t mLastKeyCleaningTime ; - time_t mLastConfigUpdate ; + time_t mLastKeyCleaningTime ; + time_t mLastConfigUpdate ; - bool mOwnIdsLoaded ; + bool mOwnIdsLoaded ; }; #endif // P3_IDENTITY_SERVICE_HEADER From 9f9f0216f0eceb933d97f957ae1de368fab321a5 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 3 Jul 2016 22:02:04 -0400 Subject: [PATCH 12/18] remved debug output in udlstck --- libretroshare/src/tcponudp/rsudpstack.h | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/tcponudp/rsudpstack.h b/libretroshare/src/tcponudp/rsudpstack.h index 007a2e845..e2dec61b0 100644 --- a/libretroshare/src/tcponudp/rsudpstack.h +++ b/libretroshare/src/tcponudp/rsudpstack.h @@ -64,8 +64,8 @@ class rsUdpStack: public UdpStack, public pqiNetListener /* from pqiNetListener */ virtual bool resetListener(const struct sockaddr_storage &local) { - std::cerr << "rsUdpStack::resetListener(" << sockaddr_storage_tostring(local) << ")"; - std::cerr << std::endl; + //std::cerr << "rsUdpStack::resetListener(" << sockaddr_storage_tostring(local) << ")"; + //std::cerr << std::endl; if (local.ss_family != AF_INET) { @@ -90,16 +90,16 @@ class rsFixedUdpStack: public UdpStack, public pqiNetListener :UdpStack(testmode, local) { return; } /* from pqiNetListener */ -virtual bool resetListener(const struct sockaddr_storage &local) +virtual bool resetListener(const struct sockaddr_storage & /* local */) { struct sockaddr_in addr; getLocalAddress(addr); // The const_cast below is not so nice but without it, the compiler can't // find the correct operator<<(). No idea why! - std::cerr << "rsFixedUdpStack::resetListener(" << sockaddr_storage_tostring(local) << ")"; - std::cerr << " Resetting with original addr: " << const_cast(addr); - std::cerr << std::endl; + //std::cerr << "rsFixedUdpStack::resetListener(" << sockaddr_storage_tostring(local) << ")"; + //std::cerr << " Resetting with original addr: " << const_cast(addr); + //std::cerr << std::endl; return resetAddress(addr); } From 09ad53981d7abe48684a68b200ad2f753726b280 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 3 Jul 2016 22:02:23 -0400 Subject: [PATCH 13/18] fixed red color in IdDialog --- libretroshare/src/services/p3gxsreputation.cc | 2 +- retroshare-gui/src/gui/Identity/IdDialog.cpp | 32 ++++++++++++------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/services/p3gxsreputation.cc b/libretroshare/src/services/p3gxsreputation.cc index f40beb145..a3e58146e 100644 --- a/libretroshare/src/services/p3gxsreputation.cc +++ b/libretroshare/src/services/p3gxsreputation.cc @@ -151,7 +151,7 @@ p3GxsReputation::p3GxsReputation(p3LinkMgr *lm) mRequestTime = 0; mStoreTime = 0; mReputationsUpdated = false; - mLastActiveFriendsUpdate = 0 ; + mLastActiveFriendsUpdate = time(NULL) - 0.5*ACTIVE_FRIENDS_UPDATE_PERIOD; // avoids doing it too soon since the TS from rsIdentity needs to be loaded already mAverageActiveFriends = 0 ; mLastBannedNodesUpdate = 0 ; } diff --git a/retroshare-gui/src/gui/Identity/IdDialog.cpp b/retroshare-gui/src/gui/Identity/IdDialog.cpp index 1a8d20fde..63d1c93d6 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDialog.cpp @@ -1359,8 +1359,8 @@ bool IdDialog::fillIdListItem(const RsGxsIdGroup& data, QTreeWidgetItem *&item, if (!item) item = new TreeWidgetItem(); - RsIdentityDetails idd ; - rsIdentity->getIdDetails(RsGxsId(data.mMeta.mGroupId),idd) ; + RsIdentityDetails idd ; + rsIdentity->getIdDetails(RsGxsId(data.mMeta.mGroupId),idd) ; item->setText(RSID_COL_NICKNAME, QString::fromUtf8(data.mMeta.mGroupName.c_str()).left(RSID_MAXIMUM_NICKNAME_SIZE)); item->setText(RSID_COL_KEYID, QString::fromStdString(data.mMeta.mGroupId.toStdString())); @@ -1368,13 +1368,21 @@ bool IdDialog::fillIdListItem(const RsGxsIdGroup& data, QTreeWidgetItem *&item, //time_t now = time(NULL) ; //item->setText(RSID_COL_LASTUSED, getHumanReadableDuration(now - data.mLastUsageTS)) ; - + if(idd.mReputation.mAssessment == RsReputations::ASSESSMENT_BAD) { - item->setForeground(RSID_COL_NICKNAME,QBrush(Qt::red)); - item->setForeground(RSID_COL_KEYID,QBrush(Qt::red)); - item->setForeground(RSID_COL_IDTYPE,QBrush(Qt::red)); + std::cerr << "BAD: Identity " << data.mMeta.mGroupId << ": BAD" << std::endl; + item->setForeground(RSID_COL_NICKNAME,QBrush(Qt::red)); + item->setForeground(RSID_COL_KEYID,QBrush(Qt::red)); + item->setForeground(RSID_COL_IDTYPE,QBrush(Qt::red)); } + else + { + item->setForeground(RSID_COL_NICKNAME,QBrush(Qt::black)); + item->setForeground(RSID_COL_KEYID,QBrush(Qt::black)); + item->setForeground(RSID_COL_IDTYPE,QBrush(Qt::black)); + } + item->setData(RSID_COL_KEYID, Qt::UserRole,QVariant(item_flags)) ; item->setTextAlignment(RSID_COL_VOTES, Qt::AlignRight); @@ -1744,11 +1752,11 @@ void IdDialog::insertIdDetails(uint32_t token) RsReputations::ReputationInfo info ; rsReputations->getReputationInfo(RsGxsId(data.mMeta.mGroupId),data.mPgpId,info) ; - - ui->neighborNodesOpinion_TF->setText(QString::number(info.mFriendAverage - 1.0f)); - ui->overallOpinion_TF->setText(QString::number(info.mOverallReputationScore - 1.0f) +" ("+ - ((info.mAssessment == RsReputations::ASSESSMENT_OK)? tr("OK") : tr("Banned")) +")" ) ; + ui->neighborNodesOpinion_TF->setText(QString::number(info.mFriendAverage - 1.0f)); + + ui->overallOpinion_TF->setText(QString::number(info.mOverallReputationScore - 1.0f) +" ("+ + ((info.mAssessment == RsReputations::ASSESSMENT_OK)? tr("OK") : tr("Banned")) +")" ) ; switch(info.mOwnOpinion) { @@ -2024,10 +2032,10 @@ void IdDialog::IdListCustomPopupMenu( QPoint ) #endif RsGxsId keyId((*it)->text(RSID_COL_KEYID).toStdString()); - RsIdentityDetails det ; + RsIdentityDetails det ; rsIdentity->getIdDetails(keyId,det) ; - switch(det.mReputation.mOwnOpinion) + switch(det.mReputation.mOwnOpinion) { case RsReputations::OPINION_NEGATIVE: ++n_negative_reputations ; break ; From 218ca45ac824bcaf3b295e28e61a0696bbad5a2e Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 3 Jul 2016 23:12:56 -0400 Subject: [PATCH 14/18] removed some debug info in rsdataservice --- libretroshare/src/gxs/rsdataservice.cc | 124 +++++++++--------- libretroshare/src/services/p3gxsreputation.cc | 1 - 2 files changed, 65 insertions(+), 60 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index f8bc384de..10f7d073e 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -1309,46 +1309,46 @@ int RsDataService::retrieveGxsGrpMetaData(std::map::const_iterator it(mGrpMetaDataCache.begin());it!=mGrpMetaDataCache.end();++it) - grp[it->first] = new RsGxsGrpMetaData(it->second); + grp[it->first] = new RsGxsGrpMetaData(it->second); } else - { + { #ifdef RS_DATA_SERVICE_DEBUG - std::cerr << "RsDataService::retrieveGxsGrpMetaData() retrieving all" << std::endl; + std::cerr << "RsDataService::retrieveGxsGrpMetaData() retrieving all" << std::endl; #endif - RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "", ""); + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "", ""); - if(c) - { - bool valid = c->moveToFirst(); + if(c) + { + bool valid = c->moveToFirst(); - while(valid) - { - RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); - if(g) - { - grp[g->mGroupId] = g; - mGrpMetaDataCache[g->mGroupId] = *g ; + while(valid) + { + RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); + if(g) + { + grp[g->mGroupId] = g; + mGrpMetaDataCache[g->mGroupId] = *g ; #ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << (void *)this << ": Retrieving (all) Grp metadata grpId=" << g->mGroupId << std::endl; + std::cerr << (void *)this << ": Retrieving (all) Grp metadata grpId=" << g->mGroupId << std::endl; #endif - } - valid = c->moveToNext(); + } + valid = c->moveToNext(); #ifdef RS_DATA_SERVICE_DEBUG_TIME - ++resultCount; + ++resultCount; #endif - } - delete c; - } - - mGrpMetaDataCache_ContainsAllDatabase = true ; - } + } + delete c; + } + + mGrpMetaDataCache_ContainsAllDatabase = true ; + } }else { @@ -1360,52 +1360,52 @@ int RsDataService::retrieveGxsGrpMetaData(std::mapfirst] = new RsGxsGrpMetaData(itt->second) ; + grp[mit->first] = new RsGxsGrpMetaData(itt->second) ; } else - { + { #ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << "Retrieving Grp metadata grpId=" << mit->first ; + std::cerr << "Retrieving Grp metadata grpId=" << mit->first ; #endif - const RsGxsGroupId& grpId = mit->first; - RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "grpId='" + grpId.toStdString() + "'", ""); + const RsGxsGroupId& grpId = mit->first; + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, mGrpMetaColumns, "grpId='" + grpId.toStdString() + "'", ""); - if(c) - { - bool valid = c->moveToFirst(); + if(c) + { + bool valid = c->moveToFirst(); //#ifdef RS_DATA_SERVICE_DEBUG_CACHE - if(!valid) - std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl; + if(!valid) + std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl; //#endif - while(valid) - { - RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); + while(valid) + { + RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); - if(g) - { - grp[g->mGroupId] = g; - mGrpMetaDataCache[g->mGroupId] = *g ; + if(g) + { + grp[g->mGroupId] = g; + mGrpMetaDataCache[g->mGroupId] = *g ; #ifdef RS_DATA_SERVICE_DEBUG_CACHE - std::cerr << ". Got it. Updating cache." << std::endl; + std::cerr << ". Got it. Updating cache." << std::endl; #endif - } - valid = c->moveToNext(); + } + valid = c->moveToNext(); #ifdef RS_DATA_SERVICE_DEBUG_TIME - ++resultCount; + ++resultCount; #endif - } - delete c; - } + } + delete c; + } #ifdef RS_DATA_SERVICE_DEBUG_CACHE - else - std::cerr << ". not found!" << std::endl; + else + std::cerr << ". not found!" << std::endl; #endif - } + } } } @@ -1442,23 +1442,29 @@ int RsDataService::resetDataStore() int RsDataService::updateGroupMetaData(GrpLocMetaData &meta) { +#ifdef RS_DATA_SERVICE_DEBUG_CACHE std::cerr << (void*)this << ": Updating Grp Meta data: grpId = " << meta.grpId << std::endl; - +#endif + RsStackMutex stack(mDbMutex); RsGxsGroupId& grpId = meta.grpId; - + +#ifdef RS_DATA_SERVICE_DEBUG_CACHE std::cerr << (void*)this << ": erasing old entry from cache." << std::endl; - +#endif + mGrpMetaDataCache_ContainsAllDatabase = false ; mGrpMetaDataCache.erase(meta.grpId) ; - + return mDb->sqlUpdate(GRP_TABLE_NAME, KEY_GRP_ID+ "='" + grpId.toStdString() + "'", meta.val) ? 1 : 0; } int RsDataService::updateMessageMetaData(MsgLocMetaData &metaData) { +#ifdef RS_DATA_SERVICE_DEBUG_CACHE std::cerr << (void*)this << ": Updating Msg Meta data: grpId = " << metaData.msgId.first << " msgId = " << metaData.msgId.second << std::endl; - +#endif + RsStackMutex stack(mDbMutex); RsGxsGroupId& grpId = metaData.msgId.first; RsGxsMessageId& msgId = metaData.msgId.second; diff --git a/libretroshare/src/services/p3gxsreputation.cc b/libretroshare/src/services/p3gxsreputation.cc index a3e58146e..680700ce3 100644 --- a/libretroshare/src/services/p3gxsreputation.cc +++ b/libretroshare/src/services/p3gxsreputation.cc @@ -39,7 +39,6 @@ /**** * #define DEBUG_REPUTATION 1 ****/ -#define DEBUG_REPUTATION 1 /************ IMPLEMENTATION NOTES ********************************* * From 42f221075a96b2db4e35395459dfbe7090ba7643 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 3 Jul 2016 23:15:05 -0400 Subject: [PATCH 15/18] removed debug info in IdDialog --- retroshare-gui/src/gui/Identity/IdDialog.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/retroshare-gui/src/gui/Identity/IdDialog.cpp b/retroshare-gui/src/gui/Identity/IdDialog.cpp index 63d1c93d6..c9c66fa2b 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDialog.cpp @@ -1365,13 +1365,8 @@ bool IdDialog::fillIdListItem(const RsGxsIdGroup& data, QTreeWidgetItem *&item, item->setText(RSID_COL_NICKNAME, QString::fromUtf8(data.mMeta.mGroupName.c_str()).left(RSID_MAXIMUM_NICKNAME_SIZE)); item->setText(RSID_COL_KEYID, QString::fromStdString(data.mMeta.mGroupId.toStdString())); - //time_t now = time(NULL) ; - //item->setText(RSID_COL_LASTUSED, getHumanReadableDuration(now - data.mLastUsageTS)) ; - - if(idd.mReputation.mAssessment == RsReputations::ASSESSMENT_BAD) { - std::cerr << "BAD: Identity " << data.mMeta.mGroupId << ": BAD" << std::endl; item->setForeground(RSID_COL_NICKNAME,QBrush(Qt::red)); item->setForeground(RSID_COL_KEYID,QBrush(Qt::red)); item->setForeground(RSID_COL_IDTYPE,QBrush(Qt::red)); @@ -1384,7 +1379,6 @@ bool IdDialog::fillIdListItem(const RsGxsIdGroup& data, QTreeWidgetItem *&item, } item->setData(RSID_COL_KEYID, Qt::UserRole,QVariant(item_flags)) ; - item->setTextAlignment(RSID_COL_VOTES, Qt::AlignRight); item->setData(RSID_COL_VOTES,Qt::DisplayRole, QString::number(idd.mReputation.mOverallReputationScore - 1.0f,'f',3)); From 95915fa31de10703adc12cce5419f4a2f54939f8 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 10 Jul 2016 22:46:37 -0400 Subject: [PATCH 16/18] removed cross deadlock between p3GxsReputation and p3IdService --- libretroshare/src/services/p3gxsreputation.cc | 73 +++++++++++-------- 1 file changed, 41 insertions(+), 32 deletions(-) diff --git a/libretroshare/src/services/p3gxsreputation.cc b/libretroshare/src/services/p3gxsreputation.cc index 680700ce3..56c2d8cc2 100644 --- a/libretroshare/src/services/p3gxsreputation.cc +++ b/libretroshare/src/services/p3gxsreputation.cc @@ -333,48 +333,57 @@ void p3GxsReputation::updateIdentityFlags() void p3GxsReputation::cleanup() { - // remove opinions from friends that havn't been seen online for more than the specified delay + // remove opinions from friends that havn't been seen online for more than the specified delay #ifdef DEBUG_REPUTATION #endif - std::cerr << "p3GxsReputation::cleanup() " << std::endl; + std::cerr << "p3GxsReputation::cleanup() " << std::endl; - // remove optionions about identities that do not exist anymore. That will in particular avoid asking p3idservice about deleted - // identities, which would cause an excess of hits to the database. + // remove optionions about identities that do not exist anymore. That will in particular avoid asking p3idservice about deleted + // identities, which would cause an excess of hits to the database. + // We do it in two steps to avoid a deadlock when calling rsIdentity from here. - bool updated = false ; - time_t now = time(NULL) ; + bool updated = false ; + time_t now = time(NULL) ; - RsStackMutex stack(mReputationMtx); /****** LOCKED MUTEX *******/ + std::list ids_to_check_for_last_usage_ts; - for(std::map::iterator it(mReputations.begin());it!=mReputations.end();) - if(it->second.mOpinions.empty() && it->second.mOwnOpinion == RsReputations::OPINION_NEUTRAL) - { - std::map::iterator tmp(it) ; - ++tmp ; - mReputations.erase(it) ; - it = tmp ; + { + RsStackMutex stack(mReputationMtx); /****** LOCKED MUTEX *******/ + + for(std::map::iterator it(mReputations.begin());it!=mReputations.end();) + if(it->second.mOpinions.empty() && it->second.mOwnOpinion == RsReputations::OPINION_NEUTRAL) + { + std::map::iterator tmp(it) ; + ++tmp ; + mReputations.erase(it) ; + it = tmp ; #ifdef DEBUG_REPUTATION - std::cerr << " ID " << it->first << ": own is neutral and no opinions from friends => remove entry" << std::endl; + std::cerr << " ID " << it->first << ": own is neutral and no opinions from friends => remove entry" << std::endl; #endif - updated = true ; - } - else if(rsIdentity->getLastUsageTS(it->first) + REPUTATION_INFO_KEEP_DELAY < now) - { -#ifdef DEBUG_REPUTATION - std::cerr << " Identity " << it->first << " has a last usage TS of " << now - rsIdentity->getLastUsageTS(it->first) << " secs ago: deleting it." << std::endl; -#endif - std::map::iterator tmp(it) ; - ++tmp ; - mReputations.erase(it) ; - it = tmp ; - updated = true ; - } - else - ++it ; + updated = true ; + } + else + { + ids_to_check_for_last_usage_ts.push_back(it->first) ; + ++it; + } + } - if(updated) - IndicateConfigChanged() ; + for(std::list::const_iterator it(ids_to_check_for_last_usage_ts.begin());it!=ids_to_check_for_last_usage_ts.end();++it) + if(rsIdentity->getLastUsageTS(*it) + REPUTATION_INFO_KEEP_DELAY < now) + { +#ifdef DEBUG_REPUTATION + std::cerr << " Identity " << *it << " has a last usage TS of " << now - rsIdentity->getLastUsageTS(*it) << " secs ago: deleting it." << std::endl; +#endif + + RsStackMutex stack(mReputationMtx); /****** LOCKED MUTEX *******/ + mReputations.erase(*it) ; + updated = true ; + } + + if(updated) + IndicateConfigChanged() ; } void p3GxsReputation::updateActiveFriends() From 951f29f15551deec2dbeb53a657f8e49c27d6284 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 10 Jul 2016 23:00:47 -0400 Subject: [PATCH 17/18] removed debug info in rsDataService --- libretroshare/src/gxs/rsdataservice.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 10f7d073e..b8eadb55d 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -1377,10 +1377,10 @@ int RsDataService::retrieveGxsGrpMetaData(std::mapmoveToFirst(); -//#ifdef RS_DATA_SERVICE_DEBUG_CACHE - if(!valid) - std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl; -//#endif +#ifdef RS_DATA_SERVICE_DEBUG_CACHE + if(!valid) + std::cerr << " Empty query! GrpId " << grpId << " is not in database" << std::endl; +#endif while(valid) { RsGxsGrpMetaData* g = locked_getGrpMeta(*c, 0); From a7d82cf197ed20057ead7498bb0c645426077261 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 11 Jul 2016 18:44:52 -0400 Subject: [PATCH 18/18] delete GXS IDs with no TS --- libretroshare/src/services/p3idservice.cc | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index f521f38be..29b5e0794 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -340,17 +340,15 @@ public: std::map::const_iterator it = mLastUsageTS.find(gxs_id) ; - if(it == mLastUsageTS.end()) - { - std::cerr << "No Ts for this ID => kept" << std::endl; - return true ; - } + bool no_ts = (it == mLastUsageTS.end()) ; - time_t last_usage_ts = it->second; + time_t last_usage_ts = no_ts?0:(it->second); time_t max_keep_time ; - if(is_id_banned) - max_keep_time = MAX_KEEP_KEYS_BANNED ; + if(no_ts) + max_keep_time = 0 ; + else if(is_id_banned) + max_keep_time = MAX_KEEP_KEYS_BANNED ; else if(is_known_id) max_keep_time = MAX_KEEP_KEYS_SIGNED_KNOWN ; else if(is_signed_id)