From 750dd526c27bc6bcf5f807de26c92176340e2806 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Fri, 16 Aug 2013 20:50:02 +0000 Subject: [PATCH 01/29] To add last couple features for GXS before 0.6 release - group update - sharing admin and publish keys - tighten up security git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6580 b45a01b8-16f6-495d-af2f-9b41ad6348cc From 33eb99d812634e0ea0d6b99872652c8ffe41e365 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Wed, 11 Sep 2013 20:42:56 +0000 Subject: [PATCH 02/29] added group update feature git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6733 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsdataservice.cc | 120 +++++++++++++++++++++++++ libretroshare/src/gxs/rsdataservice.h | 15 ++++ libretroshare/src/gxs/rsgds.h | 23 ++++- libretroshare/src/gxs/rsgenexchange.cc | 95 +++++++++++++++++++- libretroshare/src/gxs/rsgenexchange.h | 18 ++++ libretroshare/src/gxs/rsgxsutil.h | 10 +++ 6 files changed, 277 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 6836fa028..f27633af2 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -177,6 +177,8 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d // for retrieving msg offsets mMsgOffSetColumns.push_back(KEY_MSG_ID); mMsgOffSetColumns.push_back(KEY_NXS_FILE_OFFSET); mMsgOffSetColumns.push_back(KEY_NXS_FILE_LEN); + + grpIdColumn.push_back(KEY_GRP_ID); } RsDataService::~RsDataService(){ @@ -641,6 +643,99 @@ int RsDataService::storeGroup(std::map &grp) return ret; } +int RsDataService::updateGroup(std::map &grp) +{ + + RsStackMutex stack(mDbMutex); + + std::map::iterator sit = grp.begin(); + + // begin transaction + mDb->execSQL("BEGIN;"); + + for(; sit != grp.end(); sit++) + { + + RsNxsGrp* grpPtr = sit->first; + RsGxsGrpMetaData* grpMetaPtr = sit->second; + + // if data is larger than max item size do not add + if(!validSize(grpPtr)) continue; + + std::string grpFile = mServiceDir + "/" + grpPtr->grpId; + std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::app | std::ios::out); + ostrm.seekg(0, std::ios::end); // go to end to append + uint32_t offset = ostrm.tellg(); // get fill offset + + /*! + * STORE file offset, file length, file name, + * grpId, flags, publish time stamp, identity, + * id signature, admin signatue, key set, last posting ts + * and meta data + **/ + ContentValue cv; + cv.put(KEY_NXS_FILE_OFFSET, (int32_t)offset); + cv.put(KEY_NXS_FILE_LEN, (int32_t)grpPtr->grp.TlvSize()); + cv.put(KEY_NXS_FILE, grpFile); + cv.put(KEY_GRP_ID, grpPtr->grpId); + cv.put(KEY_GRP_NAME, grpMetaPtr->mGroupName); + cv.put(KEY_ORIG_GRP_ID, grpMetaPtr->mOrigGrpId); + cv.put(KEY_NXS_SERV_STRING, grpMetaPtr->mServiceString); + cv.put(KEY_NXS_FLAGS, (int32_t)grpMetaPtr->mGroupFlags); + cv.put(KEY_TIME_STAMP, (int32_t)grpMetaPtr->mPublishTs); + cv.put(KEY_GRP_SIGN_FLAGS, (int32_t)grpMetaPtr->mSignFlags); + cv.put(KEY_GRP_CIRCLE_ID, grpMetaPtr->mCircleId); + cv.put(KEY_GRP_CIRCLE_TYPE, (int32_t)grpMetaPtr->mCircleType); + cv.put(KEY_GRP_INTERNAL_CIRCLE, grpMetaPtr->mInternalCircle); + cv.put(KEY_GRP_ORIGINATOR, grpMetaPtr->mOriginator); + cv.put(KEY_GRP_AUTHEN_FLAGS, (int32_t)grpMetaPtr->mAuthenFlags); + cv.put(KEY_NXS_HASH, grpMetaPtr->mHash); + + if(! (grpMetaPtr->mAuthorId.empty()) ){ + cv.put(KEY_NXS_IDENTITY, grpMetaPtr->mAuthorId); + } + + offset = 0; + char keySetData[grpMetaPtr->keys.TlvSize()]; + grpMetaPtr->keys.SetTlv(keySetData, grpMetaPtr->keys.TlvSize(), &offset); + cv.put(KEY_KEY_SET, grpMetaPtr->keys.TlvSize(), keySetData); + + offset = 0; + char metaData[grpPtr->meta.TlvSize()]; + grpPtr->meta.SetTlv(metaData, grpPtr->meta.TlvSize(), &offset); + cv.put(KEY_NXS_META, grpPtr->meta.TlvSize(), metaData); + + // local meta data + cv.put(KEY_GRP_SUBCR_FLAG, (int32_t)grpMetaPtr->mSubscribeFlags); + cv.put(KEY_GRP_POP, (int32_t)grpMetaPtr->mPop); + cv.put(KEY_MSG_COUNT, (int32_t)grpMetaPtr->mMsgCount); + cv.put(KEY_GRP_STATUS, (int32_t)grpMetaPtr->mGroupStatus); + cv.put(KEY_GRP_LAST_POST, (int32_t)grpMetaPtr->mLastPost); + + offset = 0; + char grpData[grpPtr->grp.TlvSize()]; + grpPtr->grp.SetTlv(grpData, grpPtr->grp.TlvSize(), &offset); + ostrm.write(grpData, grpPtr->grp.TlvSize()); + ostrm.close(); + + mDb->sqlUpdate(GRP_TABLE_NAME, "grpId='" + grpPtr->grpId + "'", cv); + } + // finish transaction + bool ret = mDb->execSQL("COMMIT;"); + + for(sit = grp.begin(); sit != grp.end(); sit++) + { + //TODO: API encourages aliasing, remove this abomination + if(sit->second != sit->first->metaData) + delete sit->second; + delete sit->first; + + } + + return ret; +} + + bool RsDataService::validSize(RsNxsGrp* grp) const { if((grp->grp.TlvSize() + grp->meta.TlvSize()) <= GXS_MAX_ITEM_SIZE) return true; @@ -1177,6 +1272,31 @@ int RsDataService::removeGroups(const std::vector &grpIds) return 1; } +int RsDataService::retrieveGroupIds(std::vector &grpIds) +{ + RsStackMutex stack(mDbMutex); + + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpIdColumn, "", ""); + + if(c) + { + bool valid = c->moveToFirst(); + + while(valid) + { + std::string grpId; + c->getString(0, grpId); + grpIds.push_back(grpId); + valid = c->moveToNext(); + } + delete c; + }else + { + return 0; + } + + return 1; +} bool RsDataService::locked_updateMessageEntries(const MsgUpdates& updates) { diff --git a/libretroshare/src/gxs/rsdataservice.h b/libretroshare/src/gxs/rsdataservice.h index 3b8d0bf2a..666aa8e98 100644 --- a/libretroshare/src/gxs/rsdataservice.h +++ b/libretroshare/src/gxs/rsdataservice.h @@ -106,6 +106,13 @@ public: */ int removeGroups(const std::vector& grpIds); + /*! + * Retrieves all group ids in store + * @param grpIds all grpids in store is inserted into this vector + * @return error code + */ + int RsDataService::retrieveGroupIds(std::vector &grpIds); + /*! * @return the cache size set for this RsGeneralDataService in bytes */ @@ -130,6 +137,13 @@ public: */ int storeGroup(std::map& grp); + /*! + * Updates group entries in Db + * @param grp map of group and decoded meta data + * @return error code + */ + int updateGroup(std::map& grsp); + /*! * @param metaData The meta data item to update * @return error code @@ -238,6 +252,7 @@ private: std::list grpColumns; std::list grpMetaColumns; + std::list grpIdColumn; std::string mServiceDir, mDbName; uint16_t mServType; diff --git a/libretroshare/src/gxs/rsgds.h b/libretroshare/src/gxs/rsgds.h index c7a5cf387..9a82d3687 100644 --- a/libretroshare/src/gxs/rsgds.h +++ b/libretroshare/src/gxs/rsgds.h @@ -167,7 +167,14 @@ public: * @param grpIds ids of groups to be removed * @return error code */ - virtual int removeGroups(const std::vector& grpIds) = 0; + virtual int removeGroups(std::vector& grpIds) = 0; + + /*! + * Retrieves all group ids in store + * @param grpIds all grpids in store is inserted into this vector + * @return error code + */ + virtual int retrieveGroupIds(const std::vector& grpIds) = 0; /*! * @return the cache size set for this RsGeneralDataService in bytes @@ -186,6 +193,13 @@ public: */ virtual int storeMessage(std::map& msgs) = 0; + /*! + * Stores a list of signed messages into data store + * @param msg map of message and decoded meta data information + * @return error code + */ + virtual int storeMessage(std::map& msgs) = 0; + /*! * Stores a list of groups in data store * @param grp map of group and decoded meta data @@ -194,6 +208,13 @@ public: virtual int storeGroup(std::map& grsp) = 0; + /*! + * Updates group entries in Db + * @param grp map of group and decoded meta data + * @return error code + */ + virtual int updateGroup(std::map& grsp) = 0; + /*! * @param metaData */ diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 5133d79ca..eac88e3b0 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -2038,6 +2038,8 @@ void RsGenExchange::processRecvdData() processRecvdGroups(); processRecvdMessages(); + + performUpdateValidation(); } @@ -2193,9 +2195,11 @@ void RsGenExchange::processRecvdGroups() RsStackMutex stack(mGenMtx); NxsGrpPendValidVect::iterator vit = mReceivedGrps.begin(); + std::vector existingGrpIds, grpIds; + std::map grps; - std::list grpIds; + mDataStore->retrieveGroupIds(existingGrpIds); while( vit != mReceivedGrps.end()) { @@ -2224,9 +2228,19 @@ void RsGenExchange::processRecvdGroups() meta->mOriginator = grp->PeerId(); computeHash(grp->grp, meta->mHash); - grps.insert(std::make_pair(grp, meta)); - grpIds.push_back(grp->grpId); + // now check if group already existss + if(std::find(existingGrpIds.begin(), existingGrpIds.end(), grp->grpId) == existingGrpIds.end()) + { + grps.insert(std::make_pair(grp, meta)); + grpIds.push_back(grp->grpId); + } + else + { + GroupUpdate update; + update.newGrp = grp; + mGroupUpdates.push_back(update); + } erase = true; } else if(ret == VALIDATE_FAIL) @@ -2278,3 +2292,78 @@ void RsGenExchange::processRecvdGroups() mDataStore->storeGroup(grps); } } + +void RsGenExchange::performUpdateValidation() +{ +#ifdef GXS_GENX_DEBUG + std::cerr << "RsGenExchange::performUpdateValidation() " << std::endl; +#endif + + std::map grpMetas; + + std::vector::iterator vit = mGroupUpdates.begin(); + for(; vit != mGroupUpdates.end(); vit++) + grpMetas.insert(std::make_pair(vit->newGrp->grpId, (RsGxsGrpMetaData*)NULL)); + + mDataStore->retrieveGxsGrpMetaData(grpMetas); + + vit = mGroupUpdates.begin(); + for(; vit != mGroupUpdates.end(); vit++) + { + GroupUpdate& gu = *vit; + std::map::iterator mit = + grpMetas.find(gu.newGrp->grpId); + gu.oldGrpMeta = mit->second; + gu.validUpdate = updateValid(*(gu.oldGrpMeta), *(gu.newGrp)); + } + +#ifdef GXS_GENX_DEBUG + std::cerr << "RsGenExchange::performUpdateValidation() " << std::endl; +#endif + + vit = mGroupUpdates.begin(); + std::map grps; + for(; vit != mGroupUpdates.end(); vit++) + { + GroupUpdate& gu = *vit; + grps.insert(std::make_pair(gu.newGrp, gu.newGrp->metaData)); + delete gu.oldGrpMeta; + } + + mDataStore->updateGroup(grps); + mGroupUpdates.clear(); +} + +bool RsGenExchange::updateValid(RsGxsGrpMetaData& oldGrpMeta, RsNxsGrp& newGrp) const +{ + std::map& signSet = newGrp.metaData->signSet.keySignSet; + std::map::iterator mit = signSet.find(GXS_SERV::FLAG_AUTHEN_ADMIN); + + if(mit == signSet.end()) + { +#ifdef GXS_GENX_DEBUG + std::cerr << "RsGenExchange::updateValid() new admin sign not found! " << std::endl; + std::cerr << "RsGenExchange::updateValid() grpId: " << oldGrp.grpId << std::endl; +#endif + + return false; + } + + RsTlvKeySignature& adminSign = mit->second; + + std::map& keys = oldGrpMeta.keys.keys; + std::map::iterator keyMit = keys.find(oldGrpMeta.mGroupId); + + if(keyMit == keys.end()) + { +#ifdef GXS_GENX_DEBUG + std::cerr << "RsGenExchange::updateValid() admin key not found! " << std::endl; +#endif + return false; + } + + // also check this is the latest published group + bool latest = newGrp.metaData->mPublishTs > oldGrpMeta.mPublishTs; + + return GxsSecurity::validateNxsGrp(newGrp, adminSign, keyMit->second) && latest; +} diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index a9d3d7b4a..dc6a5026c 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -736,6 +736,20 @@ private: static void computeHash(const RsTlvBinaryData& data, std::string& hash); + /*! + * Checks validation of recently received groups to be + * updated + */ + void performUpdateValidation(); + + /*! + * Checks if the update is valid (i.e. the new admin signature is by the old admin key) + * @param oldGrp the old group to be updated (must have meta data member initialised) + * @param newGrp the new group that updates the old group (must have meta data member initialised) + * @return + */ + bool updateValid(RsGxsGrpMetaData& oldGrp, RsNxsGrp& newGrp) const; + private: RsMutex mGenMtx; @@ -798,6 +812,10 @@ private: const uint8_t CREATE_FAIL, CREATE_SUCCESS, CREATE_FAIL_TRY_LATER, SIGN_MAX_ATTEMPTS; const uint8_t SIGN_FAIL, SIGN_SUCCESS, SIGN_FAIL_TRY_LATER; const uint8_t VALIDATE_FAIL, VALIDATE_SUCCESS, VALIDATE_FAIL_TRY_LATER, VALIDATE_MAX_ATTEMPTS; + +private: + + std::vector mGroupUpdates; }; #endif // RSGENEXCHANGE_H diff --git a/libretroshare/src/gxs/rsgxsutil.h b/libretroshare/src/gxs/rsgxsutil.h index b15e3479f..f6b367f4a 100644 --- a/libretroshare/src/gxs/rsgxsutil.h +++ b/libretroshare/src/gxs/rsgxsutil.h @@ -129,4 +129,14 @@ private: }; +class GroupUpdate +{ +public: + GroupUpdate() : oldGrpMeta(NULL), newGrp(NULL), validUpdate(false) + {} + RsGxsGrpMetaData* oldGrpMeta; + RsNxsGrp* newGrp; + bool validUpdate; +}; + #endif /* GXSUTIL_H_ */ From 3799910188fef53e4c447f3d92387c9a2b230b33 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 15 Sep 2013 21:18:20 +0000 Subject: [PATCH 03/29] More changes to support front end of group update - still need to make sync changes git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6741 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsdataservice.cc | 2 +- libretroshare/src/gxs/rsdataservice.h | 2 +- libretroshare/src/gxs/rsgds.h | 11 +-- libretroshare/src/gxs/rsgenexchange.cc | 93 +++++++++++++++++++++++- libretroshare/src/gxs/rsgenexchange.h | 23 +++++- libretroshare/src/gxs/rsgxsdataaccess.cc | 7 ++ libretroshare/src/gxs/rsgxsdataaccess.h | 11 ++- 7 files changed, 131 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index f27633af2..a334d090f 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -663,7 +663,7 @@ int RsDataService::updateGroup(std::map &grp) if(!validSize(grpPtr)) continue; std::string grpFile = mServiceDir + "/" + grpPtr->grpId; - std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::app | std::ios::out); + std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::app | std::ios::trunc); ostrm.seekg(0, std::ios::end); // go to end to append uint32_t offset = ostrm.tellg(); // get fill offset diff --git a/libretroshare/src/gxs/rsdataservice.h b/libretroshare/src/gxs/rsdataservice.h index 666aa8e98..53f9c4fe8 100644 --- a/libretroshare/src/gxs/rsdataservice.h +++ b/libretroshare/src/gxs/rsdataservice.h @@ -111,7 +111,7 @@ public: * @param grpIds all grpids in store is inserted into this vector * @return error code */ - int RsDataService::retrieveGroupIds(std::vector &grpIds); + int retrieveGroupIds(std::vector &grpIds); /*! * @return the cache size set for this RsGeneralDataService in bytes diff --git a/libretroshare/src/gxs/rsgds.h b/libretroshare/src/gxs/rsgds.h index 9a82d3687..c2ba4d8ac 100644 --- a/libretroshare/src/gxs/rsgds.h +++ b/libretroshare/src/gxs/rsgds.h @@ -167,14 +167,14 @@ public: * @param grpIds ids of groups to be removed * @return error code */ - virtual int removeGroups(std::vector& grpIds) = 0; + virtual int removeGroups(const std::vector& grpIds) = 0; /*! * Retrieves all group ids in store * @param grpIds all grpids in store is inserted into this vector * @return error code */ - virtual int retrieveGroupIds(const std::vector& grpIds) = 0; + virtual int retrieveGroupIds(std::vector& grpIds) = 0; /*! * @return the cache size set for this RsGeneralDataService in bytes @@ -193,13 +193,6 @@ public: */ virtual int storeMessage(std::map& msgs) = 0; - /*! - * Stores a list of signed messages into data store - * @param msg map of message and decoded meta data information - * @return error code - */ - virtual int storeMessage(std::map& msgs) = 0; - /*! * Stores a list of groups in data store * @param grp map of group and decoded meta data diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index eac88e3b0..d67d36c00 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1369,6 +1369,20 @@ void RsGenExchange::publishGroup(uint32_t& token, RsGxsGrpItem *grpItem) } + +void RsGenExchange::updateGroup(uint32_t& token, RsGxsGrpItem* grpItem) +{ + RsStackMutex stack(mGenMtx); + token = mDataAccess->generatePublicToken(); + mGroupUpdatePublish.push_back(GroupUpdatePublish(grpItem, token)); + +#ifdef GEN_EXCH_DEBUG + std::cerr << "RsGenExchange::updateGroup() token: " << token; + std::cerr << std::endl; +#endif +} + + void RsGenExchange::publishMsg(uint32_t& token, RsGxsMsgItem *msgItem) { RsStackMutex stack(mGenMtx); @@ -1785,6 +1799,76 @@ RsGenExchange::ServiceCreate_Return RsGenExchange::service_CreateGroup(RsGxsGrpI #define PENDING_SIGN_TIMEOUT 10 // 5 seconds + +void RsGenExchange::processGroupUpdatePublish() +{ + + RsStackMutex stack(mGenMtx); + + // get keys for group update publish + + // first build meta request map for groups to be updated + std::map grpMeta; + std::vector::iterator vit = mGroupUpdatePublish.begin(); + + for(; vit != mGroupUpdatePublish.end(); vit++) + { + GroupUpdatePublish& gup = *vit; + const RsGxsGroupId& groupId = gup.grpItem->meta.mGroupId; + grpMeta.insert(std::make_pair(groupId, (RsGxsGrpMetaData*)(NULL))); + } + + mDataStore->retrieveGxsGrpMetaData(grpMeta); + + // now + vit = mGroupUpdatePublish.begin(); + for(; vit != mGroupUpdatePublish.end(); vit++) + { + GroupUpdatePublish& gup = *vit; + const RsGxsGroupId& groupId = gup.grpItem->meta.mGroupId; + RsGxsGrpMetaData* meta = grpMeta[groupId]; + + GxsGrpPendingSign ggps(ggps.mItem, ggps.mToken); + + bool split = splitKeys(meta->keys, ggps.mPrivateKeys, ggps.mPublicKeys); + + if(split) + { + ggps.mHaveKeys = true; + ggps.mStartTS = time(NULL); + ggps.mLastAttemptTS = 0; + mGrpsToPublish.push_back(ggps); + }else + { + delete gup.grpItem; + mDataAccess->updatePublicRequestStatus(gup.mToken, RsTokenService::GXS_REQUEST_V2_STATUS_FAILED); + } + + } + + mGroupUpdatePublish.clear(); +} + +bool RsGenExchange::splitKeys(const RsTlvSecurityKeySet& keySet, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet) +{ + + typedef std::map keyMap; + const keyMap& allKeys = keySet.keys; + keyMap::const_iterator cit = allKeys.begin(); + + for(; cit != allKeys.end(); cit++) + { + const RsTlvSecurityKey& key = cit->second; + if(key.keyFlags & RSTLV_KEY_TYPE_PUBLIC_ONLY) + publicKeySet.keys.insert(std::make_pair(key.keyId, key)); + else if(key.keyFlags & RSTLV_KEY_TYPE_FULL) + privateKeySet.keys.insert(std::make_pair(key.keyId, key)); + } + + // user must have both private and public parts of publish and admin keys + return (privateKeySet.keys.size() == 2) && (publicKeySet.keys.size() == 2); +} + void RsGenExchange::publishGrps() { RsStackMutex stack(mGenMtx); @@ -1906,7 +1990,11 @@ void RsGenExchange::publishGrps() { grpId = grp->grpId; computeHash(grp->grp, grp->metaData->mHash); - mDataAccess->addGroupData(grp); + + if(ggps.mIsUpdate) + mDataAccess->updateGroupData(grp); + else + mDataAccess->addGroupData(grp); } else { @@ -2195,7 +2283,8 @@ void RsGenExchange::processRecvdGroups() RsStackMutex stack(mGenMtx); NxsGrpPendValidVect::iterator vit = mReceivedGrps.begin(); - std::vector existingGrpIds, grpIds; + std::vector existingGrpIds; + std::list grpIds; std::map grps; diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index dc6a5026c..b7dc6e966 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -71,13 +71,14 @@ class GxsGrpPendingSign public: GxsGrpPendingSign(RsGxsGrpItem* item, uint32_t token): mLastAttemptTS(0), mStartTS(time(NULL)), mToken(token), - mItem(item), mHaveKeys(false) + mItem(item), mHaveKeys(false), mIsUpdate(false) {} time_t mLastAttemptTS, mStartTS; uint32_t mToken; RsGxsGrpItem* mItem; bool mHaveKeys; // mKeys->first == true if key present + bool mIsUpdate; RsTlvSecurityKeySet mPrivateKeys; RsTlvSecurityKeySet mPublicKeys; }; @@ -516,7 +517,6 @@ protected: /*! * Enables publication of a group item \n - * If the item exists already this is simply versioned \n * This will induce a related change message \n * Ownership of item passes to this rsgenexchange \n * @param token @@ -524,6 +524,16 @@ protected: */ void publishGroup(uint32_t& token, RsGxsGrpItem* grpItem); + + /*! + * Updates an existing group item \n + * This will induce a related change message \n + * Ownership of item passes to this rsgenexchange \n + * @param token + * @param grpItem + */ + void updateGroup(uint32_t& token, RsGxsGrpItem* grpItem); + public: /*! * Enables publication of a message item \n @@ -629,6 +639,8 @@ private: void publishGrps(); + void processGroupUpdatePublish(); + void publishMsgs(); /*! @@ -750,6 +762,8 @@ private: */ bool updateValid(RsGxsGrpMetaData& oldGrp, RsNxsGrp& newGrp) const; + bool splitKeys(const RsTlvSecurityKeySet& keySet, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet); + private: RsMutex mGenMtx; @@ -815,7 +829,10 @@ private: private: - std::vector mGroupUpdates; + std::vector mGroupUpdates, mPeersGroupUpdate; + + std::vector mGroupUpdatePublish; + }; #endif // RSGENEXCHANGE_H diff --git a/libretroshare/src/gxs/rsgxsdataaccess.cc b/libretroshare/src/gxs/rsgxsdataaccess.cc index 5399c746e..39c428b37 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.cc +++ b/libretroshare/src/gxs/rsgxsdataaccess.cc @@ -1487,7 +1487,14 @@ bool RsGxsDataAccess::addGroupData(RsNxsGrp* grp) { return mDataStore->storeGroup(grpM); } +bool RsGxsDataAccess::updateGroupData(RsNxsGrp* grp) { + RsStackMutex stack(mDataMutex); + + std::map grpM; + grpM.insert(std::make_pair(grp, grp->metaData)); + return mDataStore->updateGroup(grpM); +} bool RsGxsDataAccess::addMsgData(RsNxsMsg* msg) { diff --git a/libretroshare/src/gxs/rsgxsdataaccess.h b/libretroshare/src/gxs/rsgxsdataaccess.h index de5258569..3b0aa8113 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.h +++ b/libretroshare/src/gxs/rsgxsdataaccess.h @@ -126,14 +126,21 @@ public: public: /*! - * This adds a groups to the gxs data base, this is a blocking call - * Responsibility for grp still lies with callee \n + * This adds a groups to the gxs data base, this is a blocking call \n * If function returns successfully DataAccess can be queried for grp * @param grp the group to add, responsibility grp passed lies with callee * @return false if group cound not be added */ bool addGroupData(RsNxsGrp* grp); + /*! + * This updates a groups in the gxs data base, this is a blocking call \n + * If function returns successfully DataAccess can be queried for grp + * @param grp the group to add, responsibility grp passed lies with callee + * @return false if group cound not be added + */ + bool updateGroupData(RsNxsGrp* grp); + /*! * This adds a group to the gxs data base, this is a blocking call \n * Responsibility for msg still lies with callee \n From 2a841d7ab3342fca8361e9c0108a88c338d5a86f Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Wed, 18 Sep 2013 18:13:38 +0000 Subject: [PATCH 04/29] fixed mem bug git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6747 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgenexchange.cc | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index d67d36c00..5b4baff0d 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -2388,13 +2388,19 @@ void RsGenExchange::performUpdateValidation() std::cerr << "RsGenExchange::performUpdateValidation() " << std::endl; #endif + if(mGroupUpdates.empty()) + return; + std::map grpMetas; std::vector::iterator vit = mGroupUpdates.begin(); for(; vit != mGroupUpdates.end(); vit++) grpMetas.insert(std::make_pair(vit->newGrp->grpId, (RsGxsGrpMetaData*)NULL)); - mDataStore->retrieveGxsGrpMetaData(grpMetas); + if(!grpMetas.empty()) + mDataStore->retrieveGxsGrpMetaData(grpMetas); + else + return; vit = mGroupUpdates.begin(); for(; vit != mGroupUpdates.end(); vit++) From 9470e5562a2866f709d5f40cb091e021e58674c5 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sat, 5 Oct 2013 11:36:31 +0000 Subject: [PATCH 05/29] Added update capability to forums ui Added timestamp check for group update in nxs decided to add control variable to allow meta changes in updates rather than none git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6803 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgenexchange.cc | 29 ++++++++- libretroshare/src/gxs/rsgenexchange.h | 27 ++++++-- libretroshare/src/gxs/rsgxsnetservice.cc | 9 ++- libretroshare/src/gxs/rsgxsutil.h | 10 +++ libretroshare/src/retroshare/rsgxsforums.h | 8 +++ .../src/retroshare/rsgxsifacetypes.h | 64 ++++++++++++++++++- libretroshare/src/services/p3gxsforums.cc | 14 ++++ libretroshare/src/services/p3gxsforums.h | 7 ++ retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp | 53 ++++++++++++--- retroshare-gui/src/gui/gxs/GxsGroupDialog.h | 1 + .../src/gui/gxschannels/GxsChannelDialog.cpp | 1 + .../src/gui/gxsforums/GxsForumGroupDialog.cpp | 7 ++ 12 files changed, 208 insertions(+), 22 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 5b4baff0d..db29576ca 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1370,11 +1370,11 @@ void RsGenExchange::publishGroup(uint32_t& token, RsGxsGrpItem *grpItem) } -void RsGenExchange::updateGroup(uint32_t& token, RsGxsGrpItem* grpItem) +void RsGenExchange::updateGroup(uint32_t& token, RsGxsGroupUpdateMeta& updateMeta, RsGxsGrpItem* grpItem) { RsStackMutex stack(mGenMtx); token = mDataAccess->generatePublicToken(); - mGroupUpdatePublish.push_back(GroupUpdatePublish(grpItem, token)); + mGroupUpdatePublish.push_back(GroupUpdatePublish(grpItem, updateMeta, token)); #ifdef GEN_EXCH_DEBUG std::cerr << "RsGenExchange::updateGroup() token: " << token; @@ -1828,7 +1828,9 @@ void RsGenExchange::processGroupUpdatePublish() const RsGxsGroupId& groupId = gup.grpItem->meta.mGroupId; RsGxsGrpMetaData* meta = grpMeta[groupId]; - GxsGrpPendingSign ggps(ggps.mItem, ggps.mToken); + gup.grpItem->meta = *meta; + assignMetaUpdates(gup.grpItem->meta, gup.mUpdateMeta); + GxsGrpPendingSign ggps(gup.grpItem, ggps.mToken); bool split = splitKeys(meta->keys, ggps.mPrivateKeys, ggps.mPublicKeys); @@ -1849,6 +1851,27 @@ void RsGenExchange::processGroupUpdatePublish() mGroupUpdatePublish.clear(); } +void RsGenExchange::assignMetaUpdates(RsGroupMetaData& meta, const RsGxsGroupUpdateMeta metaUpdate) const +{ + const RsGxsGroupUpdateMeta::GxsMetaUpdate* updates; + RsGxsGroupUpdateMeta::GxsMetaUpdate::const_iterator mit = updates->begin(); + for(; mit != updates->end(); mit++) + { + const UpdateItem* item = mit->second; + RsGxsGroupUpdateMeta::UpdateType utype = mit->first; + + if(utype == RsGxsGroupUpdateMeta::NAME) + { + const StringUpdateItem* sitem = NULL; + + if((sitem = dynamic_cast(item)) != NULL) + { + meta.mGroupName = sitem->getUpdate(); + } + } + } +} + bool RsGenExchange::splitKeys(const RsTlvSecurityKeySet& keySet, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet) { diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index b7dc6e966..054c19ce6 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -527,12 +527,12 @@ protected: /*! * Updates an existing group item \n - * This will induce a related change message \n - * Ownership of item passes to this rsgenexchange \n - * @param token - * @param grpItem - */ - void updateGroup(uint32_t& token, RsGxsGrpItem* grpItem); + * This will induce a related change message \n + * Ownership of item passes to this rsgenexchange \n + * @param token + * @param grpItem + */ + void updateGroup(uint32_t& token, RsGxsGroupUpdateMeta& updateMeta, RsGxsGrpItem* grpItem); public: /*! @@ -762,7 +762,20 @@ private: */ bool updateValid(RsGxsGrpMetaData& oldGrp, RsNxsGrp& newGrp) const; - bool splitKeys(const RsTlvSecurityKeySet& keySet, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet); + /*! + * convenience function for splitting key sets into private and public + * @param keySet The keys set to split into a private and public set + * @param privateKeySet contains the publish and admin private keys + * @param publicKeySet contains the publish and admin public keys + * @return false, if 2 private and public keys are not found in keySet + */ + bool splitKeys(const RsTlvSecurityKeySet& keySet, RsTlvSecurityKeySet& privateKeySet, + RsTlvSecurityKeySet& publicKeySet); + + /*! + * Convenience function for assigning the meta update items to the actual group meta + */ + void assignMetaUpdates(RsGroupMetaData& meta, const RsGxsGroupUpdateMeta metaUpdate) const; private: diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 6465d4332..c0e4c712d 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -1366,6 +1366,7 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) } std::map grpMetaMap; + std::map::const_iterator metaIter; mDataStore->retrieveGxsGrpMetaData(grpMetaMap); // now do compare and add loop @@ -1380,8 +1381,14 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) { RsNxsSyncGrpItem*& grpSyncItem = *llit; const std::string& grpId = grpSyncItem->grpId; + metaIter = grpMetaMap.find(grpId); + bool haveItem = metaIter != grpMetaMap.end(); + bool latestItem = false; - if(grpMetaMap.find(grpId) == grpMetaMap.end()){ + if(!haveItem) + latestItem = grpSyncItem->publishTs > metaIter->second->mPublishTs; + + if(haveItem && latestItem){ // determine if you need to check reputation bool checkRep = !grpSyncItem->authorId.empty(); diff --git a/libretroshare/src/gxs/rsgxsutil.h b/libretroshare/src/gxs/rsgxsutil.h index f6b367f4a..9e11b5e41 100644 --- a/libretroshare/src/gxs/rsgxsutil.h +++ b/libretroshare/src/gxs/rsgxsutil.h @@ -139,4 +139,14 @@ public: bool validUpdate; }; +class GroupUpdatePublish +{ +public: + GroupUpdatePublish(RsGxsGrpItem* item, RsGxsGroupUpdateMeta updateMeta, uint32_t token) + : grpItem(item), mToken(token), mUpdateMeta(updateMeta) {} + RsGxsGrpItem* grpItem; + RsGxsGroupUpdateMeta mUpdateMeta; + uint32_t mToken; +}; + #endif /* GXSUTIL_H_ */ diff --git a/libretroshare/src/retroshare/rsgxsforums.h b/libretroshare/src/retroshare/rsgxsforums.h index de97618c0..956ac2ab5 100644 --- a/libretroshare/src/retroshare/rsgxsforums.h +++ b/libretroshare/src/retroshare/rsgxsforums.h @@ -84,6 +84,14 @@ virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgI virtual bool createGroup(uint32_t &token, RsGxsForumGroup &group) = 0; virtual bool createMsg(uint32_t &token, RsGxsForumMsg &msg) = 0; +/*! + * To update forum group with new information + * @param token the token used to check completion status of update + * @param group group to be updated, groupId element must be set or will be rejected + * @return false groupId not set, true if set and accepted (still check token for completion) + */ +virtual bool updateGroup(uint32_t &token, RsGxsGroupUpdateMeta&, RsGxsForumGroup &group); + }; diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index d4a4f4df3..a6dacd329 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -128,7 +128,6 @@ public: class GxsGroupStatistic { public: - /// number of message RsGxsGroupId mGrpId; uint32_t mNumMsgs; @@ -147,4 +146,67 @@ public: uint32_t mSizeStore; }; +class UpdateItem +{ +public: + virtual ~UpdateItem() { } +}; + +class StringUpdateItem : public UpdateItem +{ +public: + StringUpdateItem(const std::string update) : mUpdate(update) {} + const std::string& getUpdate() const { return mUpdate; } + +private: + std::string mUpdate; +}; + +class RsGxsGroupUpdateMeta +{ +public: + + // expand as support is added for other utypes + enum UpdateType { DESCRIPTION, NAME }; + + RsGxsGroupUpdateMeta(const std::string& groupId); + ~RsGxsGroupUpdateMeta() + { + GxsMetaUpdate::iterator mit = mUpdates.begin(); + for(; mit != mUpdates.end(); mit++) + delete mit->second; + } + + typedef std::map GxsMetaUpdate; + + /*! + * Only one item of a utype can exist + * @param utype the type of meta update + * @param item update item containing the change value + */ + void setMetaUpdate(UpdateType utype, UpdateItem* item) + { + GxsMetaUpdate::iterator mit; + if ((mit = mUpdates.find(utype)) != mUpdates.end()) + mUpdates[utype] = item; + else + delete mUpdates[utype]; + } + + /*! + * @param utype update type to remove + * @return false if update did not exist, true if update successfully removed + */ + bool removeUpdateType(UpdateType utype){ return mUpdates.erase(utype) == 1; } + + const GxsMetaUpdate* getUpdate() { return &mUpdates; } + + const std::string& getGroupId() { return mGroupId; } + +private: + + GxsMetaUpdate mUpdates; + std::string mGroupId; +}; + #endif /* RSGXSIFACETYPES_H_ */ diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index 6010ddc3d..1e8fd1a65 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -215,6 +215,20 @@ bool p3GxsForums::createGroup(uint32_t &token, RsGxsForumGroup &group) return true; } +bool p3GxsForums::updateGroup(uint32_t &token, RsGxsGroupUpdateMeta& meta, RsGxsForumGroup &group) +{ + std::cerr << "p3GxsForums::createGroup()" << std::endl; + + if(group.mMeta.mGroupId.empty()) + return false; + + RsGxsForumGroupItem* grpItem = new RsGxsForumGroupItem(); + grpItem->mGroup = group; + grpItem->meta = group.mMeta; + + RsGenExchange::updateGroup(token, meta, grpItem); + return true; +} bool p3GxsForums::createMsg(uint32_t &token, RsGxsForumMsg &msg) { diff --git a/libretroshare/src/services/p3gxsforums.h b/libretroshare/src/services/p3gxsforums.h index 6287a10f8..cc2311539 100644 --- a/libretroshare/src/services/p3gxsforums.h +++ b/libretroshare/src/services/p3gxsforums.h @@ -74,6 +74,13 @@ virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgI virtual bool createGroup(uint32_t &token, RsGxsForumGroup &group); virtual bool createMsg(uint32_t &token, RsGxsForumMsg &msg); +/*! + * To update forum group with new information + * @param token the token used to check completion status of update + * @param group group to be updated, groupId element must be set or will be rejected + * @return false groupId not set, true if set and accepted (still check token for completion) + */ +virtual bool updateGroup(uint32_t &token, RsGxsGroupUpdateMeta& meta, RsGxsForumGroup &group); private: diff --git a/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp index 2ecc06db4..81f314718 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp @@ -148,13 +148,13 @@ void GxsGroupDialog::initMode() { ui.buttonBox->setStandardButtons(QDialogButtonBox::Close); } - break; -//TODO -// case MODE_EDIT: -// { -// ui.createButton->setText(tr("Submit Changes")); -// } -// break; + break; + case MODE_EDIT: + { + ui.buttonBox->setStandardButtons(QDialogButtonBox::Ok | QDialogButtonBox::Cancel); + ui.buttonBox->button(QDialogButtonBox::Ok)->setText(tr("Submit Group Changes")); + } + break; } } @@ -320,14 +320,47 @@ void GxsGroupDialog::submitGroup() break; case MODE_EDIT: - { - /* TEMP: just close if down */ - cancelDialog(); + { + + editGroup(); } break; } } +void GxsGroupDialog::editGroup() +{ + std::cerr << "GxsGroupDialog::editGroup()" << std::endl; + + QString name = misc::removeNewLine(ui.groupName->text()); + uint32_t flags = GXS_SERV::FLAG_PRIVACY_PUBLIC; + + if(name.isEmpty()) + { + /* error message */ + QMessageBox::warning(this, "RetroShare", tr("Please add a Name"), QMessageBox::Ok, QMessageBox::Ok); + return; //Don't add a empty name!! + } + + uint32_t token; + RsGroupMetaData meta; + + // Fill in the MetaData as best we can. + meta.mGroupName = std::string(name.toUtf8()); + + meta.mGroupFlags = flags; + meta.mSignFlags = getGroupSignFlags(); + + if (service_CreateGroup(token, meta)) + { + // get the Queue to handle response. + if(mTokenQueue != NULL) + mTokenQueue->queueRequest(token, TOKENREQ_GROUPINFO, RS_TOKREQ_ANSTYPE_ACK, GXSGROUP_NEWGROUPID); + } + + close(); +} + void GxsGroupDialog::createGroup() { std::cerr << "GxsGroupDialog::createGroup()"; diff --git a/retroshare-gui/src/gui/gxs/GxsGroupDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupDialog.h index f8cf39cec..5f71ee364 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupDialog.h @@ -202,6 +202,7 @@ private: void setupVisibility(); void clearForm(); void createGroup(); + void editGroup(); void sendShareList(std::string forumId); void loadNewGroupId(const uint32_t &token); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index a35e7a522..8ab57794e 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -23,6 +23,7 @@ #include #include +#include #include "GxsChannelDialog.h" #include "gui/feeds/GxsChannelPostItem.h" diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.cpp index 36351363b..47edec932 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.cpp @@ -52,6 +52,13 @@ const uint32_t ForumCreateDefaultsFlags = ( GXS_GROUP_DEFAULTS_DISTRIB_PUBLIC GXS_GROUP_DEFAULTS_COMMENTS_NO | 0); + +const uint32_t ForumEditEnabledFlags = ( GXS_GROUP_FLAGS_ICON | + GXS_GROUP_FLAGS_DESCRIPTION | + 0); + +const uint32_t ForumEditDefaultsFlags = 0; + GxsForumGroupDialog::GxsForumGroupDialog(TokenQueue *tokenQueue, QWidget *parent) :GxsGroupDialog(tokenQueue, ForumCreateEnabledFlags, ForumCreateDefaultsFlags, parent) { From 18944404cc7eb7396a5b2535db48963714962e93 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sat, 5 Oct 2013 12:03:31 +0000 Subject: [PATCH 06/29] make update function pure virtual in interface git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6804 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/retroshare/rsgxsforums.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/retroshare/rsgxsforums.h b/libretroshare/src/retroshare/rsgxsforums.h index 956ac2ab5..1ae6fd9e0 100644 --- a/libretroshare/src/retroshare/rsgxsforums.h +++ b/libretroshare/src/retroshare/rsgxsforums.h @@ -90,7 +90,7 @@ virtual bool createMsg(uint32_t &token, RsGxsForumMsg &msg) = 0; * @param group group to be updated, groupId element must be set or will be rejected * @return false groupId not set, true if set and accepted (still check token for completion) */ -virtual bool updateGroup(uint32_t &token, RsGxsGroupUpdateMeta&, RsGxsForumGroup &group); +virtual bool updateGroup(uint32_t &token, RsGxsGroupUpdateMeta&, RsGxsForumGroup &group) = 0; }; From 25d721b6fe4a6d75100d57a1e7280d551b03bac9 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 6 Oct 2013 21:07:31 +0000 Subject: [PATCH 07/29] Code change to test group edit - can modify group names and content now Group edit code fixes git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6811 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/gxssecurity.cc | 4 +- libretroshare/src/gxs/gxssecurity.h | 4 +- libretroshare/src/gxs/rsgenexchange.cc | 114 +++++++++++++----- libretroshare/src/gxs/rsgenexchange.h | 20 +-- .../src/retroshare/rsgxsifacetypes.h | 23 +--- libretroshare/src/services/p3gxsforums.cc | 5 +- retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp | 11 +- retroshare-gui/src/gui/gxs/GxsGroupDialog.h | 14 ++- .../src/gui/gxsforums/GxsForumGroupDialog.cpp | 11 +- .../src/gui/gxsforums/GxsForumGroupDialog.h | 1 + 10 files changed, 133 insertions(+), 74 deletions(-) diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index ea820e1b4..f32e9f150 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -38,7 +38,7 @@ GxsSecurity::~GxsSecurity() { } -RSA *GxsSecurity::extractPublicKey(RsTlvSecurityKey& key) +RSA *GxsSecurity::extractPublicKey(const RsTlvSecurityKey& key) { const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data; long keylen = key.keyData.bin_len; @@ -489,7 +489,7 @@ void GxsSecurity::setRSAPrivateKey(RsTlvSecurityKey & key, RSA *rsa_priv) key.keyId = keyId; } -RSA *GxsSecurity::extractPrivateKey(RsTlvSecurityKey & key) +RSA *GxsSecurity::extractPrivateKey(const RsTlvSecurityKey & key) { const unsigned char *keyptr = (const unsigned char *) key.keyData.bin_data; long keylen = key.keyData.bin_len; diff --git a/libretroshare/src/gxs/gxssecurity.h b/libretroshare/src/gxs/gxssecurity.h index 2bccf542f..4ebd122f4 100644 --- a/libretroshare/src/gxs/gxssecurity.h +++ b/libretroshare/src/gxs/gxssecurity.h @@ -51,14 +51,14 @@ public: * @param key RsTlvSecurityKey to extract public RSA key from * @return pointer to the public RSA key if successful, null otherwise */ - static RSA *extractPublicKey(RsTlvSecurityKey &key); + static RSA *extractPublicKey(const RsTlvSecurityKey &key); /*! * extracts the public key from an RsTlvSecurityKey * @param key RsTlvSecurityKey to extract private RSA key from * @return pointer to the private RSA key if successful, null otherwise */ - static RSA *extractPrivateKey(RsTlvSecurityKey &key); + static RSA *extractPrivateKey(const RsTlvSecurityKey &key); /*! * stores the rsa public key in a RsTlvSecurityKey diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index db29576ca..db4a4c214 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -118,6 +118,8 @@ void RsGenExchange::tick() publishMsgs(); + processGroupUpdatePublish(); + processRecvdData(); if(!mNotifications.empty()) @@ -278,6 +280,54 @@ void RsGenExchange::generateGroupKeys(RsTlvSecurityKeySet& privatekeySet, } } +void RsGenExchange::generatePublicFromPrivateKeys(const RsTlvSecurityKeySet &privatekeySet, + RsTlvSecurityKeySet &publickeySet) +{ + + // actually just copy settings of one key except mark its key flags public + + typedef std::map keyMap; + const keyMap& allKeys = privatekeySet.keys; + keyMap::const_iterator cit = allKeys.begin(); + bool adminFound = false, publishFound = false; + for(; cit != allKeys.end(); cit++) + { + const RsTlvSecurityKey& privKey = cit->second; + if(privKey.keyFlags & RSTLV_KEY_TYPE_FULL) + { + RsTlvSecurityKey pubKey; + + pubKey = privKey; + + RSA *rsaPrivKey = NULL, *rsaPubKey = NULL; + + rsaPrivKey = GxsSecurity::extractPrivateKey(privKey); + + if(rsaPrivKey) + rsaPubKey = RSAPublicKey_dup(rsaPrivKey); + + if(rsaPrivKey && rsaPubKey) + { + GxsSecurity::setRSAPublicKey(pubKey, rsaPubKey); + + if(pubKey.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) + pubKey.keyFlags = RSTLV_KEY_DISTRIB_ADMIN | RSTLV_KEY_TYPE_PUBLIC_ONLY; + + if(pubKey.keyFlags & RSTLV_KEY_DISTRIB_PRIVATE) + pubKey.keyFlags = RSTLV_KEY_DISTRIB_PRIVATE | RSTLV_KEY_TYPE_PUBLIC_ONLY; + + publickeySet.keys.insert(std::make_pair(pubKey.keyId, pubKey)); + } + + if(rsaPrivKey) + RSA_free(rsaPrivKey); + + if(rsaPubKey) + RSA_free(rsaPubKey); + } + } +} + uint8_t RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet) { std::cerr << "RsGenExchange::createGroup()"; @@ -1818,6 +1868,9 @@ void RsGenExchange::processGroupUpdatePublish() grpMeta.insert(std::make_pair(groupId, (RsGxsGrpMetaData*)(NULL))); } + if(grpMeta.empty()) + return; + mDataStore->retrieveGxsGrpMetaData(grpMeta); // now @@ -1832,20 +1885,23 @@ void RsGenExchange::processGroupUpdatePublish() assignMetaUpdates(gup.grpItem->meta, gup.mUpdateMeta); GxsGrpPendingSign ggps(gup.grpItem, ggps.mToken); - bool split = splitKeys(meta->keys, ggps.mPrivateKeys, ggps.mPublicKeys); + bool publishAndAdminPrivatePresent = checkKeys(meta->keys); - if(split) + if(publishAndAdminPrivatePresent) { - ggps.mHaveKeys = true; - ggps.mStartTS = time(NULL); - ggps.mLastAttemptTS = 0; - mGrpsToPublish.push_back(ggps); + ggps.mPrivateKeys = meta->keys; + generatePublicFromPrivateKeys(ggps.mPrivateKeys, ggps.mPublicKeys); + ggps.mHaveKeys = true; + ggps.mStartTS = time(NULL); + ggps.mLastAttemptTS = 0; + ggps.mIsUpdate = true; + mGrpsToPublish.push_back(ggps); }else { - delete gup.grpItem; - mDataAccess->updatePublicRequestStatus(gup.mToken, RsTokenService::GXS_REQUEST_V2_STATUS_FAILED); + delete gup.grpItem; + mDataAccess->updatePublicRequestStatus(gup.mToken, RsTokenService::GXS_REQUEST_V2_STATUS_FAILED); } - + delete meta; } mGroupUpdatePublish.clear(); @@ -1853,43 +1909,39 @@ void RsGenExchange::processGroupUpdatePublish() void RsGenExchange::assignMetaUpdates(RsGroupMetaData& meta, const RsGxsGroupUpdateMeta metaUpdate) const { - const RsGxsGroupUpdateMeta::GxsMetaUpdate* updates; + const RsGxsGroupUpdateMeta::GxsMetaUpdate* updates = metaUpdate.getUpdates(); RsGxsGroupUpdateMeta::GxsMetaUpdate::const_iterator mit = updates->begin(); for(; mit != updates->end(); mit++) { - const UpdateItem* item = mit->second; - RsGxsGroupUpdateMeta::UpdateType utype = mit->first; - if(utype == RsGxsGroupUpdateMeta::NAME) - { - const StringUpdateItem* sitem = NULL; - - if((sitem = dynamic_cast(item)) != NULL) - { - meta.mGroupName = sitem->getUpdate(); - } - } + if(mit->first == RsGxsGroupUpdateMeta::NAME) + meta.mGroupName = mit->second; } } -bool RsGenExchange::splitKeys(const RsTlvSecurityKeySet& keySet, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet) +bool RsGenExchange::checkKeys(const RsTlvSecurityKeySet& keySet) { typedef std::map keyMap; const keyMap& allKeys = keySet.keys; keyMap::const_iterator cit = allKeys.begin(); - + bool adminFound = false, publishFound = false; for(; cit != allKeys.end(); cit++) { - const RsTlvSecurityKey& key = cit->second; - if(key.keyFlags & RSTLV_KEY_TYPE_PUBLIC_ONLY) - publicKeySet.keys.insert(std::make_pair(key.keyId, key)); - else if(key.keyFlags & RSTLV_KEY_TYPE_FULL) - privateKeySet.keys.insert(std::make_pair(key.keyId, key)); + const RsTlvSecurityKey& key = cit->second; + if(key.keyFlags & RSTLV_KEY_TYPE_FULL) + { + if(key.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) + adminFound = true; + + if(key.keyFlags & RSTLV_KEY_DISTRIB_PRIVATE) + publishFound = true; + + } } // user must have both private and public parts of publish and admin keys - return (privateKeySet.keys.size() == 2) && (publicKeySet.keys.size() == 2); + return adminFound && publishFound; } void RsGenExchange::publishGrps() @@ -1962,10 +2014,6 @@ void RsGenExchange::publishGrps() // get group id from private admin key id grpItem->meta.mGroupId = grp->grpId = privAdminKey.keyId; - // what!? this will remove the private keys! - privatekeySet.keys.insert(publicKeySet.keys.begin(), - publicKeySet.keys.end()); - ServiceCreate_Return ret = service_CreateGroup(grpItem, privatekeySet); diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index 054c19ce6..b495d7922 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -712,11 +712,20 @@ private: /*! * Generate a set of keys that can define a GXS group * @param privatekeySet contains private generated keys - * @param privatekeySet contains public generated keys (counterpart of private) + * @param publickeySet contains public generated keys (counterpart of private) * @param genPublicKeys should publish key pair also be generated */ void generateGroupKeys(RsTlvSecurityKeySet& privatekeySet, RsTlvSecurityKeySet& publickeySet, bool genPublishKeys); + /*! + * Generate public set of keys from their private counterparts + * No keys will be generated if one fails + * @param privatekeySet contains private generated keys + * @param publickeySet contains public generated keys (counterpart of private) + * @return false if key gen failed for a key set + */ + void generatePublicFromPrivateKeys(const RsTlvSecurityKeySet& privatekeySet, RsTlvSecurityKeySet& publickeySet); + /*! * Attempts to validate msg signatures * @param msg message to be validated @@ -763,14 +772,11 @@ private: bool updateValid(RsGxsGrpMetaData& oldGrp, RsNxsGrp& newGrp) const; /*! - * convenience function for splitting key sets into private and public + * convenience function for checking private publish and admin keys are present * @param keySet The keys set to split into a private and public set - * @param privateKeySet contains the publish and admin private keys - * @param publicKeySet contains the publish and admin public keys - * @return false, if 2 private and public keys are not found in keySet + * @return false, if private admin and publish keys cannot be found, true otherwise */ - bool splitKeys(const RsTlvSecurityKeySet& keySet, RsTlvSecurityKeySet& privateKeySet, - RsTlvSecurityKeySet& publicKeySet); + bool checkKeys(const RsTlvSecurityKeySet& keySet); /*! * Convenience function for assigning the meta update items to the actual group meta diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index a6dacd329..f2181e363 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -169,28 +169,18 @@ public: // expand as support is added for other utypes enum UpdateType { DESCRIPTION, NAME }; - RsGxsGroupUpdateMeta(const std::string& groupId); - ~RsGxsGroupUpdateMeta() - { - GxsMetaUpdate::iterator mit = mUpdates.begin(); - for(; mit != mUpdates.end(); mit++) - delete mit->second; - } + RsGxsGroupUpdateMeta(const std::string& groupId) : mGroupId(groupId) {} - typedef std::map GxsMetaUpdate; + typedef std::map GxsMetaUpdate; /*! * Only one item of a utype can exist * @param utype the type of meta update * @param item update item containing the change value */ - void setMetaUpdate(UpdateType utype, UpdateItem* item) + void setMetaUpdate(UpdateType utype, const std::string& update) { - GxsMetaUpdate::iterator mit; - if ((mit = mUpdates.find(utype)) != mUpdates.end()) - mUpdates[utype] = item; - else - delete mUpdates[utype]; + mUpdates[utype] = update; } /*! @@ -199,9 +189,8 @@ public: */ bool removeUpdateType(UpdateType utype){ return mUpdates.erase(utype) == 1; } - const GxsMetaUpdate* getUpdate() { return &mUpdates; } - - const std::string& getGroupId() { return mGroupId; } + const GxsMetaUpdate* getUpdates() const { return &mUpdates; } + const std::string& getGroupId() const { return mGroupId; } private: diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index 1e8fd1a65..7b7766bd2 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -219,12 +219,13 @@ bool p3GxsForums::updateGroup(uint32_t &token, RsGxsGroupUpdateMeta& meta, RsGxs { std::cerr << "p3GxsForums::createGroup()" << std::endl; - if(group.mMeta.mGroupId.empty()) + if(meta.getGroupId().empty()) return false; RsGxsForumGroupItem* grpItem = new RsGxsForumGroupItem(); grpItem->mGroup = group; - grpItem->meta = group.mMeta; + grpItem->meta = group.mMeta; + grpItem->meta.mGroupId = meta.getGroupId(); RsGenExchange::updateGroup(token, meta, grpItem); return true; diff --git a/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp index 81f314718..d3f93cc24 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupDialog.cpp @@ -343,15 +343,10 @@ void GxsGroupDialog::editGroup() } uint32_t token; - RsGroupMetaData meta; + RsGxsGroupUpdateMeta updateMeta(mGrpMeta.mGroupId); + updateMeta.setMetaUpdate(RsGxsGroupUpdateMeta::NAME, std::string(name.toUtf8())); - // Fill in the MetaData as best we can. - meta.mGroupName = std::string(name.toUtf8()); - - meta.mGroupFlags = flags; - meta.mSignFlags = getGroupSignFlags(); - - if (service_CreateGroup(token, meta)) + if (service_EditGroup(token, updateMeta)) { // get the Queue to handle response. if(mTokenQueue != NULL) diff --git a/retroshare-gui/src/gui/gxs/GxsGroupDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupDialog.h index 5f71ee364..35db177d0 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupDialog.h @@ -163,13 +163,21 @@ protected: void setUiText(UiType uiType, const QString &text); /*! - * Main purpose is to help tansfer meta data to service - * + * It is up to the service to do the actual group creation + * Service can also modify initial meta going into group * @param token This should be set to the token retrieved * @param meta The deriving GXS service should set their grp meta to this value */ virtual bool service_CreateGroup(uint32_t &token, const RsGroupMetaData &meta) = 0; + /*! + * It is up to the service to do the actual group editing + * TODO: make pure virtual + * @param token This should be set to the token retrieved + * @param meta The deriving GXS service should set their grp meta to this value + */ + virtual bool service_EditGroup(uint32_t &token, RsGxsGroupUpdateMeta &updateMeta) {} + /*! * This returns a group logo from the ui \n * Should be calleld by deriving service @@ -216,6 +224,8 @@ private: uint32_t mReadonlyFlags; uint32_t mDefaultsFlags; + protected: + /** Qt Designer generated object */ Ui::GxsGroupDialog ui; }; diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.cpp index 47edec932..77d0611ac 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.cpp @@ -95,8 +95,17 @@ bool GxsForumGroupDialog::service_CreateGroup(uint32_t &token, const RsGroupMeta // Specific Function. RsGxsForumGroup grp; grp.mMeta = meta; - //grp.mDescription = std::string(desc.toUtf8()); + grp.mDescription = std::string(ui.groupDesc->toPlainText().toUtf8()); rsGxsForums->createGroup(token, grp); return true; } + +bool GxsForumGroupDialog::service_EditGroup(uint32_t &token, RsGxsGroupUpdateMeta &updateMeta) +{ + RsGxsForumGroup grp; + grp.mDescription = std::string(ui.groupDesc->toPlainText().toUtf8()); + + rsGxsForums->updateGroup(token, updateMeta, grp); + return true; +} diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.h b/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.h index 96d431319..777139efb 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.h +++ b/retroshare-gui/src/gui/gxsforums/GxsForumGroupDialog.h @@ -37,6 +37,7 @@ protected: virtual void initUi(); virtual QPixmap serviceImage(); virtual bool service_CreateGroup(uint32_t &token, const RsGroupMetaData &meta); + virtual bool service_EditGroup(uint32_t &token, RsGxsGroupUpdateMeta &updateMeta); }; #endif From fc1b7fcb5ceb085814aa1ac6d1de8e3b10685736 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 13 Oct 2013 21:49:17 +0000 Subject: [PATCH 08/29] more debugging of group update - did not correctly perform sync - admin signature validation was being skipped! - still niggly issue with admin signature validation git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6829 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsdataservice.cc | 5 ++--- libretroshare/src/gxs/rsgenexchange.cc | 25 ++++++++++++++++++++---- libretroshare/src/gxs/rsgxsnetservice.cc | 7 +++---- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index a334d090f..9b7769ed8 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -663,9 +663,8 @@ int RsDataService::updateGroup(std::map &grp) if(!validSize(grpPtr)) continue; std::string grpFile = mServiceDir + "/" + grpPtr->grpId; - std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::app | std::ios::trunc); - ostrm.seekg(0, std::ios::end); // go to end to append - uint32_t offset = ostrm.tellg(); // get fill offset + std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::trunc); + uint32_t offset = 0; // get file offset /*! * STORE file offset, file length, file name, diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index db4a4c214..da9a6d080 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -316,6 +316,8 @@ void RsGenExchange::generatePublicFromPrivateKeys(const RsTlvSecurityKeySet &pri if(pubKey.keyFlags & RSTLV_KEY_DISTRIB_PRIVATE) pubKey.keyFlags = RSTLV_KEY_DISTRIB_PRIVATE | RSTLV_KEY_TYPE_PUBLIC_ONLY; + pubKey.endTS = pubKey.startTS + 60 * 60 * 24 * 365 * 5; /* approx 5 years */ + publickeySet.keys.insert(std::make_pair(pubKey.keyId, pubKey)); } @@ -2384,14 +2386,14 @@ void RsGenExchange::processRecvdGroups() meta->mGroupStatus = GXS_SERV::GXS_GRP_STATUS_UNPROCESSED | GXS_SERV::GXS_GRP_STATUS_UNREAD; meta->mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED; - if(meta->mCircleType == GXS_CIRCLE_TYPE_YOUREYESONLY) - meta->mOriginator = grp->PeerId(); - computeHash(grp->grp, meta->mHash); // now check if group already existss if(std::find(existingGrpIds.begin(), existingGrpIds.end(), grp->grpId) == existingGrpIds.end()) { + if(meta->mCircleType == GXS_CIRCLE_TYPE_YOUREYESONLY) + meta->mOriginator = grp->PeerId(); + grps.insert(std::make_pair(grp, meta)); grpIds.push_back(grp->grpId); } @@ -2455,6 +2457,9 @@ void RsGenExchange::processRecvdGroups() void RsGenExchange::performUpdateValidation() { + + RsStackMutex stack(mGenMtx); + #ifdef GXS_GENX_DEBUG std::cerr << "RsGenExchange::performUpdateValidation() " << std::endl; #endif @@ -2492,7 +2497,19 @@ void RsGenExchange::performUpdateValidation() for(; vit != mGroupUpdates.end(); vit++) { GroupUpdate& gu = *vit; - grps.insert(std::make_pair(gu.newGrp, gu.newGrp->metaData)); + + if(gu.validUpdate) + { + if(gu.newGrp->metaData->mCircleType == GXS_CIRCLE_TYPE_YOUREYESONLY) + gu.newGrp->metaData->mOriginator = gu.newGrp->PeerId(); + + grps.insert(std::make_pair(gu.newGrp, gu.newGrp->metaData)); + } + else + { + delete gu.newGrp; + } + delete gu.oldGrpMeta; } diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index c0e4c712d..c1690e96a 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -1383,12 +1383,11 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) const std::string& grpId = grpSyncItem->grpId; metaIter = grpMetaMap.find(grpId); bool haveItem = metaIter != grpMetaMap.end(); - bool latestItem = false; + bool latestVersion = false; - if(!haveItem) - latestItem = grpSyncItem->publishTs > metaIter->second->mPublishTs; + latestVersion = grpSyncItem->publishTs > metaIter->second->mPublishTs; - if(haveItem && latestItem){ + if(!haveItem || (haveItem && latestVersion) ){ // determine if you need to check reputation bool checkRep = !grpSyncItem->authorId.empty(); From 19d7faa572b8fb03a55293f86cde040227c27196 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Mon, 14 Oct 2013 22:03:11 +0000 Subject: [PATCH 09/29] finally found group signature bug, was aliasing signature in validateGrp call (perfect example of why one should use const)! git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6831 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgenexchange.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index da9a6d080..54899fe0e 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -2532,7 +2532,7 @@ bool RsGenExchange::updateValid(RsGxsGrpMetaData& oldGrpMeta, RsNxsGrp& newGrp) return false; } - RsTlvKeySignature& adminSign = mit->second; + RsTlvKeySignature adminSign = mit->second; std::map& keys = oldGrpMeta.keys.keys; std::map::iterator keyMit = keys.find(oldGrpMeta.mGroupId); From fc588614478762128b469d85f942d167c5bcbf43 Mon Sep 17 00:00:00 2001 From: drbob Date: Sun, 20 Oct 2013 09:43:30 +0000 Subject: [PATCH 10/29] Added RecognTags. - items are described in serialiser. - util functions in util/rsrecogn.cc are used to manipulate it. - these are attached to GxsIds, with new interface fns. - Associated Signing Code is in a separate program. Other Tweaks. - Added RsMemCache::erase() - Added RsTlvStringSetRef - Fix for rsturtleitem (already added to trunk). - Formatting and debugging. Status: There is a bug in RsGenExchange::updateGroup which prevents full testing, The basic generation, parsing and validation functions have been tested and are ok. The processing as part of p3IdService still needs to be fully debugged. git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6854 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/libretroshare.pro | 6 +- .../src/retroshare/rsgxsifacetypes.h | 2 +- libretroshare/src/retroshare/rsidentity.h | 45 + libretroshare/src/serialiser/rsgxsiditems.cc | 20 + libretroshare/src/serialiser/rsgxsitems.cc | 3 +- .../src/serialiser/rsgxsrecognitems.cc | 600 ++++++++++++ .../src/serialiser/rsgxsrecognitems.h | 152 +++ libretroshare/src/serialiser/rsserviceids.h | 2 + libretroshare/src/serialiser/rstlvbase.h | 1 + libretroshare/src/serialiser/rstlvtypes.cc | 159 +++ libretroshare/src/serialiser/rstlvtypes.h | 16 + libretroshare/src/services/p3gxsforums.cc | 2 +- libretroshare/src/services/p3idservice.cc | 921 ++++++++++++++++-- libretroshare/src/services/p3idservice.h | 69 +- libretroshare/src/services/p3posted.cc | 1 - libretroshare/src/turtle/rsturtleitem.cc | 4 +- libretroshare/src/util/rsmemcache.h | 42 + libretroshare/src/util/rsrecogn.cc | 640 ++++++++++++ libretroshare/src/util/rsrecogn.h | 61 ++ 19 files changed, 2645 insertions(+), 101 deletions(-) create mode 100644 libretroshare/src/serialiser/rsgxsrecognitems.cc create mode 100644 libretroshare/src/serialiser/rsgxsrecognitems.h create mode 100644 libretroshare/src/util/rsrecogn.cc create mode 100644 libretroshare/src/util/rsrecogn.h diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index de0cfed2a..42fc6e8f8 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -7,7 +7,7 @@ CONFIG += test_voip #GXS Stuff. # This should be disabled for releases until further notice. -#CONFIG += gxs debug +CONFIG += gxs debug # Beware: All data of the stripped services are lost DEFINES *= PQI_DISABLE_TUNNEL @@ -387,6 +387,7 @@ HEADERS += serialiser/rsbaseitems.h \ serialiser/rstlvbanlist.h \ serialiser/rsbanlistitems.h \ serialiser/rsbwctrlitems.h \ + serialiser/rsgxsrecognitems.h \ serialiser/rstunnelitems.h HEADERS += services/p3channels.h \ @@ -430,6 +431,7 @@ HEADERS += util/folderiterator.h \ util/pugiconfig.h \ util/rsmemcache.h \ util/rstickevent.h \ + util/rsrecogn.h \ SOURCES += dbase/cachestrapper.cc \ dbase/fimonitor.cc \ @@ -529,6 +531,7 @@ SOURCES += serialiser/rsbaseitems.cc \ serialiser/rstlvbanlist.cc \ serialiser/rsbanlistitems.cc \ serialiser/rsbwctrlitems.cc \ + serialiser/rsgxsrecognitems.cc \ serialiser/rstunnelitems.cc SOURCES += services/p3channels.cc \ @@ -571,6 +574,7 @@ SOURCES += util/folderiterator.cc \ util/rsaes.cc \ util/rsrandom.cc \ util/rstickevent.cc \ + util/rsrecogn.cc \ upnp_miniupnpc { diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index f2181e363..830df9f52 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -47,7 +47,7 @@ public: mGroupStatus = 0; mCircleType = 0; - //mPublishTs = 0; + mPublishTs = 0; } void operator =(const RsGxsGrpMetaData& rGxsMeta); diff --git a/libretroshare/src/retroshare/rsidentity.h b/libretroshare/src/retroshare/rsidentity.h index 4b78f0a38..a1891e6ac 100644 --- a/libretroshare/src/retroshare/rsidentity.h +++ b/libretroshare/src/retroshare/rsidentity.h @@ -55,6 +55,8 @@ extern RsIdentity *rsIdentity; #define RSID_RELATION_OTHER 0x0008 #define RSID_RELATION_UNKNOWN 0x0010 +#define RSRECOGN_MAX_TAGINFO 5 + std::string rsIdTypeToString(uint32_t idtype); class RsGxsIdGroup @@ -82,6 +84,9 @@ class RsGxsIdGroup std::string mPgpIdHash; std::string mPgpIdSign; // Need a signature as proof - otherwise anyone could add others Hashes. + // Recognition Strings. MAX# defined above. + std::list mRecognTags; + // Not Serialised - for GUI's benefit. bool mPgpKnown; std::string mPgpId; @@ -163,6 +168,36 @@ class RsIdOpinion typedef std::string RsGxsId; // TMP. => +class RsRecognTag +{ + public: + RsRecognTag(uint16_t tc, uint16_t tt, bool v) + :tag_class(tc), tag_type(tt), valid(v) { return; } + uint16_t tag_class; + uint16_t tag_type; + bool valid; +}; + + +class RsRecognTagDetails +{ + public: + RsRecognTagDetails() + :valid_from(0), valid_to(0), tag_class(0), tag_type(0), + is_valid(false), is_pending(false) { return; } + + time_t valid_from; + time_t valid_to; + uint16_t tag_class; + uint16_t tag_type; + + std::string signer; + + bool is_valid; + bool is_pending; +}; + + class RsIdentityDetails { public: @@ -181,6 +216,9 @@ class RsIdentityDetails bool mPgpKnown; std::string mPgpId; + // Recogn details. + std::list mRecognTags; + // reputation details. double mOpinion; double mReputation; @@ -230,6 +268,13 @@ virtual bool getOwnIds(std::list &ownIds) = 0; virtual bool submitOpinion(uint32_t& token, RsIdOpinion &opinion) = 0; virtual bool createIdentity(uint32_t& token, RsIdentityParameters ¶ms) = 0; +virtual bool updateIdentity(uint32_t& token, RsGxsIdGroup &group) = 0; + +virtual bool parseRecognTag(const RsGxsId &id, const std::string &nickname, + const std::string &tag, RsRecognTagDetails &details) = 0; +virtual bool getRecognTagRequest(const RsGxsId &id, const std::string &comment, + uint16_t tag_class, uint16_t tag_type, std::string &tag) = 0; + // Specific RsIdentity Functions.... /* Specific Service Data */ /* We expose these initially for testing / GUI purposes. diff --git a/libretroshare/src/serialiser/rsgxsiditems.cc b/libretroshare/src/serialiser/rsgxsiditems.cc index e22055b7a..7853afeaf 100644 --- a/libretroshare/src/serialiser/rsgxsiditems.cc +++ b/libretroshare/src/serialiser/rsgxsiditems.cc @@ -122,20 +122,30 @@ void RsGxsIdGroupItem::clear() group.mPgpIdHash.clear(); group.mPgpIdSign.clear(); + group.mRecognTags.clear(); + group.mPgpKnown = false; group.mPgpId.clear(); } + std::ostream& RsGxsIdGroupItem::print(std::ostream& out, uint16_t indent) { printRsItemBase(out, "RsGxsIdGroupItem", indent); uint16_t int_Indent = indent + 2; + printIndent(out, int_Indent); + out << "MetaData: " << meta << std::endl; printIndent(out, int_Indent); out << "PgpIdHash: " << group.mPgpIdHash << std::endl; printIndent(out, int_Indent); out << "PgpIdSign: " << group.mPgpIdSign << std::endl; + printIndent(out, int_Indent); + out << "RecognTags:" << std::endl; + + RsTlvStringSetRef set(TLV_TYPE_RECOGNSET, group.mRecognTags); + set.print(out, int_Indent + 2); printRsItemEnd(out ,"RsGxsIdGroupItem", indent); return out; @@ -151,6 +161,9 @@ uint32_t RsGxsIdSerialiser::sizeGxsIdGroupItem(RsGxsIdGroupItem *item) s += GetTlvStringSize(group.mPgpIdHash); s += GetTlvStringSize(group.mPgpIdSign); + RsTlvStringSetRef set(TLV_TYPE_RECOGNSET, item->group.mRecognTags); + s += set.TlvSize(); + return s; } @@ -181,6 +194,9 @@ bool RsGxsIdSerialiser::serialiseGxsIdGroupItem(RsGxsIdGroupItem *item, void *da /* GxsIdGroupItem */ ok &= SetTlvString(data, tlvsize, &offset, 1, item->group.mPgpIdHash); ok &= SetTlvString(data, tlvsize, &offset, 1, item->group.mPgpIdSign); + + RsTlvStringSetRef set(TLV_TYPE_RECOGNSET, item->group.mRecognTags); + ok &= set.SetTlv(data, tlvsize, &offset); if(offset != tlvsize) { @@ -238,6 +254,10 @@ RsGxsIdGroupItem* RsGxsIdSerialiser::deserialiseGxsIdGroupItem(void *data, uint3 ok &= GetTlvString(data, rssize, &offset, 1, item->group.mPgpIdHash); ok &= GetTlvString(data, rssize, &offset, 1, item->group.mPgpIdSign); + + RsTlvStringSetRef set(TLV_TYPE_RECOGNSET, item->group.mRecognTags); + ok &= set.GetTlv(data, rssize, &offset); + if (offset != rssize) { diff --git a/libretroshare/src/serialiser/rsgxsitems.cc b/libretroshare/src/serialiser/rsgxsitems.cc index 6cc57e90c..e1acdf0fd 100644 --- a/libretroshare/src/serialiser/rsgxsitems.cc +++ b/libretroshare/src/serialiser/rsgxsitems.cc @@ -52,7 +52,8 @@ std::ostream &operator<<(std::ostream &out, const RsGroupMetaData &meta) { - out << "[ GroupId: " << meta.mGroupId << " Name: " << meta.mGroupName << " ]"; + out << "[ GroupId: " << meta.mGroupId << " Name: " << meta.mGroupName; + out << " PublishTs: " << meta.mPublishTs << " ]"; return out; } diff --git a/libretroshare/src/serialiser/rsgxsrecognitems.cc b/libretroshare/src/serialiser/rsgxsrecognitems.cc new file mode 100644 index 000000000..89b4576aa --- /dev/null +++ b/libretroshare/src/serialiser/rsgxsrecognitems.cc @@ -0,0 +1,600 @@ +/* + * libretroshare/src/serialiser: rsgxsrecogitems.cc + * + * RetroShare Serialiser. + * + * Copyright 2013-2013 by Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include "serialiser/rsbaseserial.h" +#include "serialiser/rsgxsrecognitems.h" + +/*** +#define RSSERIAL_DEBUG 1 +***/ + +#include + +/*************************************************************************/ + +RsGxsRecognReqItem::~RsGxsRecognReqItem() +{ + return; +} + +void RsGxsRecognReqItem::clear() +{ + issued_at = 0; + period = 0; + tag_class = 0; + tag_type = 0; + + identity.clear(); + nickname.clear(); + comment.clear(); + + sign.TlvClear(); + +} + +std::ostream &RsGxsRecognReqItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsGxsRecognReqItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out, int_Indent); + out << "issued_at: " << issued_at << std::endl; + + printIndent(out, int_Indent); + out << "period: " << period << std::endl; + + printIndent(out, int_Indent); + out << "tag_class: " << tag_class << std::endl; + + printIndent(out, int_Indent); + out << "tag_type: " << tag_type << std::endl; + + printIndent(out, int_Indent); + out << "identity: " << identity << std::endl; + + printIndent(out, int_Indent); + out << "nickname: " << nickname << std::endl; + + printIndent(out, int_Indent); + out << "comment: " << comment << std::endl; + + printIndent(out, int_Indent); + out << "signature: " << std::endl; + sign.print(out, int_Indent + 2); + + printRsItemEnd(out, "RsGxsRecognReqItem", indent); + return out; +} + + +uint32_t RsGxsRecognSerialiser::sizeReq(RsGxsRecognReqItem *item) +{ + uint32_t s = 8; /* header */ + s += 4; // issued_at; + s += 4; // period; + s += 2; // tag_class; + s += 2; // tag_type; + s += GetTlvStringSize(item->identity); + s += GetTlvStringSize(item->nickname); + s += GetTlvStringSize(item->comment); + s += item->sign.TlvSize(); + + return s; +} + +/* serialise the data to the buffer */ +bool RsGxsRecognSerialiser::serialiseReq(RsGxsRecognReqItem *item, void *data, uint32_t *pktsize) +{ + uint32_t tlvsize = sizeReq(item); + uint32_t offset = 0; + + if (*pktsize < tlvsize) + return false; /* not enough space */ + + *pktsize = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize); + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsRecognSerialiser::serialiseReq() Header: " << ok << std::endl; + std::cerr << "RsGxsRecognSerialiser::serialiseReq() Size: " << tlvsize << std::endl; +#endif + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= setRawUInt32(data, tlvsize, &offset, item->issued_at); + ok &= setRawUInt32(data, tlvsize, &offset, item->period); + ok &= setRawUInt16(data, tlvsize, &offset, item->tag_class); + ok &= setRawUInt16(data, tlvsize, &offset, item->tag_type); + + ok &= SetTlvString(data, tlvsize, &offset, 1, item->identity); + ok &= SetTlvString(data, tlvsize, &offset, 1, item->nickname); + ok &= SetTlvString(data, tlvsize, &offset, 1, item->comment); + ok &= item->sign.SetTlv(data, tlvsize, &offset); + + + if (offset != tlvsize) + { + ok = false; +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsRecognSerialiser::serialiseReq() Size Error! " << std::endl; +#endif + } + + return ok; +} + +RsGxsRecognReqItem *RsGxsRecognSerialiser::deserialiseReq(void *data, uint32_t *pktsize) +{ + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t tlvsize = getRsItemSize(data); + + uint32_t offset = 0; + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (RS_SERVICE_TYPE_GXS_RECOGN != getRsItemService(rstype)) || + (RS_PKT_SUBTYPE_RECOGN_REQ != getRsItemSubType(rstype))) + { + return NULL; /* wrong type */ + } + + if (*pktsize < tlvsize) /* check size */ + return NULL; /* not enough data */ + + /* set the packet length */ + *pktsize = tlvsize; + + bool ok = true; + + /* ready to load */ + RsGxsRecognReqItem *item = new RsGxsRecognReqItem(); + item->clear(); + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= getRawUInt32(data, tlvsize, &offset, &(item->issued_at)); + ok &= getRawUInt32(data, tlvsize, &offset, &(item->period)); + ok &= getRawUInt16(data, tlvsize, &offset, &(item->tag_class)); + ok &= getRawUInt16(data, tlvsize, &offset, &(item->tag_type)); + + ok &= GetTlvString(data, tlvsize, &offset, 1, item->identity); + ok &= GetTlvString(data, tlvsize, &offset, 1, item->nickname); + ok &= GetTlvString(data, tlvsize, &offset, 1, item->comment); + ok &= item->sign.GetTlv(data, tlvsize, &offset); + + + if (offset != tlvsize) + { + /* error */ + delete item; + return NULL; + } + + if (!ok) + { + delete item; + return NULL; + } + + return item; +} + +/*************************************************************************/ + +RsGxsRecognTagItem::~RsGxsRecognTagItem() +{ + return; +} + +void RsGxsRecognTagItem::clear() +{ + valid_from = 0; + valid_to = 0; + + tag_class = 0; + tag_type = 0; + + identity.clear(); + nickname.clear(); + + sign.TlvClear(); +} + +std::ostream &RsGxsRecognTagItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsGxsRecognTagItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out, int_Indent); + out << "valid_from: " << valid_from << std::endl; + + printIndent(out, int_Indent); + out << "valid_to: " << valid_to << std::endl; + + printIndent(out, int_Indent); + out << "tag_class: " << tag_class << std::endl; + + printIndent(out, int_Indent); + out << "tag_type: " << tag_type << std::endl; + + printIndent(out, int_Indent); + out << "identity: " << identity << std::endl; + + printIndent(out, int_Indent); + out << "nickname: " << nickname << std::endl; + + printIndent(out, int_Indent); + out << "signature: " << std::endl; + sign.print(out, int_Indent + 2); + + printRsItemEnd(out, "RsGxsRecognTagItem", indent); + return out; +} + + +uint32_t RsGxsRecognSerialiser::sizeTag(RsGxsRecognTagItem *item) +{ + uint32_t s = 8; /* header */ + s += 4; // valid_from; + s += 4; // valid_to; + s += 2; // tag_class; + s += 2; // tag_type; + + s += GetTlvStringSize(item->identity); + s += GetTlvStringSize(item->nickname); + + s += item->sign.TlvSize(); + + return s; +} + +/* serialise the data to the buffer */ +bool RsGxsRecognSerialiser::serialiseTag(RsGxsRecognTagItem *item, void *data, uint32_t *pktsize) +{ + uint32_t tlvsize = sizeTag(item); + uint32_t offset = 0; + + if (*pktsize < tlvsize) + return false; /* not enough space */ + + *pktsize = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize); + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsRecognSerialiser::serialiseTag() Header: " << ok << std::endl; + std::cerr << "RsGxsRecognSerialiser::serialiseTag() Size: " << tlvsize << std::endl; +#endif + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= setRawUInt32(data, tlvsize, &offset, item->valid_from); + ok &= setRawUInt32(data, tlvsize, &offset, item->valid_to); + + ok &= setRawUInt16(data, tlvsize, &offset, item->tag_class); + ok &= setRawUInt16(data, tlvsize, &offset, item->tag_type); + + ok &= SetTlvString(data, tlvsize, &offset, 1, item->identity); + ok &= SetTlvString(data, tlvsize, &offset, 1, item->nickname); + ok &= item->sign.SetTlv(data, tlvsize, &offset); + + + if (offset != tlvsize) + { + ok = false; +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsRecognSerialiser::serialiseTag() Size Error! " << std::endl; +#endif + } + + return ok; +} + +RsGxsRecognTagItem *RsGxsRecognSerialiser::deserialiseTag(void *data, uint32_t *pktsize) +{ + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t tlvsize = getRsItemSize(data); + + uint32_t offset = 0; + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (RS_SERVICE_TYPE_GXS_RECOGN != getRsItemService(rstype)) || + (RS_PKT_SUBTYPE_RECOGN_TAG != getRsItemSubType(rstype))) + { + return NULL; /* wrong type */ + } + + if (*pktsize < tlvsize) /* check size */ + return NULL; /* not enough data */ + + /* set the packet length */ + *pktsize = tlvsize; + + bool ok = true; + + /* ready to load */ + RsGxsRecognTagItem *item = new RsGxsRecognTagItem(); + item->clear(); + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= getRawUInt32(data, tlvsize, &offset, &(item->valid_from)); + ok &= getRawUInt32(data, tlvsize, &offset, &(item->valid_to)); + + ok &= getRawUInt16(data, tlvsize, &offset, &(item->tag_class)); + ok &= getRawUInt16(data, tlvsize, &offset, &(item->tag_type)); + + ok &= GetTlvString(data, tlvsize, &offset, 1, item->identity); + ok &= GetTlvString(data, tlvsize, &offset, 1, item->nickname); + ok &= item->sign.GetTlv(data, tlvsize, &offset); + + + if (offset != tlvsize) + { + /* error */ + delete item; + return NULL; + } + + if (!ok) + { + delete item; + return NULL; + } + + return item; +} + +/*************************************************************************/ + +RsGxsRecognSignerItem::~RsGxsRecognSignerItem() +{ + return; +} + +void RsGxsRecognSignerItem::clear() +{ + signing_classes.TlvClear(); + key.TlvClear(); + sign.TlvClear(); +} + +std::ostream &RsGxsRecognSignerItem::print(std::ostream &out, uint16_t indent) +{ + printRsItemBase(out, "RsGxsRecognSignerItem", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out, int_Indent); + out << "signing_classes: " << std::endl; + signing_classes.print(out, int_Indent + 2); + + printIndent(out, int_Indent); + out << "key: " << std::endl; + key.print(out, int_Indent + 2); + + printIndent(out, int_Indent); + out << "signature: " << std::endl; + sign.print(out, int_Indent + 2); + + + printRsItemEnd(out, "RsGxsRecognSignerItem", indent); + return out; +} + + + + +uint32_t RsGxsRecognSerialiser::sizeSigner(RsGxsRecognSignerItem *item) +{ + uint32_t s = 8; /* header */ + s += item->signing_classes.TlvSize(); + s += item->key.TlvSize(); + s += item->sign.TlvSize(); + + return s; +} + +/* serialise the data to the buffer */ +bool RsGxsRecognSerialiser::serialiseSigner(RsGxsRecognSignerItem *item, void *data, uint32_t *pktsize) +{ + uint32_t tlvsize = sizeSigner(item); + uint32_t offset = 0; + + if (*pktsize < tlvsize) + return false; /* not enough space */ + + *pktsize = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize); + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsRecognSerialiser::serialiseSigner() Header: " << ok << std::endl; + std::cerr << "RsGxsRecognSerialiser::serialiseSigner() Size: " << tlvsize << std::endl; +#endif + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= item->signing_classes.SetTlv(data, tlvsize, &offset); + ok &= item->key.SetTlv(data, tlvsize, &offset); + ok &= item->sign.SetTlv(data, tlvsize, &offset); + + if (offset != tlvsize) + { + ok = false; +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsRecognSerialiser::serialiseSigner() Size Error! " << std::endl; +#endif + } + + return ok; +} + +RsGxsRecognSignerItem *RsGxsRecognSerialiser::deserialiseSigner(void *data, uint32_t *pktsize) +{ + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t tlvsize = getRsItemSize(data); + + uint32_t offset = 0; + + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (RS_SERVICE_TYPE_GXS_RECOGN != getRsItemService(rstype)) || + (RS_PKT_SUBTYPE_RECOGN_SIGNER != getRsItemSubType(rstype))) + { + return NULL; /* wrong type */ + } + + if (*pktsize < tlvsize) /* check size */ + return NULL; /* not enough data */ + + /* set the packet length */ + *pktsize = tlvsize; + + bool ok = true; + + /* ready to load */ + RsGxsRecognSignerItem *item = new RsGxsRecognSignerItem(); + item->clear(); + + /* skip the header */ + offset += 8; + + /* add mandatory parts first */ + ok &= item->signing_classes.GetTlv(data, tlvsize, &offset); + ok &= item->key.GetTlv(data, tlvsize, &offset); + ok &= item->sign.GetTlv(data, tlvsize, &offset); + + if (offset != tlvsize) + { + /* error */ + delete item; + return NULL; + } + + if (!ok) + { + delete item; + return NULL; + } + + return item; +} + + +/*************************************************************************/ + +uint32_t RsGxsRecognSerialiser::size(RsItem *i) +{ + RsGxsRecognReqItem *rqi; + RsGxsRecognTagItem *rti; + RsGxsRecognSignerItem *rsi; + + if (NULL != (rqi = dynamic_cast(i))) + { + return sizeReq(rqi); + } + if (NULL != (rti = dynamic_cast(i))) + { + return sizeTag(rti); + } + if (NULL != (rsi = dynamic_cast(i))) + { + return sizeSigner(rsi); + } + return 0; +} + +bool RsGxsRecognSerialiser::serialise(RsItem *i, void *data, uint32_t *pktsize) +{ + RsGxsRecognReqItem *rri; + RsGxsRecognTagItem *rti; + RsGxsRecognSignerItem *rsi; + + if (NULL != (rri = dynamic_cast(i))) + { + return serialiseReq(rri, data, pktsize); + } + if (NULL != (rti = dynamic_cast(i))) + { + return serialiseTag(rti, data, pktsize); + } + if (NULL != (rsi = dynamic_cast(i))) + { + return serialiseSigner(rsi, data, pktsize); + } + return false; +} + +RsItem *RsGxsRecognSerialiser::deserialise(void *data, uint32_t *pktsize) +{ + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (RS_SERVICE_TYPE_GXS_RECOGN != getRsItemService(rstype))) + { + return NULL; /* wrong type */ + } + + switch(getRsItemSubType(rstype)) + { + case RS_PKT_SUBTYPE_RECOGN_REQ: + return deserialiseReq(data, pktsize); + break; + case RS_PKT_SUBTYPE_RECOGN_TAG: + return deserialiseTag(data, pktsize); + break; + case RS_PKT_SUBTYPE_RECOGN_SIGNER: + return deserialiseSigner(data, pktsize); + break; + default: + return NULL; + break; + } +} + +/*************************************************************************/ + + + diff --git a/libretroshare/src/serialiser/rsgxsrecognitems.h b/libretroshare/src/serialiser/rsgxsrecognitems.h new file mode 100644 index 000000000..fdbc9cfc2 --- /dev/null +++ b/libretroshare/src/serialiser/rsgxsrecognitems.h @@ -0,0 +1,152 @@ +#ifndef RS_GXS_RECOG_ITEMS_H +#define RS_GXS_RECOG_ITEMS_H + +/* + * libretroshare/src/serialiser: rsgxsrecogitems.h + * + * RetroShare Serialiser. + * + * Copyright 2013-2013 by Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include + +#include "serialiser/rsserviceids.h" +#include "serialiser/rsserial.h" +#include "serialiser/rstlvbase.h" +#include "serialiser/rstlvtypes.h" +#include "serialiser/rstlvkeys.h" + +/**************************************************************************/ + +#define RS_PKT_SUBTYPE_RECOGN_REQ 0x01 +#define RS_PKT_SUBTYPE_RECOGN_TAG 0x02 +#define RS_PKT_SUBTYPE_RECOGN_SIGNER 0x03 + + +class RsGxsRecognReqItem: public RsItem +{ + public: + RsGxsRecognReqItem() + :RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_RECOGN, + RS_PKT_SUBTYPE_RECOGN_REQ) + { + setPriorityLevel(QOS_PRIORITY_DEFAULT); + return; + } +virtual ~RsGxsRecognReqItem(); +virtual void clear(); +std::ostream &print(std::ostream &out, uint16_t indent = 0); + + + uint32_t issued_at; + uint32_t period; + uint16_t tag_class; + uint16_t tag_type; + + std::string identity; + std::string nickname; + std::string comment; + + RsTlvKeySignature sign; +}; + + +class RsGxsRecognTagItem: public RsItem +{ + public: + RsGxsRecognTagItem() + :RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_RECOGN, + RS_PKT_SUBTYPE_RECOGN_TAG) + { + setPriorityLevel(QOS_PRIORITY_DEFAULT); + return; + } +virtual ~RsGxsRecognTagItem(); +virtual void clear(); +std::ostream &print(std::ostream &out, uint16_t indent = 0); + + uint32_t valid_from; + uint32_t valid_to; + uint16_t tag_class; + uint16_t tag_type; + + std::string identity; + std::string nickname; + + RsTlvKeySignature sign; +}; + + +class RsGxsRecognSignerItem: public RsItem +{ + public: + RsGxsRecognSignerItem() + :RsItem(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_RECOGN, + RS_PKT_SUBTYPE_RECOGN_SIGNER) + { + setPriorityLevel(QOS_PRIORITY_DEFAULT); + return; + } +virtual ~RsGxsRecognSignerItem(); +virtual void clear(); +std::ostream &print(std::ostream &out, uint16_t indent = 0); + + RsTlvServiceIdSet signing_classes; + RsTlvSecurityKey key; // has from->to, and flags. + RsTlvKeySignature sign; +}; + + +class RsGxsRecognSerialiser: public RsSerialType +{ + public: + RsGxsRecognSerialiser() + :RsSerialType(RS_PKT_VERSION_SERVICE, RS_SERVICE_TYPE_GXS_RECOGN) + { return; } +virtual ~RsGxsRecognSerialiser() + { return; } + + +virtual uint32_t size(RsItem *); +virtual bool serialise (RsItem *item, void *data, uint32_t *size); +virtual RsItem * deserialise(void *data, uint32_t *size); + + private: + +virtual uint32_t sizeReq(RsGxsRecognReqItem *); +virtual bool serialiseReq(RsGxsRecognReqItem *item, void *data, uint32_t *size); +virtual RsGxsRecognReqItem *deserialiseReq(void *data, uint32_t *size); + +virtual uint32_t sizeTag(RsGxsRecognTagItem *); +virtual bool serialiseTag(RsGxsRecognTagItem *item, void *data, uint32_t *size); +virtual RsGxsRecognTagItem *deserialiseTag(void *data, uint32_t *size); + +virtual uint32_t sizeSigner(RsGxsRecognSignerItem *); +virtual bool serialiseSigner(RsGxsRecognSignerItem *item, void *data, uint32_t *size); +virtual RsGxsRecognSignerItem *deserialiseSigner(void *data, uint32_t *size); + +}; + +/**************************************************************************/ + +#endif /* RS_GXS_RECOGN_ITEMS_H */ + + diff --git a/libretroshare/src/serialiser/rsserviceids.h b/libretroshare/src/serialiser/rsserviceids.h index 8095ed575..efa22dc32 100644 --- a/libretroshare/src/serialiser/rsserviceids.h +++ b/libretroshare/src/serialiser/rsserviceids.h @@ -149,6 +149,8 @@ const uint16_t RS_SERVICE_GXSV3_TYPE_POSTED = 0xf326; const uint16_t RS_SERVICE_GXSV3_TYPE_CHANNELS = 0xf327; const uint16_t RS_SERVICE_GXSV3_TYPE_GXSCIRCLE = 0xf328; +const uint16_t RS_SERVICE_TYPE_GXS_RECOGN = 0xf331; + /***************** IDS ALLOCATED FOR PLUGINS ******************/ const uint16_t RS_SERVICE_TYPE_PLUGIN_ARADO_TEST_ID1 = 0xf401; diff --git a/libretroshare/src/serialiser/rstlvbase.h b/libretroshare/src/serialiser/rstlvbase.h index 70698df64..613545440 100644 --- a/libretroshare/src/serialiser/rstlvbase.h +++ b/libretroshare/src/serialiser/rstlvbase.h @@ -201,6 +201,7 @@ const uint16_t TLV_TYPE_WKEYVALUESET = 0x1013; const uint16_t TLV_TYPE_STRINGSET = 0x1020; /* dummy non-existant */ const uint16_t TLV_TYPE_PEERSET = 0x1021; const uint16_t TLV_TYPE_HASHSET = 0x1022; +const uint16_t TLV_TYPE_RECOGNSET = 0x1023; const uint16_t TLV_TYPE_SERVICESET = 0x1030; const uint16_t TLV_TYPE_SECURITYKEY = 0x1040; diff --git a/libretroshare/src/serialiser/rstlvtypes.cc b/libretroshare/src/serialiser/rstlvtypes.cc index 6a2fc0579..1c1db3cd8 100644 --- a/libretroshare/src/serialiser/rstlvtypes.cc +++ b/libretroshare/src/serialiser/rstlvtypes.cc @@ -428,6 +428,165 @@ std::ostream &RsTlvStringSet::printHex(std::ostream &out, uint16_t indent) } +/************************************* String Set Ref ************************************/ +/* This is exactly the same as StringSet, but it uses an alternative list. + */ +RsTlvStringSetRef::RsTlvStringSetRef(uint16_t type, std::list &refids) + :mType(type), ids(refids) +{ +} + +void RsTlvStringSetRef::TlvClear() +{ + ids.clear(); + +} + +uint32_t RsTlvStringSetRef::TlvSize() +{ + + uint32_t s = TLV_HEADER_SIZE; /* header */ + + /* determine the total size of ids strings in list */ + + std::list::iterator it; + + for(it = ids.begin(); it != ids.end() ; ++it) + { + if (it->length() > 0) + s += GetTlvStringSize(*it); + } + + return s; +} + + +bool RsTlvStringSetRef::SetTlv(void *data, uint32_t size, uint32_t *offset) /* serialise */ +{ + /* must check sizes */ + uint32_t tlvsize = TlvSize(); + uint32_t tlvend = *offset + tlvsize; + + if (size < tlvend) + return false; /* not enough space */ + + bool ok = true; + + + /* start at data[offset] */ + ok &= SetTlvBase(data, tlvend, offset, mType , tlvsize); + + /* determine the total size of ids strings in list */ + + std::list::iterator it; + + for(it = ids.begin(); it != ids.end() ; ++it) + { + if (it->length() > 0) + ok &= SetTlvString(data, tlvend, offset, TLV_TYPE_STR_GENID, *it); + } + + return ok; + +} + + +bool RsTlvStringSetRef::GetTlv(void *data, uint32_t size, uint32_t *offset) /* serialise */ +{ + if (size < *offset + TLV_HEADER_SIZE) + return false; + + uint16_t tlvtype = GetTlvType( &(((uint8_t *) data)[*offset]) ); + uint32_t tlvsize = GetTlvSize( &(((uint8_t *) data)[*offset]) ); + uint32_t tlvend = *offset + tlvsize; + + + + if (size < tlvend) /* check size */ + return false; /* not enough space */ + + if (tlvtype != mType) /* check type */ + return false; + + bool ok = true; + + /* ready to load */ + TlvClear(); + + /* skip the header */ + (*offset) += TLV_HEADER_SIZE; + + + +/* while there is TLV */ + while((*offset) + 2 < tlvend) + { + /* get the next type */ + uint16_t tlvsubtype = GetTlvType( &(((uint8_t *) data)[*offset]) ); + + if (tlvsubtype == TLV_TYPE_STR_GENID) + { + std::string newIds; + ok &= GetTlvString(data, tlvend, offset, TLV_TYPE_STR_GENID, newIds); + if(ok) + { + ids.push_back(newIds); + + } + } + else + { + /* Step past unknown TLV TYPE */ + ok &= SkipUnknownTlv(data, tlvend, offset); + } + + if (!ok) + { + break; + } + } + + /*************************************************************************** + * NB: extra components could be added (for future expansion of the type). + * or be present (if this code is reading an extended version). + * + * We must chew up the extra characters to conform with TLV specifications + ***************************************************************************/ + if (*offset != tlvend) + { +#ifdef TLV_DEBUG + std::cerr << "RsTlvPeerIdSetRef::GetTlv() Warning extra bytes at end of item"; + std::cerr << std::endl; +#endif + *offset = tlvend; + } + + return ok; +} + +/// print to screen RsTlvStringSet contents +std::ostream &RsTlvStringSetRef::print(std::ostream &out, uint16_t indent) +{ + printBase(out, "RsTlvStringSetRef", indent); + uint16_t int_Indent = indent + 2; + + printIndent(out, int_Indent); + out << "type:" << mType; + out << std::endl; + + std::list::iterator it; + for(it = ids.begin(); it != ids.end() ; ++it) + { + printIndent(out, int_Indent); + out << "id:" << *it; + out << std::endl; + } + + printEnd(out, "RsTlvStringSetRef", indent); + return out; + +} + /************************************* Service Id Set ************************************/ void RsTlvServiceIdSet::TlvClear() diff --git a/libretroshare/src/serialiser/rstlvtypes.h b/libretroshare/src/serialiser/rstlvtypes.h index 4148faa62..9bae52f89 100644 --- a/libretroshare/src/serialiser/rstlvtypes.h +++ b/libretroshare/src/serialiser/rstlvtypes.h @@ -137,6 +137,22 @@ virtual std::ostream &print(std::ostream &out, uint16_t indent); }; +class RsTlvStringSetRef: public RsTlvItem +{ + public: + RsTlvStringSetRef(uint16_t type, std::list &refids); +virtual ~RsTlvStringSetRef() { return; } +virtual uint32_t TlvSize(); +virtual void TlvClear(); +virtual bool SetTlv(void *data, uint32_t size, uint32_t *offset); /* serialise */ +virtual bool GetTlv(void *data, uint32_t size, uint32_t *offset); /* deserialise */ +virtual std::ostream &print(std::ostream &out, uint16_t indent); + + uint16_t mType; + std::list &ids; /* Mandatory */ +}; + + /**** MORE TLV ***** * * File Items/Data. diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index 7b7766bd2..bdc8472e0 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -217,7 +217,7 @@ bool p3GxsForums::createGroup(uint32_t &token, RsGxsForumGroup &group) bool p3GxsForums::updateGroup(uint32_t &token, RsGxsGroupUpdateMeta& meta, RsGxsForumGroup &group) { - std::cerr << "p3GxsForums::createGroup()" << std::endl; + std::cerr << "p3GxsForums::updateGroup()" << std::endl; if(meta.getGroupId().empty()) return false; diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 96e0e6569..5864ae32d 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -29,6 +29,7 @@ #include "retroshare/rsiface.h" #include "util/rsrandom.h" #include "util/rsstring.h" +#include "util/radix64.h" #include "pqi/authgpg.h" @@ -39,10 +40,13 @@ /**** * #define DEBUG_IDS 1 + * #define DEBUG_RECOGN 1 * #define GXSID_GEN_DUMMY_DATA 1 * #define ENABLE_PGP_SIGNATURES 1 ****/ +#define DEBUG_IDS 1 +#define DEBUG_RECOGN 1 #define ENABLE_PGP_SIGNATURES 1 @@ -76,14 +80,16 @@ RsIdentity *rsIdentity = NULL; #define RSGXSID_MAX_SERVICE_STRING 1024 #define BG_PGPHASH 1 -#define BG_REPUTATION 2 +#define BG_RECOGN 2 +#define BG_REPUTATION 3 #define GXSIDREQ_CACHELOAD 0x0001 #define GXSIDREQ_CACHEOWNIDS 0x0002 #define GXSIDREQ_PGPHASH 0x0010 -#define GXSIDREQ_REPUTATION 0x0020 +#define GXSIDREQ_RECOGN 0x0020 +#define GXSIDREQ_REPUTATION 0x0030 #define GXSIDREQ_CACHETEST 0x1000 @@ -94,7 +100,10 @@ RsIdentity *rsIdentity = NULL; #define GXSID_EVENT_PGPHASH 0x0010 #define GXSID_EVENT_PGPHASH_PROC 0x0011 -#define GXSID_EVENT_REPUTATION 0x0020 +#define GXSID_EVENT_RECOGN 0x0020 +#define GXSID_EVENT_RECOGN_PROC 0x0021 + +#define GXSID_EVENT_REPUTATION 0x0030 #define GXSID_EVENT_CACHETEST 0x1000 @@ -115,6 +124,10 @@ RsIdentity *rsIdentity = NULL; #define PGPHASH_RETRY_PERIOD 11 #define PGPHASH_PROC_PERIOD 1 +#define RECOGN_PERIOD 90 +#define RECOGN_RETRY_PERIOD 17 +#define RECOGN_PROC_PERIOD 1 + #define REPUTATION_PERIOD 60 #define REPUTATION_RETRY_PERIOD 13 #define REPUTATION_PROC_PERIOD 1 @@ -126,14 +139,15 @@ RsIdentity *rsIdentity = NULL; p3IdService::p3IdService(RsGeneralDataService *gds, RsNetworkExchangeService *nes) : RsGxsIdExchange(gds, nes, new RsGxsIdSerialiser(), RS_SERVICE_GXSV2_TYPE_GXSID, idAuthenPolicy()), RsIdentity(this), GxsTokenQueue(this), RsTickEvent(), mIdMtx("p3IdService"), - mPublicKeyCache(DEFAULT_MEM_CACHE_SIZE, "GxsIdPublicKeyCache"), - mPrivateKeyCache(DEFAULT_MEM_CACHE_SIZE, "GxsIdPrivateKeyCache") + mPublicKeyCache(DEFAULT_MEM_CACHE_SIZE, "GxsIdPublicKeyCache"), + mPrivateKeyCache(DEFAULT_MEM_CACHE_SIZE, "GxsIdPrivateKeyCache") { mBgSchedule_Mode = 0; mBgSchedule_Active = false; // Kick off Cache Testing, + Others. RsTickEvent::schedule_in(GXSID_EVENT_PGPHASH, PGPHASH_PERIOD); + RsTickEvent::schedule_in(GXSID_EVENT_RECOGN, RECOGN_PERIOD); RsTickEvent::schedule_in(GXSID_EVENT_REPUTATION, REPUTATION_PERIOD); RsTickEvent::schedule_now(GXSID_EVENT_CACHEOWNIDS); @@ -146,6 +160,7 @@ p3IdService::p3IdService(RsGeneralDataService *gds, RsNetworkExchangeService *ne #endif #endif + loadRecognKeys(); } @@ -300,6 +315,81 @@ bool p3IdService::createIdentity(uint32_t& token, RsIdentityParameters ¶ms) return true; } +bool p3IdService::updateIdentity(uint32_t& token, RsGxsIdGroup &group) +{ + std::cerr << "p3IdService::updateIdentity()"; + std::cerr << std::endl; + + updateGroup(token, group); + + return false; +} + + +bool p3IdService::parseRecognTag(const RsGxsId &id, const std::string &nickname, + const std::string &tag, RsRecognTagDetails &details) +{ + std::cerr << "p3IdService::parseRecognTag()"; + std::cerr << std::endl; + + RsGxsRecognTagItem *tagitem = RsRecogn::extractTag(tag); + if (!tagitem) + { + return false; + } + + bool isPending = false; + bool isValid = recogn_checktag(id, nickname, tagitem, true, isPending); + + details.valid_from = tagitem->valid_from; + details.valid_to = tagitem->valid_to; + details.tag_class = tagitem->tag_class; + details.tag_type = tagitem->tag_type; + details.signer = tagitem->sign.keyId; + + details.is_valid = isValid; + details.is_pending = isPending; + + delete tagitem; + + return true; +} + +bool p3IdService::getRecognTagRequest(const RsGxsId &id, const std::string &comment, uint16_t tag_class, uint16_t tag_type, std::string &tag) +{ + std::cerr << "p3IdService::getRecognTagRequest()"; + std::cerr << std::endl; + + if (!havePrivateKey(id)) + { + std::cerr << "p3IdService::getRecognTagRequest() Dont have private key"; + std::cerr << std::endl; + // attempt to load it. + cache_request_load(id); + return false; + } + + RsTlvSecurityKey key; + std::string nickname; + + { + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + RsGxsIdCache data; + if (!mPrivateKeyCache.fetch(id, data)) + { + std::cerr << "p3IdService::getRecognTagRequest() Cache failure"; + std::cerr << std::endl; + return false; + } + + key = data.pubkey; + nickname = data.details.mNickname; + } + + return RsRecogn::createTagRequest(key, id, nickname, tag_class, tag_type, comment, tag); +} + + /********************************************************************************/ /******************* RsGixs Interface ***************************************/ @@ -409,6 +499,12 @@ bool p3IdService::getGroupData(const uint32_t &token, std::vector RsGxsIdGroupItem* item = dynamic_cast(*vit); if (item) { +#ifdef DEBUG_IDS + std::cerr << "p3IdService::getGroupData() Item is:"; + std::cerr << std::endl; + item->print(std::cerr); + std::cerr << std::endl; +#endif // DEBUG_IDS RsGxsIdGroup group = item->group; group.mMeta = item->meta; @@ -493,20 +589,64 @@ bool p3IdService::getMsgData(const uint32_t &token, std::vector bool p3IdService::createGroup(uint32_t& token, RsGxsIdGroup &group) { - RsGxsIdGroupItem* item = new RsGxsIdGroupItem(); - item->group = group; - item->meta = group.mMeta; - RsGenExchange::publishGroup(token, item); - return true; + RsGxsIdGroupItem* item = new RsGxsIdGroupItem(); + item->group = group; + item->meta = group.mMeta; + RsGenExchange::publishGroup(token, item); + return true; +} + +bool p3IdService::updateGroup(uint32_t& token, RsGxsIdGroup &group) +{ + RsGxsId id = group.mMeta.mGroupId; + RsGxsIdGroupItem* item = new RsGxsIdGroupItem(); + item->group = group; + item->meta = group.mMeta; + + std::cerr << "p3IdService::updateGroup() Updating RsGxsId: " << id; + std::cerr << std::endl; + + //RsGenExchange::updateGroup(token, item); + + RsGxsGroupUpdateMeta updateMeta(id); + RsGenExchange::updateGroup(token, updateMeta, item); + + // if its in the cache - clear it. + { + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + if (mPublicKeyCache.erase(id)) + { + std::cerr << "p3IdService::updateGroup() Removed from PublicKeyCache"; + std::cerr << std::endl; + } + else + { + std::cerr << "p3IdService::updateGroup() Not in PublicKeyCache"; + std::cerr << std::endl; + } + + if (mPrivateKeyCache.erase(id)) + { + std::cerr << "p3IdService::updateGroup() Removed from PrivateKeyCache"; + std::cerr << std::endl; + } + else + { + std::cerr << "p3IdService::updateGroup() Not in PrivateKeyCache"; + std::cerr << std::endl; + } + } + + return true; } bool p3IdService::createMsg(uint32_t& token, RsGxsIdOpinion &opinion) { - RsGxsIdOpinionItem* item = new RsGxsIdOpinionItem(); - item->opinion = opinion; - item->meta = opinion.mMeta; - RsGenExchange::publishMsg(token, item); - return true; + RsGxsIdOpinionItem* item = new RsGxsIdOpinionItem(); + item->opinion = opinion; + item->meta = opinion.mMeta; + RsGenExchange::publishMsg(token, item); + return true; } /************************************************************************************/ @@ -558,6 +698,85 @@ std::string SSGxsIdPgp::save() const return output; } + +/* Encoding / Decoding Group Service String stuff + * + * RecognTags. + */ + +bool SSGxsIdRecognTags::load(const std::string &input) +{ + char pgpline[RSGXSID_MAX_SERVICE_STRING]; + int pubTs = 0; + int lastTs = 0; + uint32_t flags = 0; + + if (3 == sscanf(input.c_str(), "F:%u P:%d T:%d", &flags, &pubTs, &lastTs)) + { + publishTs = pubTs; + lastCheckTs = lastTs; + tagFlags = flags; + } + else + { + return false; + } + return true; +} + +std::string SSGxsIdRecognTags::save() const +{ + std::string output; + rs_sprintf(output, "F:%u P:%d T:%d", tagFlags, publishTs, lastCheckTs); + return output; +} + +bool SSGxsIdRecognTags::tagsProcessed() const +{ + return (tagFlags & 0x1000); +} + +bool SSGxsIdRecognTags::tagsPending() const +{ + return (tagFlags & 0x2000); +} + +bool SSGxsIdRecognTags::tagValid(int i) const +{ + uint32_t idx = 0x01 << i; + +#ifdef DEBUG_RECOGN + std::cerr << "SSGxsIdRecognTags::tagValid(" << i << ") idx: " << idx; + std::cerr << " result: " << (tagFlags & idx); + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + return (tagFlags & idx); +} + +void SSGxsIdRecognTags::setTags(bool processed, bool pending, uint32_t flags) +{ + flags &= 0x00ff; // clear top bits; + if (processed) + { + flags |= 0x1000; + } + if (pending) + { + flags |= 0x2000; + } + +#ifdef DEBUG_RECOGN + std::cerr << "SSGxsIdRecognTags::setTags(" << processed << "," << pending << "," << flags << ")"; + std::cerr << " tagFlags: " << tagFlags; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + tagFlags = flags; +} + + + bool SSGxsIdScore::load(const std::string &input) { return (1 == sscanf(input.c_str(), "%d", &score)); @@ -585,12 +804,13 @@ std::string SSGxsIdCumulator::save() const bool SSGxsIdGroup::load(const std::string &input) { char pgpstr[RSGXSID_MAX_SERVICE_STRING]; + char recognstr[RSGXSID_MAX_SERVICE_STRING]; char scorestr[RSGXSID_MAX_SERVICE_STRING]; char opinionstr[RSGXSID_MAX_SERVICE_STRING]; char repstr[RSGXSID_MAX_SERVICE_STRING]; - // split into two parts. - if (4 != sscanf(input.c_str(), "v1 {P:%[^}]} {Y:%[^}]} {O:%[^}]} {R:%[^}]}", pgpstr, scorestr, opinionstr, repstr)) + // split into parts. + if (5 != sscanf(input.c_str(), "v1 {P:%[^}]} {T:%[^}]} {Y:%[^}]} {O:%[^}]} {R:%[^}]}", pgpstr, recognstr, scorestr, opinionstr, repstr)) { #ifdef DEBUG_IDS std::cerr << "SSGxsIdGroup::load() Failed to extract 4 Parts"; @@ -616,6 +836,22 @@ bool SSGxsIdGroup::load(const std::string &input) ok = false; } + if (recogntags.load(recognstr)) + { +#ifdef DEBUG_RECOGN + std::cerr << "SSGxsIdGroup::load() recognstr: " << recognstr; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + } + else + { +#ifdef DEBUG_RECOGN + std::cerr << "SSGxsIdGroup::load() Invalid recognstr: " << recognstr; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + ok = false; + } + if (score.load(scorestr)) { #ifdef DEBUG_IDS @@ -682,6 +918,10 @@ std::string SSGxsIdGroup::save() const output += pgp.save(); output += "}"; + output += "{T:"; + output += recogntags.save(); + output += "}"; + output += "{Y:"; output += score.save(); output += "}"; @@ -725,11 +965,17 @@ RsGxsIdCache::RsGxsIdCache() return; } -RsGxsIdCache::RsGxsIdCache(const RsGxsIdGroupItem *item, const RsTlvSecurityKey &in_pkey) +RsGxsIdCache::RsGxsIdCache(const RsGxsIdGroupItem *item, const RsTlvSecurityKey &in_pkey, const std::list &tagList) { // Save Keys. pubkey = in_pkey; + // Save Time for ServiceString comparisions. + mPublishTs = item->meta.mPublishTs; + + // Save RecognTags. + mRecognTags = tagList; + // Fill in Details. details.mNickname = item->meta.mGroupName; details.mId = item->meta.mGroupId; @@ -749,6 +995,8 @@ RsGxsIdCache::RsGxsIdCache(const RsGxsIdGroupItem *item, const RsTlvSecurityKey void RsGxsIdCache::updateServiceString(std::string serviceString) { + details.mRecognTags.clear(); + SSGxsIdGroup ssdata; if (ssdata.load(serviceString)) { @@ -765,6 +1013,54 @@ void RsGxsIdCache::updateServiceString(std::string serviceString) } } + + // process RecognTags. + if (ssdata.recogntags.tagsProcessed()) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsGxsIdCache::updateServiceString() updating recogntags"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + if (ssdata.recogntags.publishTs == mPublishTs) + { + std::list::iterator it; + int i = 0; + for(it = mRecognTags.begin(); it != mRecognTags.end(); it++, i++) + { + if (ssdata.recogntags.tagValid(i) && it->valid) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsGxsIdCache::updateServiceString() Valid Tag: " << it->tag_class << ":" << it->tag_type; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + details.mRecognTags.push_back(*it); + } + else + { +#ifdef DEBUG_RECOGN + std::cerr << "RsGxsIdCache::updateServiceString() Invalid Tag: " << it->tag_class << ":" << it->tag_type; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + } + } + } + else + { +#ifdef DEBUG_RECOGN + std::cerr << "RsGxsIdCache::updateServiceString() recogntags old publishTs"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + } + + } + else + { +#ifdef DEBUG_RECOGN + std::cerr << "RsGxsIdCache::updateServiceString() recogntags unprocessed"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + } + details.mOpinion = 0; details.mReputation = ssdata.score.score; } @@ -779,9 +1075,137 @@ void RsGxsIdCache::updateServiceString(std::string serviceString) } +bool p3IdService::recogn_extract_taginfo(const RsGxsIdGroupItem *item, std::list &tagItems) +{ +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_extract_taginfo()"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + /* process Recogn Tags */ + + std::list::const_iterator rit; + int count = 0; + for(rit = item->group.mRecognTags.begin(); rit != item->group.mRecognTags.end(); rit++) + { + if (++count > RSRECOGN_MAX_TAGINFO) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_extract_taginfo() Too many tags."; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + return true; + } + + RsGxsRecognTagItem *tagitem = RsRecogn::extractTag(*rit); + + if (!tagitem) + { + continue; + } + +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_extract_taginfo() Got TagItem: "; + std::cerr << std::endl; + tagitem->print(std::cerr); +#endif // DEBUG_RECOGN + + tagItems.push_back(tagitem); + } + return true; +} +bool p3IdService::cache_process_recogntaginfo(const RsGxsIdGroupItem *item, std::list &tagList) +{ + /* ServiceString decode */ + SSGxsIdGroup ssdata; + bool recognProcess = false; + if (ssdata.load(item->meta.mServiceString)) + { + if (!ssdata.recogntags.tagsProcessed()) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::cache_process_recogntaginfo() tags not processed"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + /* we need to reprocess it */ + recognProcess = true; + } + else + { + if (item->meta.mPublishTs != ssdata.recogntags.publishTs) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::cache_process_recogntaginfo() publishTs old"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + recognProcess = true; + } + else if (ssdata.recogntags.tagsPending()) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::cache_process_recogntaginfo() tagsPending"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + /* reprocess once a day */ + recognProcess = true; + } + } + } + else + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::cache_process_recogntaginfo() ServiceString invalid"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + recognProcess = true; + } + + std::list tagItems; + std::list::iterator it; + + recogn_extract_taginfo(item, tagItems); + + time_t now = time(NULL); + for(it = tagItems.begin(); it != tagItems.end(); it++) + { + RsRecognTag info((*it)->tag_class, (*it)->tag_type, false); + bool isPending = false; + if (recogn_checktag(item->meta.mGroupId, item->meta.mGroupName, *it, false, isPending)) + { + info.valid = true; + } +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::cache_process_recogntaginfo() Adding Tag: "; + std::cerr << info.tag_class << ":"; + std::cerr << info.tag_type << ":"; + std::cerr << info.valid; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + tagList.push_back(info); + delete *it; + } + + + if (recognProcess) + { + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + mRecognGroupIds.push_back(item->meta.mGroupId); + +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::cache_process_recogntaginfo() Reprocessing groupId: "; + std::cerr << item->meta.mGroupId; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + } + + return true; +} bool p3IdService::cache_store(const RsGxsIdGroupItem *item) @@ -793,7 +1217,7 @@ bool p3IdService::cache_store(const RsGxsIdGroupItem *item) //item->print(std::cerr, 0); NEEDS CONST!!!! TODO //std::cerr << std::endl; - /* extract key from keys */ + /* extract key from keys */ RsTlvSecurityKeySet keySet; RsTlvSecurityKey pubkey; RsTlvSecurityKey fullkey; @@ -814,9 +1238,9 @@ bool p3IdService::cache_store(const RsGxsIdGroupItem *item) //std::cerr << "p3IdService::cache_store() KeySet is:"; //keySet.print(std::cerr, 10); - for (kit = keySet.keys.begin(); kit != keySet.keys.end(); kit++) - { - if (kit->second.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) + for (kit = keySet.keys.begin(); kit != keySet.keys.end(); kit++) + { + if (kit->second.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) { #ifdef DEBUG_IDS std::cerr << "p3IdService::cache_store() Found Admin Key"; @@ -824,7 +1248,7 @@ bool p3IdService::cache_store(const RsGxsIdGroupItem *item) #endif // DEBUG_IDS /* save full key - if we have it */ - if (kit->second.keyFlags & RSTLV_KEY_TYPE_FULL) + if (kit->second.keyFlags & RSTLV_KEY_TYPE_FULL) { fullkey = kit->second; full_key_ok = true; @@ -847,16 +1271,20 @@ bool p3IdService::cache_store(const RsGxsIdGroupItem *item) return false; } + // extract tags. + std::list tagList; + cache_process_recogntaginfo(item, tagList); + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ // Create Cache Data. - RsGxsIdCache pubcache(item, pubkey); + RsGxsIdCache pubcache(item, pubkey, tagList); mPublicKeyCache.store(id, pubcache); mPublicKeyCache.resize(); if (full_key_ok) { - RsGxsIdCache fullcache(item, fullkey); + RsGxsIdCache fullcache(item, fullkey, tagList); mPrivateKeyCache.store(id, fullcache); mPrivateKeyCache.resize(); } @@ -932,8 +1360,8 @@ bool p3IdService::cache_start_load() #endif // DEBUG_IDS uint32_t ansType = RS_TOKREQ_ANSTYPE_DATA; - RsTokReqOptions opts; - opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA; + RsTokReqOptions opts; + opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA; uint32_t token = 0; RsGenExchange::getTokenService()->requestGroupInfo(token, ansType, opts, groupIds); @@ -1041,8 +1469,8 @@ bool p3IdService::cache_request_ownids() uint32_t ansType = RS_TOKREQ_ANSTYPE_DATA; RsTokReqOptions opts; - opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA; - //opts.mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_ADMIN; + opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA; + //opts.mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_ADMIN; uint32_t token = 0; @@ -1132,8 +1560,8 @@ bool p3IdService::cachetest_getlist() #endif // DEBUG_IDS uint32_t ansType = RS_TOKREQ_ANSTYPE_LIST; - RsTokReqOptions opts; - opts.mReqType = GXS_REQUEST_TYPE_GROUP_IDS; + RsTokReqOptions opts; + opts.mReqType = GXS_REQUEST_TYPE_GROUP_IDS; uint32_t token = 0; RsGenExchange::getTokenService()->requestGroupInfo(token, ansType, opts); @@ -1154,11 +1582,11 @@ bool p3IdService::cachetest_handlerequest(uint32_t token) std::list grpIds; bool ok = RsGenExchange::getGroupList(token, grpIds); - if(ok) - { - std::list::iterator vit = grpIds.begin(); - for(; vit != grpIds.end(); vit++) - { + if(ok) + { + std::list::iterator vit = grpIds.begin(); + for(; vit != grpIds.end(); vit++) + { /* 5% chance of checking it! */ if (RSRandom::random_f32() < 0.25) { @@ -1189,7 +1617,7 @@ bool p3IdService::cachetest_handlerequest(uint32_t token) #endif // DEBUG_IDS // success! - seckey.print(std::cerr, 10); + seckey.print(std::cerr, 10); std::cerr << std::endl; @@ -1229,7 +1657,7 @@ bool p3IdService::cachetest_handlerequest(uint32_t token) } } } - } + } else { std::cerr << "p3IdService::cache_load_for_token() ERROR no data"; @@ -1245,13 +1673,13 @@ bool p3IdService::cachetest_handlerequest(uint32_t token) /************************************************************************************/ /* - * We have two background tasks that use the ServiceString: PGPHash & Reputation. + * We have three background tasks that use the ServiceString: PGPHash & Reputation & Recogn * * Only one task can be run at a time - otherwise potential overwrite issues. * So this part coordinates that part of the code. * - * - * + * We are going to have a "fetcher task", which gets all the UNPROCESSED / UPDATED GROUPS. + * and sets the CHECK_PGP, CHECK_RECOGN, etc... this will reduce the "Get All" calls. * */ @@ -1263,7 +1691,7 @@ bool p3IdService::CacheArbitration(uint32_t mode) if (!mBgSchedule_Active) { #ifdef DEBUG_IDS - std::cerr << "p3IdService::CacheArbitration() Okay"; + std::cerr << "p3IdService::CacheArbitration() Okay: mode " << mode; std::cerr << std::endl; #endif // DEBUG_IDS @@ -1273,7 +1701,7 @@ bool p3IdService::CacheArbitration(uint32_t mode) } #ifdef DEBUG_IDS - std::cerr << "p3IdService::CacheArbitration() Is busy..."; + std::cerr << "p3IdService::CacheArbitration() Is busy in mode: " << mBgSchedule_Mode; std::cerr << std::endl; #endif // DEBUG_IDS @@ -1347,6 +1775,13 @@ RsGenExchange::ServiceCreate_Return p3IdService::service_CreateGroup(RsGxsGrpIte return SERVICE_CREATE_FAIL; } +#ifdef DEBUG_IDS + std::cerr << "p3IdService::service_CreateGroup() Item is:"; + std::cerr << std::endl; + item->print(std::cerr); + std::cerr << std::endl; +#endif // DEBUG_IDS + /********************* TEMP HACK UNTIL GXS FILLS IN GROUP_ID *****************/ // find private admin key std::map::iterator mit = keySet.keys.begin(); @@ -1539,11 +1974,9 @@ bool p3IdService::pgphash_start() // Also need to use opts.groupFlags to filter stuff properly to REALID's only. // TODO - //uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY; uint32_t ansType = RS_TOKREQ_ANSTYPE_DATA; - RsTokReqOptions opts; - //opts.mReqType = GXS_REQUEST_TYPE_GROUP_META; - opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA; + RsTokReqOptions opts; + opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA; uint32_t token = 0; RsGenExchange::getTokenService()->requestGroupInfo(token, ansType, opts); @@ -1711,29 +2144,6 @@ bool p3IdService::pgphash_process() ssdata.pgp.idKnown = true; ssdata.pgp.pgpId = pgpId.toStdString(); -// SHOULD BE PUSHED TO CACHE! -#if 0 - id.mGpgIdKnown = true; - - id.mGpgId = *it; - id.mGpgName = details.name; - id.mGpgEmail = details.email; - - if (*it == ownId) - { - id.mIdType |= RSID_RELATION_YOURSELF; - } - else if (rsPeers->isGPGAccepted(*it)) - { - id.mIdType |= RSID_RELATION_FRIEND; - } - else - { - id.mIdType |= RSID_RELATION_OTHER; - } - -#endif - } else { @@ -1768,7 +2178,7 @@ bool p3IdService::checkId(const RsGxsIdGroup &grp, PGPIdType &pgpId) std::cerr << std::endl; #endif // DEBUG_IDS - /* some sanity checking... make sure hash is the right size */ + /* some sanity checking... make sure hash is the right size */ #ifdef DEBUG_IDS std::cerr << "p3IdService::checkId() PgpIdHash is: " << grp.mPgpIdHash; @@ -1903,13 +2313,13 @@ void calcPGPHash(const RsGxsId &id, const PGPFingerprintType &pgp, GxsIdPgpHash { unsigned char signature[SHA_DIGEST_LENGTH]; /* hash id + pubkey => pgphash */ - SHA_CTX *sha_ctx = new SHA_CTX; - SHA1_Init(sha_ctx); + SHA_CTX *sha_ctx = new SHA_CTX; + SHA1_Init(sha_ctx); - SHA1_Update(sha_ctx, id.c_str(), id.length()); // TO FIX ONE DAY. - SHA1_Update(sha_ctx, pgp.toByteArray(), pgp.SIZE_IN_BYTES); - SHA1_Final(signature, sha_ctx); - hash = GxsIdPgpHash(signature); + SHA1_Update(sha_ctx, id.c_str(), id.length()); // TO FIX ONE DAY. + SHA1_Update(sha_ctx, pgp.toByteArray(), pgp.SIZE_IN_BYTES); + SHA1_Final(signature, sha_ctx); + hash = GxsIdPgpHash(signature); #ifdef DEBUG_IDS std::cerr << "calcPGPHash():"; @@ -1922,9 +2332,325 @@ void calcPGPHash(const RsGxsId &id, const PGPFingerprintType &pgp, GxsIdPgpHash std::cerr << std::endl; #endif // DEBUG_IDS - delete sha_ctx; + delete sha_ctx; } +/************************************************************************************/ +/************************************************************************************/ +/************************************************************************************/ +/************************************************************************************/ + +/* Task to validate Recogn Tags. + * + * Info to be stored in GroupServiceString + Cache. + **/ + + +bool p3IdService::recogn_start() +{ + if (!CacheArbitration(BG_RECOGN)) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_start() Other Events running... Rescheduling"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + /* reschedule in a bit */ + RsTickEvent::schedule_in(GXSID_EVENT_RECOGN, RECOGN_RETRY_PERIOD); + return false; + } + + // SCHEDULE NEXT ONE. + RsTickEvent::schedule_in(GXSID_EVENT_RECOGN, RECOGN_PERIOD); + + +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_start() making request"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + std::list recognList; + { + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + + recognList = mRecognGroupIds; + mRecognGroupIds.clear(); + } + + if (recognList.empty()) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_start() List is Empty, cancelling"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + // FINISHED. + CacheArbitrationDone(BG_RECOGN); + return false; + } + + uint32_t ansType = RS_TOKREQ_ANSTYPE_DATA; + RsTokReqOptions opts; + opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA; + uint32_t token = 0; + + RsGenExchange::getTokenService()->requestGroupInfo(token, ansType, opts, recognList); + GxsTokenQueue::queueRequest(token, GXSIDREQ_RECOGN); + return true; + +} + + +bool p3IdService::recogn_handlerequest(uint32_t token) +{ +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_handlerequest(" << token << ")"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + std::vector grpData; + bool ok = RsGenExchange::getGroupData(token, grpData); + + if(ok) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_request() Have " << grpData.size() << " Groups"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + std::vector::iterator vit = grpData.begin(); + + for(; vit != grpData.end(); vit++) + { + RsGxsIdGroupItem* item = dynamic_cast(*vit); + if (item) + { + +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_request() Group Id: " << item->meta.mGroupId; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + mRecognGroupsToProcess.push_back(item); + } + else + { + delete (*vit); + } + } + } + else + { + std::cerr << "p3IdService::recogn_request() getGroupData ERROR"; + std::cerr << std::endl; + } + + // Schedule Processing. + RsTickEvent::schedule_in(GXSID_EVENT_RECOGN_PROC, RECOGN_PROC_PERIOD); + return true; +} + + +bool p3IdService::recogn_process() +{ + /* each time this is called - process one Id from mGroupsToProcess */ + RsGxsIdGroupItem *item; + bool isDone = false; + { + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + if (!mRecognGroupsToProcess.empty()) + { + item = mRecognGroupsToProcess.front(); + mGroupsToProcess.pop_front(); + +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_process() Popped Group: " << item->meta.mGroupId; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + } + else + { + isDone = true; + } + } + + if (isDone) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_process() List Empty... Done"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + // FINISHED. + CacheArbitrationDone(BG_RECOGN); + return true; + } + + + + std::list tagItems; + std::list::iterator it; + + recogn_extract_taginfo(item, tagItems); + + bool isPending = false; + int i = 1; + uint32_t tagValidFlags = 0; + for(it = tagItems.begin(); it != tagItems.end(); it++) + { + bool isTagPending = false; + bool isTagOk = recogn_checktag(item->meta.mGroupId, item->meta.mGroupName, *it, true, isPending); + if (isTagOk) + { + tagValidFlags |= i; + } + else + { + isPending |= isTagPending; + } + + delete *it; + i *= 2; + } + +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_process() Tags Checked, saving"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + SSGxsIdGroup ssdata; + ssdata.load(item->meta.mServiceString); // attempt load - okay if fails. + + ssdata.recogntags.setTags(true, isPending, tagValidFlags); + ssdata.recogntags.lastCheckTs = time(NULL); + ssdata.recogntags.publishTs = item->meta.mPublishTs; + + /* set new Group ServiceString */ + uint32_t dummyToken = 0; + std::string serviceString = ssdata.save(); + setGroupServiceString(dummyToken, item->meta.mGroupId, serviceString); + + cache_update_if_cached(item->meta.mGroupId, serviceString); + + delete item; + + // Schedule Next Processing. + RsTickEvent::schedule_in(GXSID_EVENT_RECOGN_PROC, RECOGN_PROC_PERIOD); + return false; // as there are more items on the queue to process. +} + + +bool p3IdService::recogn_checktag(const RsGxsId &id, const std::string &nickname, RsGxsRecognTagItem *item, bool doSignCheck, bool &isPending) +{ + +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_checktag() groupId: " << id; + std::cerr << std::endl; + std::cerr << "p3IdService::recogn_checktag() nickname: " << nickname; + std::cerr << std::endl; + std::cerr << "p3IdService::recogn_checktag() item: "; + std::cerr << std::endl; + ((RsGxsRecognTagItem *) item)->print(std::cerr); +#endif // DEBUG_RECOGN + + // To check: + // ------------------- + // date range. + // id matches. + // nickname matches. + // signer is valid. + // ------ + // signature is valid. (only if doSignCheck == true) + + time_t now = time(NULL); + isPending = false; + + // check date range. + if ((item->valid_from > now) || (item->valid_to < now)) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_checktag() failed timestamp"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + return false; + } + + // id match. + if (id != item->identity) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_checktag() failed identity"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return false; + } + + // nickname match. + if (nickname != item->nickname) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_checktag() failed nickname"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return false; + } + + + + { + /* check they validity of the Tag */ + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + + + std::map::iterator it; + it = mRecognSignKeys.find(item->sign.keyId); + if (it == mRecognSignKeys.end()) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_checktag() failed to find signkey"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + + // If OldKey, then we don't want to reprocess. + if (mRecognOldSignKeys.end() != + mRecognOldSignKeys.find(item->sign.keyId)) + { + isPending = true; // need to reprocess later with new key + } + return false; + } + + // Check tag_class is okay for signer. + if (it->second->signing_classes.ids.end() == + std::find(it->second->signing_classes.ids.begin(), it->second->signing_classes.ids.end(), item->tag_class)) + { +#ifdef DEBUG_RECOGN + std::cerr << "p3IdService::recogn_checktag() failed signing_class check"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return false; + } + + // ALL Okay, just signature to check. + if (!doSignCheck) + { + return true; + } + + return RsRecogn::validateTagSignature(it->second, item); + } +} + + +void p3IdService::loadRecognKeys() +{ + RsStackMutex stack(mIdMtx); /**** LOCKED MUTEX ****/ + + RsRecogn::loadSigningKeys(mRecognSignKeys); +} + + /************************************************************************************/ /************************************************************************************/ @@ -2003,13 +2729,13 @@ void p3IdService::generateDummy_OwnIds() for(i = 0; i < nIds; i++) { RsGxsIdGroup id; - RsPeerDetails details; + RsPeerDetails details; id.mMeta.mGroupFlags = RSGXSID_GROUPFLAG_REALID; // HACK FOR DUMMY GENERATION. id.mMeta.mAuthorId = ownId; - if (rsPeers->getPeerDetails(ownId, details)) + if (rsPeers->getPeerDetails(ownId, details)) { std::ostringstream out; out << details.name << "_" << i + 1; @@ -2339,7 +3065,7 @@ bool p3IdService::background_checkTokenRequest() uint32_t anstype; time_t ts; - + status = RsGenExchange::getTokenService()->requestStatus(token); //checkRequestStatus(token, status, reqtype, anstype, ts); @@ -2390,7 +3116,7 @@ bool p3IdService::background_requestGroups() } uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY; - RsTokReqOptions opts; + RsTokReqOptions opts; std::list groupIds; /** @@ -2445,7 +3171,7 @@ bool p3IdService::background_requestNewMessages() for(it = modGroupList.begin(); it != modGroupList.end(); it++) { /*** TODO - uint32_t dummyToken = 0; + uint32_t dummyToken = 0; setGroupStatusFlags(dummyToken, it->mGroupId, 0, RSGXS_GROUP_STATUS_NEWMSG); ***/ @@ -2455,7 +3181,7 @@ bool p3IdService::background_requestNewMessages() } uint32_t ansType = RS_TOKREQ_ANSTYPE_SUMMARY; - RsTokReqOptions opts; + RsTokReqOptions opts; token = 0; /* TODO @@ -2501,7 +3227,7 @@ bool p3IdService::background_processNewMessages() * and flag these items as modified - so we rewrite them to the db later. * * If a message is not an original -> store groupId for requiring full analysis later. - */ + */ std::map::iterator mit; @@ -2639,7 +3365,7 @@ bool p3IdService::background_processNewMessages() #endif // DEBUG_IDS /* set Cache */ - uint32_t dummyToken = 0; + uint32_t dummyToken = 0; setGroupServiceString(dummyToken, mit->second.mGroupId, mit->second.mServiceString); } else @@ -2688,7 +3414,7 @@ bool p3IdService::background_FullCalcRequest() /* request the summary info from the parents */ uint32_t ansType = RS_TOKREQ_ANSTYPE_DATA; uint32_t token = 0; - RsTokReqOptions opts; + RsTokReqOptions opts; opts.mOptions = RS_TOKREQOPT_MSG_LATEST; RsGenExchange::getTokenService()->requestMsgInfo(token, ansType, opts, groupIds); @@ -2727,10 +3453,10 @@ bool p3IdService::background_processFullCalc() double rep_sum = 0; double rep_sumsq = 0; - std::vector opinions; - std::vector::iterator vit; + std::vector opinions; + std::vector::iterator vit; - if (!getMsgData(mBgToken, opinions)) + if (!getMsgData(mBgToken, opinions)) { std::cerr << "p3IdService::background_processFullCalc() ERROR Failed to get Opinions"; std::cerr << std::endl; @@ -2739,7 +3465,7 @@ bool p3IdService::background_processFullCalc() } std::string groupId; - for(vit = opinions.begin(); vit != opinions.end(); vit++) + for(vit = opinions.begin(); vit != opinions.end(); vit++) { RsGxsIdOpinion &opinion = *vit; @@ -2878,7 +3604,7 @@ std::ostream &operator<<(std::ostream &out, const RsGxsIdOpinion &opinion) - // Overloaded from GxsTokenQueue for Request callbacks. +// Overloaded from GxsTokenQueue for Request callbacks. void p3IdService::handleResponse(uint32_t token, uint32_t req_type) { #ifdef DEBUG_IDS @@ -2898,6 +3624,9 @@ void p3IdService::handleResponse(uint32_t token, uint32_t req_type) case GXSIDREQ_PGPHASH: pgphash_handlerequest(token); break; + case GXSIDREQ_RECOGN: + recogn_handlerequest(token); + break; case GXSIDREQ_CACHETEST: cachetest_handlerequest(token); break; @@ -2949,6 +3678,14 @@ void p3IdService::handle_event(uint32_t event_type, const std::string &elabel) pgphash_process(); break; + case GXSID_EVENT_RECOGN: + recogn_start(); + break; + + case GXSID_EVENT_RECOGN_PROC: + recogn_process(); + break; + case GXSID_EVENT_DUMMYDATA: generateDummyData(); break; diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index e9560e8df..a42712ed4 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -38,9 +38,12 @@ #include "util/rsmemcache.h" #include "util/rstickevent.h" +#include "util/rsrecogn.h" #include "pqi/authgpg.h" +#include "serialiser/rsgxsrecognitems.h" + /* * Identity Service * @@ -73,6 +76,27 @@ virtual std::string save() const; std::string pgpId; }; +class SSGxsIdRecognTags: public SSBit +{ + public: + SSGxsIdRecognTags() + :tagFlags(0), publishTs(0), lastCheckTs(0) { return; } + +virtual bool load(const std::string &input); +virtual std::string save() const; + + void setTags(bool processed, bool pending, uint32_t flags); + + bool tagsProcessed() const; // have we processed? + bool tagsPending() const; // should we reprocess? + bool tagValid(int i) const; + + time_t publishTs; + time_t lastCheckTs; + uint32_t tagFlags; +}; + + class SSGxsIdScore: public SSBit { public: @@ -113,11 +137,14 @@ virtual std::string save() const; // pgphash status SSGxsIdPgp pgp; + // recogTags. + SSGxsIdRecognTags recogntags; + // reputation score. SSGxsIdScore score; SSGxsIdCumulator opinion; SSGxsIdCumulator reputation; - + }; #define ID_LOCAL_STATUS_FULL_CALC_FLAG 0x00010000 @@ -133,10 +160,14 @@ class RsGxsIdCache { public: RsGxsIdCache(); - RsGxsIdCache(const RsGxsIdGroupItem *item, const RsTlvSecurityKey &in_pkey); + RsGxsIdCache(const RsGxsIdGroupItem *item, const RsTlvSecurityKey &in_pkey, + const std::list &tagList); void updateServiceString(std::string serviceString); + time_t mPublishTs; + std::list mRecognTags; // Only partially validated. + RsIdentityDetails details; RsTlvSecurityKey pubkey; }; @@ -175,6 +206,7 @@ virtual bool getMsgData(const uint32_t &token, std::vector &opin // These are local - and not exposed via RsIdentity. virtual bool createGroup(uint32_t& token, RsGxsIdGroup &group); +virtual bool updateGroup(uint32_t& token, RsGxsIdGroup &group); virtual bool createMsg(uint32_t& token, RsGxsIdOpinion &opinion); /**************** RsIdentity External Interface. @@ -197,6 +229,13 @@ virtual bool getOwnIds(std::list &ownIds); virtual bool submitOpinion(uint32_t& token, RsIdOpinion &opinion); virtual bool createIdentity(uint32_t& token, RsIdentityParameters ¶ms); +virtual bool updateIdentity(uint32_t& token, RsGxsIdGroup &group); + +virtual bool parseRecognTag(const RsGxsId &id, const std::string &nickname, + const std::string &tag, RsRecognTagDetails &details); +virtual bool getRecognTagRequest(const RsGxsId &id, const std::string &comment, + uint16_t tag_class, uint16_t tag_type, std::string &tag); + /**************** RsGixs Implementation * Notes: @@ -303,6 +342,32 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); std::map mPgpFingerprintMap; std::list mGroupsToProcess; +/************************************************************************ + * recogn processing. + * + */ + bool recogn_start(); + bool recogn_handlerequest(uint32_t token); + bool recogn_process(); + + // helper functions. + bool recogn_extract_taginfo(const RsGxsIdGroupItem *item, std::list &tagItems); + bool cache_process_recogntaginfo(const RsGxsIdGroupItem *item, std::list &tagList); + + bool recogn_checktag(const RsGxsId &id, const std::string &nickname, RsGxsRecognTagItem *item, bool doSignCheck, bool &isPending); + + void loadRecognKeys(); + + /* MUTEX PROTECTED DATA (mIdMtx - maybe should use a 2nd?) */ + + bool checkRecognSignature_locked(std::string encoded, RSA &key, std::string signature); + bool getRecognKey_locked(std::string signer, RSA &key); + + std::list mRecognGroupIds; + std::list mRecognGroupsToProcess; + std::map mRecognSignKeys; + std::map mRecognOldSignKeys; + /************************************************************************ * Below is the background task for processing opinions => reputations * diff --git a/libretroshare/src/services/p3posted.cc b/libretroshare/src/services/p3posted.cc index b6afe5e2d..46a4220a0 100644 --- a/libretroshare/src/services/p3posted.cc +++ b/libretroshare/src/services/p3posted.cc @@ -40,7 +40,6 @@ /**** * #define POSTED_DEBUG 1 ****/ -#define POSTED_DEBUG 1 RsPosted *rsPosted = NULL; diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index 9446e8686..5813fd84e 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -544,7 +544,7 @@ RsTurtleGenericDataItem::RsTurtleGenericDataItem(void *data,uint32_t pktsize) if(data_bytes != NULL) { - memcpy(data_bytes,data+offset,data_size) ; + memcpy(data_bytes,(void *)((uint8_t *)data+offset),data_size) ; offset += data_size ; } else @@ -585,7 +585,7 @@ bool RsTurtleGenericDataItem::serialize(void *data,uint32_t& pktsize) ok &= setRawUInt32(data, tlvsize, &offset, tunnel_id); ok &= setRawUInt32(data, tlvsize, &offset, data_size); - memcpy(data+offset,data_bytes,data_size) ; + memcpy((void *)((uint8_t *)data+offset),data_bytes,data_size) ; offset += data_size ; if (offset != tlvsize) diff --git a/libretroshare/src/util/rsmemcache.h b/libretroshare/src/util/rsmemcache.h index 5c12e3183..0f5fb966f 100644 --- a/libretroshare/src/util/rsmemcache.h +++ b/libretroshare/src/util/rsmemcache.h @@ -65,6 +65,7 @@ template class RsMemCache bool fetch(const Key &key, Value &data); Value &ref(const Key &key); // like map[] installs empty one if non-existent. bool store(const Key &key, const Value &data); + bool erase(const Key &key); // clean up cache. bool resize(); // should be called periodically to cleanup old entries. @@ -169,6 +170,47 @@ template bool RsMemCache::fetch(const Key &k } +template bool RsMemCache::erase(const Key &key) +{ +#ifdef DEBUG_RSMEMCACHE + std::cerr << "RsMemCache::erase()"; + std::cerr << std::endl; + printStats(std::cerr); +#endif // DEBUG_RSMEMCACHE + + typename std::map::iterator it; + it = mDataMap.find(key); + if (it == mDataMap.end()) + { +#ifdef DEBUG_RSMEMCACHE + std::cerr << "RsMemCache::erase(" << key << ") false"; + std::cerr << std::endl; +#endif // DEBUG_RSMEMCACHE + + mStats_accessmiss++; + return false; + } + +#ifdef DEBUG_RSMEMCACHE + std::cerr << "RsMemCache::erase(" << key << ") OK"; + std::cerr << std::endl; +#endif // DEBUG_RSMEMCACHE + + + /* get timestamps */ + time_t old_ts = it->second.ts; + time_t new_ts = 0; + + // remove from lru. + mDataMap.erase(it); + update_lrumap(key, old_ts, new_ts); + + mStats_access++; + return true; +} + + + template Value &RsMemCache::ref(const Key &key) { #ifdef DEBUG_RSMEMCACHE diff --git a/libretroshare/src/util/rsrecogn.cc b/libretroshare/src/util/rsrecogn.cc new file mode 100644 index 000000000..19c189c6b --- /dev/null +++ b/libretroshare/src/util/rsrecogn.cc @@ -0,0 +1,640 @@ +/* + * libretroshare/src/util: rsrecogn.cc + * + * RetroShare Utilities + * + * Copyright 2013 by Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include +#include + +#include "pqi/pqi_base.h" + +#include "util/rsrecogn.h" +#include "util/radix64.h" +#include "util/rsstring.h" + +#include "gxs/gxssecurity.h" + +/*** + * #define DEBUG_RECOGN 1 + ***/ + +#define DEBUG_RECOGN 1 + +static std::string RecognKey = "MIICCgKCAgEA4/i2fy+zR27/H8fzphM8mR/Nz+yXjMJTtqKlCvEMQlyk7lKzDbKifNjGSiAXjSv3b9TRgMtje7hfEhs3//Oeu4KsCf6sz17aj2StBF579IdJTSUPDwq6jCsZ6NDEYpG8xz3FVV+Ac8q5Vpr/+jdg23ta09zq4aV8VIdIsroVOmZQqjwPcmQK57iWHd538i/XBtc2rnzbYq5bprnmtAKdx55gXVXDfALa0s6yR0HYvCaWguMEJhMIKWfi/9PEgLgwF9OmRwywc2TU/EdvYJo8fYHLfGk0PnYBuL1oSnn3cwAAef02W2JyCzQ84g30tLSUk+hC1LLi+iYj3x7IRR4q7Rlf/FYv/Q5fvjRtPT9eqM6fKyJ9ZO4NjlrSPFGydNbgABzP6WMhBzFjUkEKS27bGmr8Qxdj3Zp0TvR2IkyM6oM+6YknuM4RndUEgC1ZxtoIhugMjm6HdMQmoaHNK3kXewgQB90HHqzKA/J1gok3NcqL8Yls5g0LHepVHsU4cuaIqQr5yr665ZTLU2oqn1HIdkgydBYYUt6G3eWJKXYRbDhWPthGo/HK+W+iw6cTGWxzlCZS40EU9efxz4mDuhow67jOe704lBP3kiYXu05Y5uspaYnuvrvIwaRWBYapyR9UmKktnY8xJYrZvrcZgCovAbiodTzWeYg37xjFfGgYNI8CAwEAAQ=="; + +#define NUM_RECOGN_SIGN_KEYS 3 +static std::string RecognSigningKeys[NUM_RECOGN_SIGN_KEYS] = +{ + "AvMxAwAAA5YQMAAAABAANAAAAAoAAAABEEAAAAFMAKQAAAAmYjI2ZTUyNGFlZjczYmY3Y2MyMzUwNTc0ZTMyMjcxZWEAAAAAUl8ogFRAXAABEAAAARQwggEKAoIBAQCyblJK73O/fMI1BXTjInHqIWma62Z2r3K7/giT6Xm3k+lyNokvpR+I45XdEvPRmFVZmTU7XT2n3YiPDLe7y2r9fnYiLvBCdu+FBaVv5UQG8nvFGMLKbhdyRpOSBgDc+Y+8plMPn8jqgfNhLROMowmvDJQkJQjlm80d/9wj+VZ+tLiPPo8uOlghqNhdXDGK7HnfeLrJyD8kLEW7+4huaxR8IsLgjzuK8rovGLYCBcnx4TXvtbEeafJBBBt8S/GPeUaB1rxWpVV6fi+oBU6cvjbEqPzSalIrwNPyqlj+1SbL1jGEGEr1XIMzDa96SVsJ0F93lS3H9c8HdvByAifgzbPZAgMBAAEQUAAAAjIApAAAACZlM2Y4YjY3ZjJmYjM0NzZlZmYxZmM3ZjNhNjEzM2M5OQEgAAACBp1w449QGjchVotgHvGWRh18zpsDUHRv8PlRX1vXy8FMstTrnRjaDofFitmpJm8K6F1t/9jviCdB+BCvRzAS4SxER49YCBp04xZfX7c03xdq/e27jYRds2w6YHTiEgNi5v1cyWhrwDcCdefXRnHTH1UOw3jOoWnlnmM6jEsL39XI5fvsog9z8GxcG54APKA7JgiqhgMcrKRwNk74XJAzcjB6FS8xaV2gzpZZLNZ1TU+tJoLSiRqTU8UiAGbAR85lYLT5Ozdd2C+bTQ9f6vltz8bpzicJzxGCIsYtSL44InQsO/Oar9IgZu8QE4pTuunGJhVqEZru7ZN+oV+wXt51n+24SS0sNgNKVUFS74RfvsFi67CrXSWTOI8bVS0Lvv3EMWMdSF9dHGbdCFnp2/wqbW/4Qz7XYF4lcu9gLe4UtIrZ6TkAvBtnSfvTTdXj7kD6oHDjrUCjHPxdhz3BLRbj1wENZsoS3QDl22Ts7dbO8wHjutsS3/zx4DLlADoFlU8p7HJaCdrsq20P4WCeQJb6JbbLpGRAccKAidAPHMxQ4fr3b+GtjxpLJtXaytr4CPSXsCt4TloE9g5yCE6n/2UxQACp8Guh9l2MXmrD7qEGexhYqFB/OG84u3vL+gskmsKXTEqi2SiSmhvzta2p2hGCLCKRQeYbn+4TsIQfgWtYNQvC", + "AvMxAwAAA5YQMAAAABAANAAAAAoAAAACEEAAAAFMAKQAAAAmYjY0OTJkODMzNTI5ZjMxMGM1MmRjMDc3ZjBmZDgyMjcAAAAAUl8ogFRAXAABEAAAARQwggEKAoIBAQC2SS2DNSnzEMUtwHfw/YInm/XLXEUMktyZTmyMWACBbEfmU6aztT3vxz6UHoCBYtKkzKrfDZLvXe0a5TRLMmK+yfl5IzIVUPdqTg6FF3Bx/GXdj4v/ZP6lAuqY5YeI4wPcKldrrIJ9DTUdhZhgdtgDtxGvrXZ8eFjcl9zM+QEykYKMwfnTCixzVOPCCo3q1lJO13NmlhVQDO+f9vvTZsYDCcZHMqlKZWcCEyY1ZpQiCqlsL8wN6tKxMuSQO8EGdH/tNzsGHwCoZq6EEL7SX/pmc2ABjpDQTLixgXwJtCpw8Fwj1xiavsFFbqSLu3SjUCcrMz9f8U5p2ROyv//lWxsXAgMBAAEQUAAAAjIApAAAACZlM2Y4YjY3ZjJmYjM0NzZlZmYxZmM3ZjNhNjEzM2M5OQEgAAACBksDPQ93PdZBGCEnKXcQsdB4yBA9NpImVR81JZdPmWlTwZGAXGJwt4EkBcz+xdey84JDujVtHJUzIL9Ws/Jq5MuXHr0tP5ebah1GCQF2/Ov7sctUk3UPBxeroon7gZQhuzaIJVhl0rzwWriFUbTu7H7g9eaTHMvyfUg+S0Z2p+e0+PdL5rfGOJJZ6+NJCXxxbQ//cF4s0PAzkjAuwDmC+OiUiU5V6fY4XtRMCEI7w+UCj+wQn2Wu1Wc7xVM9uow13rGaLPYkWZ/9v+wNhg0KCsVfKGhkAGGzGyKI9LAppFVTu52pBlRu9Ung7VkhF0JC2aadYKKFl99wCbsGqUYN/gtfgHYCV24LNVah2dAy8CI9UmHdWk1kIwWazbPTYKLfpYCTFxqEqXqo3ijLf0YPsfhIvCQpc5VHAvLJlDm0RFKwzK6N9Zu9s9IvJHzIpaAAHCQJPtYxPwWMdt83njGo9wu1+aVkl5Sb5X8N16AybbnQ7fCBqJruGBM0LHtWVbHEiEygD7OStzyhT5rXKZSQYMA9I2CvK1t7qfDXDM40k8SVQ5CrS9R8x1wqQbe+DqNJ9tMfbUmN0xrO/w2pTl/4edKW30TShW/fr3vCWpVq8gcm3CVFSZUaC4T9wqH96K6KgIPbmg1Hk158pxXYXopEv6ZxR7UTPxKB0O22aIHB6UQ5", + "AvMxAwAAA5YQMAAAABAANAAAAAoAAAABEEAAAAFMAKQAAAAmOTdhNTJkMThjMDBjYWE3YmZlYmQ4NTg0MDJkMzBhY2QAAAAAUl8ogFRAXAABEAAAARQwggEKAoIBAQCXpS0YwAyqe/69hYQC0wrNz7eUHAmJfR5EV7NVFQeOxtTlFwbdvRMK8ZpfqEoRhIPXAYCc9Dv3F7WcmcFer8d50EWhlK7rCQScaRdwL1UmF1dUY8bR8QxhJOUgwmrlzeKOHi2DJ3/9AXm7NJR8XMJgHEQQwi3z/aQsWrwCUA0mk68C8a3vjLtcMj5XBuNXRtGZ9zFjiI9Xt19y0iIKdYpfzOnJTKVETcjH7XPBBbJETWkrEyToHXPjcfhESAbJDOoyfQQbxHMQNE7no7owN08LoWX2kOSGtl2m6JbE2OEdJig83a6U3PDYfYM5LCfsAJEIroYhB3qZJDE98zGC8jihAgMBAAEQUAAAAjIApAAAACZlM2Y4YjY3ZjJmYjM0NzZlZmYxZmM3ZjNhNjEzM2M5OQEgAAACBiwl7oRPJzLlwDd8AzVolFQH1ZS+MWLA4B1eHCjCXSMn+zS0Su6CrpC6/vLwECaKSfNZ8y7T2fNDPJHMLmc1F6jJkdNZq3TZGNRgJx24OF3G5MU6mAH7DBsz7muFto+URTJl9CdJviIyQAn5E+R4Gp531RJdKlbqJl/gWuQMVem+eo3elpVEn8Ckg0yvFaFdhGFTOPyrXOZ6fI0pdCX0SH2q/vAIxGDRzaSYmsR0Y+oYZs0AeRnZD9iEh1v17xnVEdSoLZmZbjlLXXgqhbdXGik6ZoXQg3bTfl5D1j8Tk/d/CXqf0SUKBnIafaNgUeQSMY95M3k3vjPQN7vHdXmg19GnqQmBnGq45qdKI7+0Erfhl4po1z6yVvx9JfIMIDOsKwO3U/As5zbO2BYso0pUP4+gndissfDfqlPRni3orA0tlV6NuLmXi1wkHCu8HQ8WOqEUlWDJNLNpHW5OmgjMFqlIPt7hX5jlc9eXd4oMyaqXm1Tg8Cgbh5DYaT9A7He47+QhqYlPygqK9Fm0ZnH3Yz51cm3p2tRB1JU7qH9h5UqLLKJMBuIx7e9L5ieTfzKmTw6tqpIpHpiR/8bSQlKkw2LxikFy3OXL5obY1t9sWk35BNZQqcqflI6mkPrvGQKwN+co8GjUon5/Y1HSM6ursaJtkD8dz+oXVyWAokkuD7QZ", + + +}; + + + +EVP_PKEY *RsRecogn::loadMasterKey() +{ + /* load master signing key */ + size_t keylen; + char *keyptr; + + Radix64::decode(RecognKey, keyptr, keylen); + + const unsigned char *keyptr2 = (const unsigned char *) keyptr; + long keylen2 = keylen; + + RSA *rsakey = d2i_RSAPublicKey(NULL, &(keyptr2), keylen2); + delete []keyptr; + + if (!rsakey) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::loadMasterKeys() failed rsakey load"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return NULL; + } + + + EVP_PKEY *signKey = EVP_PKEY_new(); + EVP_PKEY_assign_RSA(signKey, rsakey); + + return signKey; +} + + +bool RsRecogn::loadSigningKeys(std::map &signMap) +{ + + EVP_PKEY *signKey = loadMasterKey(); + RsGxsRecognSerialiser recognSerialiser; + + if (!signKey) + { + std::cerr << "RsRecogn::loadSigningKeys() missing Master Key"; + return false; + } + + + time_t now = time(NULL); + + for(int i = 0; i < NUM_RECOGN_SIGN_KEYS; i++) + { + char *signerbuf; + size_t len; + Radix64::decode(RecognSigningKeys[i], signerbuf, len); + + uint32_t pktsize = len; + RsItem *pktitem = recognSerialiser.deserialise(signerbuf, &pktsize); + RsGxsRecognSignerItem *item = dynamic_cast(pktitem); + + delete []signerbuf; + + if (!item) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::loadSigningKeys() failed to deserialise SignerItem"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + continue; + } + +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::loadSigningKeys() SignerItem: "; + std::cerr << std::endl; + item->print(std::cerr); +#endif // DEBUG_RECOGN + + /* check dates */ + if ((item->key.startTS > (unsigned) now) || (item->key.endTS < (unsigned) now)) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::loadSigningKeys() failed timestamp"; + std::cerr << std::endl; + std::cerr << "RsRecogn::loadSigningKeys() key.startTS: " << item->key.startTS; + std::cerr << std::endl; + std::cerr << "RsRecogn::loadSigningKeys() now: " << now; + std::cerr << std::endl; + std::cerr << "RsRecogn::loadSigningKeys() key.endTS: " << item->key.endTS; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + delete item; + continue; + } + + /* check signature */ + RsTlvKeySignature signature = item->sign; + item->sign.TlvShallowClear(); + + unsigned int siglen = signature.signData.bin_len; + unsigned char *sigbuf = (unsigned char *) signature.signData.bin_data; + + /* store in */ + uint32_t datalen = recognSerialiser.size(item); + uint8_t *data = (uint8_t *) malloc(datalen); + uint32_t pktlen = datalen; + int signOk = 0; + + if (recognSerialiser.serialise(item, data, &pktlen)) + { + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + + EVP_VerifyInit(mdctx, EVP_sha1()); + EVP_VerifyUpdate(mdctx, data, pktlen); + signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey); + + EVP_MD_CTX_destroy(mdctx); + + item->sign = signature; + signature.TlvShallowClear(); + + if (signOk) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::loadSigningKeys() signature ok"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + std::string signerId = item->key.keyId; + signMap[signerId] = item; + } + } + + if (!signOk) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::loadSigningKeys() signature failed"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + delete item; + } + + free(data); + } + + /* clean up */ + EVP_PKEY_free(signKey); + return true; +} + + + +bool RsRecogn::validateTagSignature(RsGxsRecognSignerItem *signer, RsGxsRecognTagItem *item) +{ + if (item->sign.keyId != signer->key.keyId) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::validateTagSignature() keyId mismatch"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return false; + } + + const unsigned char *keyptr = (const unsigned char *) signer->key.keyData.bin_data; + long keylen = signer->key.keyData.bin_len; + + /* extract admin key */ + RSA *rsakey = d2i_RSAPublicKey(NULL, &(keyptr), keylen); + if (!rsakey) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::validateTagSignature() failed extract signkey"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return false; + } + + /* check signature */ + RsTlvKeySignature signature = item->sign; + item->sign.TlvShallowClear(); + + unsigned int siglen = signature.signData.bin_len; + unsigned char *sigbuf = (unsigned char *) signature.signData.bin_data; + + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + EVP_PKEY *signKey = EVP_PKEY_new(); + EVP_PKEY_assign_RSA(signKey, rsakey); + + + /* store in */ + RsGxsRecognSerialiser serialiser; + + uint32_t datalen = serialiser.size(item); + uint8_t *data = (uint8_t *) malloc(datalen); + int signOk = 0; + + uint32_t pktlen = datalen; + if (serialiser.serialise(item, data, &pktlen)) + { + + EVP_VerifyInit(mdctx, EVP_sha1()); + EVP_VerifyUpdate(mdctx, data, pktlen); + signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey); +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::validateTagSignature() sign_result: " << signOk; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + } + else + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::validateTagSignature() failed to serialise"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + } + + // Clean up. + item->sign = signature; + signature.TlvShallowClear(); + + EVP_MD_CTX_destroy(mdctx); + EVP_PKEY_free(signKey); + + free(data); + + return (signOk == 1); +} + + +bool rsa_sanity_check(RSA *rsa) +{ + std::cerr << "rsa_sanity_check()"; + std::cerr << std::endl; + + if (!rsa) + { + std::cerr << "rsa_sanity_check() RSA == NULL"; + std::cerr << std::endl; + return false; + } + + RSA *pubkey = RSAPublicKey_dup(rsa); + + std::string signId = RsRecogn::getRsaKeyId(rsa); + std::string signId2 = RsRecogn::getRsaKeyId(pubkey); + + bool ok = true; + if (signId != signId2) + { + std::cerr << "rsa_sanity_check() ERROR SignId Failure"; + std::cerr << std::endl; + ok = false; + } + + if (1 != RSA_check_key(rsa)) + { + std::cerr << "rsa_sanity_check() ERROR RSA key is not private"; + std::cerr << std::endl; + ok = false; + } + +#if 0 + if (1 == RSA_check_key(pubkey)) + { + std::cerr << "rsa_sanity_check() ERROR RSA dup key is private"; + std::cerr << std::endl; + ok = false; + } +#endif + + RSA_free(pubkey); + + if (!ok) + { + exit(1); + } + + return true; +} + + + +bool RsRecogn::signTag(EVP_PKEY *signKey, RsGxsRecognTagItem *item) +{ + RsGxsRecognSerialiser serialiser; + + RSA *rsa = EVP_PKEY_get1_RSA(signKey); + std::string signId = getRsaKeyId(rsa); + rsa_sanity_check(rsa); + RSA_free(rsa); + + item->sign.TlvClear(); + + /* write out the item for signing */ + uint32_t len = serialiser.size(item); + char *buf = new char[len]; + if (!serialiser.serialise(item, buf, &len)) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::signTag() Failed serialise TagItem:"; + std::cerr << std::endl; + item->print(std::cerr); +#endif // DEBUG_RECOGN + delete []buf; + return false; + } + + /* calc and check signature */ + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + + EVP_SignInit(mdctx, EVP_sha1()); + EVP_SignUpdate(mdctx, buf, len); + + unsigned int siglen = EVP_PKEY_size(signKey); + unsigned char sigbuf[siglen]; + EVP_SignFinal(mdctx, sigbuf, &siglen, signKey); + + /* save signature */ + item->sign.signData.setBinData(sigbuf, siglen); + item->sign.keyId = signId; + + /* clean up */ + EVP_MD_CTX_destroy(mdctx); + delete []buf; + + return true; +} + +bool RsRecogn::signSigner(EVP_PKEY *signKey, RsGxsRecognSignerItem *item) +{ + std::cerr << "RsRecogn::signSigner()"; + std::cerr << std::endl; + + RsGxsRecognSerialiser serialiser; + + std::cerr << "RsRecogn::signSigner() Checking Key"; + std::cerr << std::endl; + + RSA *rsa = EVP_PKEY_get1_RSA(signKey); + std::string signId = getRsaKeyId(rsa); + rsa_sanity_check(rsa); + RSA_free(rsa); + + std::cerr << "RsRecogn::signSigner() Key Okay"; + std::cerr << std::endl; + + item->sign.TlvClear(); + + /* write out the item for signing */ + uint32_t len = serialiser.size(item); + char *buf = new char[len]; + if (!serialiser.serialise(item, buf, &len)) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::signSigner() Failed serialise SignerItem:"; + std::cerr << std::endl; + item->print(std::cerr); +#endif // DEBUG_RECOGN + delete []buf; + return false; + } + + /* calc and check signature */ + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + + EVP_SignInit(mdctx, EVP_sha1()); + EVP_SignUpdate(mdctx, buf, len); + + unsigned int siglen = EVP_PKEY_size(signKey); + unsigned char sigbuf[siglen]; + EVP_SignFinal(mdctx, sigbuf, &siglen, signKey); + + /* save signature */ + item->sign.signData.setBinData(sigbuf, siglen); + item->sign.keyId = signId; + + /* clean up */ + EVP_MD_CTX_destroy(mdctx); + delete []buf; + + return true; +} + + +bool RsRecogn::signTagRequest(EVP_PKEY *signKey, RsGxsRecognReqItem *item) +{ + std::cerr << "RsRecogn::signTagRequest()"; + std::cerr << std::endl; + + RsGxsRecognSerialiser serialiser; + + RSA *rsa = EVP_PKEY_get1_RSA(signKey); + std::string signId = getRsaKeyId(rsa); + rsa_sanity_check(rsa); + RSA_free(rsa); + + item->sign.TlvClear(); + + /* write out the item for signing */ + uint32_t len = serialiser.size(item); + char *buf = new char[len]; + if (!serialiser.serialise(item, buf, &len)) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::signTagRequest() Failed serialise Tag Request:"; + std::cerr << std::endl; + item->print(std::cerr); +#endif // DEBUG_RECOGN + delete []buf; + return false; + } + + /* calc and check signature */ + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + + EVP_SignInit(mdctx, EVP_sha1()); + EVP_SignUpdate(mdctx, buf, len); + + unsigned int siglen = EVP_PKEY_size(signKey); + unsigned char sigbuf[siglen]; + EVP_SignFinal(mdctx, sigbuf, &siglen, signKey); + + /* save signature */ + item->sign.signData.setBinData(sigbuf, siglen); + item->sign.keyId = signId; + + /* clean up */ + EVP_MD_CTX_destroy(mdctx); + delete []buf; + + return true; +} + + +bool RsRecogn::itemToRadix64(RsItem *item, std::string &radstr) +{ + RsGxsRecognSerialiser serialiser; + + /* write out the item for signing */ + uint32_t len = serialiser.size(item); + char *buf = new char[len]; + if (!serialiser.serialise(item, buf, &len)) + { + return false; + } + + radstr.clear(); + Radix64::encode(buf, len, radstr); + + return true; +} + + +std::string RsRecogn::getRsaKeyId(RSA *pubkey) +{ + int len = BN_num_bytes(pubkey -> n); + unsigned char tmp[len]; + BN_bn2bin(pubkey -> n, tmp); + + // copy first CERTSIGNLEN bytes... + if (len > CERTSIGNLEN) + { + len = CERTSIGNLEN; + } + + std::string id; + for(uint32_t i = 0; i < CERTSIGNLEN; i++) + { + rs_sprintf_append(id, "%02x", (uint16_t) (((uint8_t *) (tmp))[i])); + } + + return id; +} + + + +RsGxsRecognTagItem *RsRecogn::extractTag(const std::string &encoded) +{ + // Decode from Radix64 encoded Packet. + size_t buflen; + char *buffer; + uint32_t pktsize; + + Radix64::decode(encoded, buffer, buflen); + pktsize = buflen; + + RsGxsRecognSerialiser serialiser; + RsItem *item = serialiser.deserialise(buffer, &pktsize); + delete []buffer; + + if (!item) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::extractTag() ERROR Deserialise failed"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return NULL; + } + + RsGxsRecognTagItem *tagitem = dynamic_cast(item); + + if (!tagitem) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::extractTag() ERROR Not TagItem, is: "; + std::cerr << std::endl; + item->print(std::cerr); +#endif // DEBUG_RECOGN + delete item; + } + + return tagitem; +} + + +bool RsRecogn::createTagRequest(const RsTlvSecurityKey &key, const std::string &id, const std::string &nickname, uint16_t tag_class, uint16_t tag_type, const std::string &comment, std::string &tag) +{ + RsGxsRecognReqItem *item = new RsGxsRecognReqItem(); + + EVP_PKEY *signKey = EVP_PKEY_new(); + RSA *rsakey = GxsSecurity::extractPrivateKey(key); + if (!rsakey) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::createTagRequest() Failed to extract key"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return false; + } + + if (!EVP_PKEY_assign_RSA(signKey, rsakey)) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::createTagRequest() Failed to assign key"; + std::cerr << std::endl; +#endif // DEBUG_RECOGN + return false; + } + + item->issued_at = time(NULL); + item->period = 365 * 24 * 3600; + item->tag_class = tag_class; + item->tag_type = tag_type; + + item->nickname = nickname; + item->identity = id; + item->comment = comment; + + bool signOk = RsRecogn::signTagRequest(signKey,item); + EVP_PKEY_free(signKey); + + if (!signOk) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::createTagRequest() Failed to sign Tag Request:"; + std::cerr << std::endl; + item->print(std::cerr); +#endif // DEBUG_RECOGN + delete item; + return false; + } + + /* write out the item for signing */ + RsGxsRecognSerialiser serialiser; + uint32_t len = serialiser.size(item); + char *buf = new char[len]; + bool serOk = serialiser.serialise(item, buf, &len); + delete item; + + if (serOk) + { + Radix64::encode(buf, len, tag); + } + + delete []buf; + + if (!serOk) + { +#ifdef DEBUG_RECOGN + std::cerr << "RsRecogn::createTagRequest() Failed serialise Tag Request:"; + std::cerr << std::endl; + item->print(std::cerr); +#endif // DEBUG_RECOGN + return false; + } + + return serOk; +} + + diff --git a/libretroshare/src/util/rsrecogn.h b/libretroshare/src/util/rsrecogn.h new file mode 100644 index 000000000..a8df7f25e --- /dev/null +++ b/libretroshare/src/util/rsrecogn.h @@ -0,0 +1,61 @@ + +/* + * libretroshare/src/util: rsrecogn.h + * + * RetroShare Utilities + * + * Copyright 2013 by Robert Fernie. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2.1 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + + +#ifndef RSUTIL_RECOGN_H +#define RSUTIL_RECOGN_H + +#include +#include +#include +#include + +#include "serialiser/rsgxsrecognitems.h" + +namespace RsRecogn { + +EVP_PKEY * loadMasterKey(); +bool loadSigningKeys(std::map &signMap); +bool validateTagSignature(RsGxsRecognSignerItem *signer, RsGxsRecognTagItem *item); + +bool signTag(EVP_PKEY *signKey, RsGxsRecognTagItem *item); +bool signSigner(EVP_PKEY *signKey, RsGxsRecognSignerItem *item); +bool signTagRequest(EVP_PKEY *signKey, RsGxsRecognReqItem *item); + +bool itemToRadix64(RsItem *item, std::string &radstr); + +std::string getRsaKeyId(RSA *pubkey); + +RsGxsRecognTagItem *extractTag(const std::string &encoded); + +bool createTagRequest(const RsTlvSecurityKey &key, + const std::string &id, const std::string &nickname, + uint16_t tag_class, uint16_t tag_type, + const std::string &comment, std::string &tag); + +} + +#endif From 2ce92da8ad05c010787c0adf1c4f9004c03d1c7c Mon Sep 17 00:00:00 2001 From: drbob Date: Sun, 20 Oct 2013 09:56:12 +0000 Subject: [PATCH 11/29] GUI modifications to incorporate RecognTags. - Enable EditDialog for own Ids. - Addition, parsing and removal of Tags in IdEditDialog. - Display of Tags (only text) in GxsId Label/Chooser/TreewidgetItem .. TODO: - Add icon sets for RecognTags. - display of Tags as Icons. git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6855 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- retroshare-gui/src/gui/Identity/IdDialog.cpp | 3 +- .../src/gui/Identity/IdEditDialog.cpp | 287 ++++++++++++--- .../src/gui/Identity/IdEditDialog.h | 20 +- .../src/gui/Identity/IdEditDialog.ui | 332 +++++++++++------- retroshare-gui/src/gui/gxs/GxsIdChooser.cpp | 11 + retroshare-gui/src/gui/gxs/GxsIdLabel.cpp | 10 + .../src/gui/gxs/GxsIdTreeWidgetItem.cpp | 11 + retroshare-gui/src/retroshare-gui.pro | 7 +- 8 files changed, 507 insertions(+), 174 deletions(-) diff --git a/retroshare-gui/src/gui/Identity/IdDialog.cpp b/retroshare-gui/src/gui/Identity/IdDialog.cpp index 541615d9e..f1702ac11 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDialog.cpp @@ -473,8 +473,7 @@ void IdDialog::insertIdDetails(uint32_t token) mStateHelper->setWidgetEnabled(ui.pushButton_Reputation, false); // No Delete Ids yet! mStateHelper->setWidgetEnabled(ui.pushButton_Delete, /*true*/ false); - // No Editing Ids yet! - mStateHelper->setWidgetEnabled(ui.pushButton_EditId, /*true*/ false); + mStateHelper->setWidgetEnabled(ui.pushButton_EditId, true); } else { diff --git a/retroshare-gui/src/gui/Identity/IdEditDialog.cpp b/retroshare-gui/src/gui/Identity/IdEditDialog.cpp index de204ae84..9593746e4 100644 --- a/retroshare-gui/src/gui/Identity/IdEditDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdEditDialog.cpp @@ -61,10 +61,19 @@ IdEditDialog::IdEditDialog(QWidget *parent) /* Connect signals */ connect(ui.radioButton_GpgId, SIGNAL(toggled(bool)), this, SLOT(idTypeToggled(bool))); connect(ui.radioButton_Pseudo, SIGNAL(toggled(bool)), this, SLOT(idTypeToggled(bool))); - connect(ui.buttonBox, SIGNAL(accepted()), this, SLOT(updateId())); + connect(ui.buttonBox, SIGNAL(accepted()), this, SLOT(submit())); connect(ui.buttonBox, SIGNAL(rejected()), this, SLOT(close())); + connect(ui.plainTextEdit_Tag, SIGNAL(textChanged()), this, SLOT(checkNewTag())); + connect(ui.pushButton_Tag, SIGNAL(clicked(bool)), this, SLOT(addRecognTag())); + connect(ui.toolButton_Tag1, SIGNAL(clicked(bool)), this, SLOT(rmTag1())); + connect(ui.toolButton_Tag2, SIGNAL(clicked(bool)), this, SLOT(rmTag2())); + connect(ui.toolButton_Tag3, SIGNAL(clicked(bool)), this, SLOT(rmTag3())); + connect(ui.toolButton_Tag4, SIGNAL(clicked(bool)), this, SLOT(rmTag4())); + connect(ui.toolButton_Tag5, SIGNAL(clicked(bool)), this, SLOT(rmTag5())); + mIdQueue = new TokenQueue(rsIdentity->getTokenService(), this); + ui.pushButton_Tag->setEnabled(false); } void IdEditDialog::setupNewId(bool pseudo) @@ -89,6 +98,10 @@ void IdEditDialog::setupNewId(bool pseudo) // force - incase it wasn't triggered. idTypeToggled(true); + + ui.frame_Tags->setHidden(true); + ui.radioButton_GpgId->setEnabled(true); + ui.radioButton_Pseudo->setEnabled(true); } void IdEditDialog::idTypeToggled(bool checked) @@ -144,7 +157,6 @@ void IdEditDialog::loadExistingId(uint32_t token) mStateHelper->setLoading(IDEDITDIALOG_LOADID, false); /* get details from libretroshare */ - RsGxsIdGroup data; std::vector datavector; if (!rsIdentity->getGroupData(token, datavector)) { @@ -166,9 +178,9 @@ void IdEditDialog::loadExistingId(uint32_t token) return; } - data = datavector[0]; + mEditGroup = datavector[0]; - bool realid = (data.mMeta.mGroupFlags & RSGXSID_GROUPFLAG_REALID); + bool realid = (mEditGroup.mMeta.mGroupFlags & RSGXSID_GROUPFLAG_REALID); if (realid) { @@ -178,25 +190,28 @@ void IdEditDialog::loadExistingId(uint32_t token) { ui.radioButton_Pseudo->setChecked(true); } + // these are not editable for existing Id. + ui.radioButton_GpgId->setEnabled(false); + ui.radioButton_Pseudo->setEnabled(false); // DOES THIS TRIGGER ALREADY??? // force - incase it wasn't triggered. idTypeToggled(true); - ui.lineEdit_Nickname->setText(QString::fromUtf8(data.mMeta.mGroupName.c_str())); - ui.lineEdit_KeyId->setText(QString::fromStdString(data.mMeta.mGroupId)); + ui.lineEdit_Nickname->setText(QString::fromUtf8(mEditGroup.mMeta.mGroupName.c_str())); + ui.lineEdit_KeyId->setText(QString::fromStdString(mEditGroup.mMeta.mGroupId)); if (realid) { - ui.lineEdit_GpgHash->setText(QString::fromStdString(data.mPgpIdHash)); + ui.lineEdit_GpgHash->setText(QString::fromStdString(mEditGroup.mPgpIdHash)); - if (data.mPgpKnown) + if (mEditGroup.mPgpKnown) { RsPeerDetails details; - rsPeers->getGPGDetails(data.mPgpId, details); + rsPeers->getGPGDetails(mEditGroup.mPgpId, details); ui.lineEdit_GpgName->setText(QString::fromUtf8(details.name.c_str())); - ui.lineEdit_GpgId->setText(QString::fromStdString(data.mPgpId)); + ui.lineEdit_GpgId->setText(QString::fromStdString(mEditGroup.mPgpId)); } else { @@ -210,53 +225,209 @@ void IdEditDialog::loadExistingId(uint32_t token) ui.lineEdit_GpgId->setText(tr("N/A")); ui.lineEdit_GpgName->setText(tr("N/A")); } + + // RecognTags. + ui.frame_Tags->setHidden(false); + + loadRecognTags(); } -void IdEditDialog::updateId() +#define MAX_RECOGN_TAGS 5 + +void IdEditDialog::checkNewTag() { - RsGxsIdGroup rid; - // Must set, Nickname, KeyId(if existing), mIdType, GpgId. + std::string tag = ui.plainTextEdit_Tag->toPlainText().toStdString(); + std::string id = ui.lineEdit_KeyId->text().toStdString(); + std::string name = ui.lineEdit_Nickname->text().toUtf8().data(); - rid.mMeta.mGroupName = ui.lineEdit_Nickname->text().toUtf8().constData(); + QString desc; + bool ok = tagDetails(id, name, tag, desc); + ui.label_TagCheck->setText(desc); - if (rid.mMeta.mGroupName.size() < 2) + if (mEditGroup.mRecognTags.size() >= MAX_RECOGN_TAGS) { - std::cerr << "IdEditDialog::updateId() Nickname too short"; + ok = false; + } + + ui.pushButton_Tag->setEnabled(ok); +} + +void IdEditDialog::addRecognTag() +{ + std::string tag = ui.plainTextEdit_Tag->toPlainText().toStdString(); + if (mEditGroup.mRecognTags.size() >= MAX_RECOGN_TAGS) + { + std::cerr << "IdEditDialog::addRecognTag() Too many Tags, delete one first"; + std::cerr << std::endl; + } + + mEditGroup.mRecognTags.push_back(tag); + loadRecognTags(); +} + +void IdEditDialog::rmTag1() +{ + rmTag(0); +} + +void IdEditDialog::rmTag2() +{ + rmTag(1); +} + +void IdEditDialog::rmTag3() +{ + rmTag(2); +} + +void IdEditDialog::rmTag4() +{ + rmTag(3); +} + +void IdEditDialog::rmTag5() +{ + rmTag(4); +} + +void IdEditDialog::rmTag(int idx) +{ + std::list::iterator it; + int i = 0; + for(it = mEditGroup.mRecognTags.begin(); it != mEditGroup.mRecognTags.end() && (idx < i); it++, i++) ; + + if (it != mEditGroup.mRecognTags.end()) + { + mEditGroup.mRecognTags.erase(it); + } + loadRecognTags(); +} + +bool IdEditDialog::tagDetails(const std::string &id, const std::string &name, const std::string &tag, QString &desc) +{ + if (tag.empty()) + { + desc += "Empty Tag"; + return false; + } + + /* extract details for each tag */ + RsRecognTagDetails tagDetails; + + bool ok = false; + if (rsIdentity->parseRecognTag(id, name, tag, tagDetails)) + { + desc += QString::number(tagDetails.tag_class); + desc += ":"; + desc += QString::number(tagDetails.tag_type); + + if (tagDetails.is_valid) + { + ok = true; + desc += " Valid"; + } + else + { + desc += " Invalid"; + } + + if (tagDetails.is_pending) + { + ok = true; + desc += " Pending"; + } + } + else + { + desc += "Unparseable"; + } + return ok; +} + + +void IdEditDialog::loadRecognTags() +{ + std::cerr << "IdEditDialog::loadRecognTags()"; + std::cerr << std::endl; + + // delete existing items. + ui.label_Tag1->setHidden(true); + ui.label_Tag2->setHidden(true); + ui.label_Tag3->setHidden(true); + ui.label_Tag4->setHidden(true); + ui.label_Tag5->setHidden(true); + ui.toolButton_Tag1->setHidden(true); + ui.toolButton_Tag2->setHidden(true); + ui.toolButton_Tag3->setHidden(true); + ui.toolButton_Tag4->setHidden(true); + ui.toolButton_Tag5->setHidden(true); + ui.plainTextEdit_Tag->setPlainText(""); + + int i = 0; + std::list::const_iterator it; + for(it = mEditGroup.mRecognTags.begin(); it != mEditGroup.mRecognTags.end(); it++, i++) + { + QString recognTag; + tagDetails(mEditGroup.mMeta.mGroupId, mEditGroup.mMeta.mGroupName, *it, recognTag); + + switch(i) + { + default: + case 0: + ui.label_Tag1->setText(recognTag); + ui.label_Tag1->setHidden(false); + ui.toolButton_Tag1->setHidden(false); + break; + case 1: + ui.label_Tag2->setText(recognTag); + ui.label_Tag2->setHidden(false); + ui.toolButton_Tag2->setHidden(false); + break; + case 2: + ui.label_Tag3->setText(recognTag); + ui.label_Tag3->setHidden(false); + ui.toolButton_Tag3->setHidden(false); + break; + case 3: + ui.label_Tag4->setText(recognTag); + ui.label_Tag4->setHidden(false); + ui.toolButton_Tag4->setHidden(false); + break; + case 4: + ui.label_Tag5->setText(recognTag); + ui.label_Tag5->setHidden(false); + ui.toolButton_Tag5->setHidden(false); + break; + } + } +} + +void IdEditDialog::submit() +{ + if (mIsNew) + { + createId(); + } + else + { + updateId(); + } +} + + +void IdEditDialog::createId() +{ + std::string groupname = ui.lineEdit_Nickname->text().toUtf8().constData(); + + if (groupname.size() < 2) + { + std::cerr << "IdEditDialog::createId() Nickname too short"; std::cerr << std::endl; return; } - //rid.mIdType = RSID_RELATION_YOURSELF; - if (mIsNew) - { - rid.mMeta.mGroupId = ""; - } - else - { - rid.mMeta.mGroupId = ui.lineEdit_KeyId->text().toStdString(); - } - - if (ui.radioButton_GpgId->isChecked()) - { - //rid.mIdType |= RSID_TYPE_REALID; - - //rid.mGpgId = ui.lineEdit_GpgId->text().toStdString(); - rid.mPgpIdHash = ui.lineEdit_GpgHash->text().toStdString(); - //rid.mGpgName = ui.lineEdit_GpgName->text().toUtf8().constData(); - } - else - { - //rid.mIdType |= RSID_TYPE_PSEUDONYM; - - //rid.mGpgId = ""; - rid.mPgpIdHash = ""; - //rid.mGpgName = ""; - //rid.mGpgEmail = ""; - } - - // Can only create Identities for the moment! RsIdentityParameters params; - params.nickname = rid.mMeta.mGroupName; + params.nickname = groupname; params.isPgpLinked = (ui.radioButton_GpgId->isChecked()); uint32_t dummyToken = 0; @@ -265,6 +436,30 @@ void IdEditDialog::updateId() close(); } + +void IdEditDialog::updateId() +{ + /* submit updated details */ + std::string groupname = ui.lineEdit_Nickname->text().toUtf8().constData(); + + if (groupname.size() < 2) + { + std::cerr << "IdEditDialog::updateId() Nickname too short"; + std::cerr << std::endl; + return; + } + + mEditGroup.mMeta.mGroupName = groupname; + + uint32_t dummyToken = 0; + rsIdentity->updateIdentity(dummyToken, mEditGroup); + + close(); +} + + + + void IdEditDialog::loadRequest(const TokenQueue */*queue*/, const TokenRequest &req) { std::cerr << "IdDialog::loadRequest() UserType: " << req.mUserType; diff --git a/retroshare-gui/src/gui/Identity/IdEditDialog.h b/retroshare-gui/src/gui/Identity/IdEditDialog.h index 8ef672710..1703073dc 100644 --- a/retroshare-gui/src/gui/Identity/IdEditDialog.h +++ b/retroshare-gui/src/gui/Identity/IdEditDialog.h @@ -29,6 +29,7 @@ #include #include "util/TokenQueue.h" +#include class UIStateHelper; @@ -46,16 +47,33 @@ public: private slots: void idTypeToggled(bool checked); - void updateId(); + void submit(); + + void addRecognTag(); + void checkNewTag(); + void rmTag1(); + void rmTag2(); + void rmTag3(); + void rmTag4(); + void rmTag5(); private: + void createId(); + void updateId(); void updateIdType(bool pseudo); void loadExistingId(uint32_t token); + void loadRecognTags(); + // extract details. + bool tagDetails(const std::string &id, const std::string &name, const std::string &tag, QString &desc); + void rmTag(int idx); + protected: Ui::IdEditDialog ui; bool mIsNew; UIStateHelper *mStateHelper; + + RsGxsIdGroup mEditGroup; TokenQueue *mIdQueue; }; diff --git a/retroshare-gui/src/gui/Identity/IdEditDialog.ui b/retroshare-gui/src/gui/Identity/IdEditDialog.ui index 2a9dd713b..197ad6dc5 100644 --- a/retroshare-gui/src/gui/Identity/IdEditDialog.ui +++ b/retroshare-gui/src/gui/Identity/IdEditDialog.ui @@ -6,147 +6,233 @@ 0 0 - 557 - 179 + 510 + 595 - - - - - QFormLayout::AllNonFixedFieldsGrow + + + + + QFrame::StyledPanel - - - - Nickname - - - - - - - - - - Key ID - - - - - - - true - - - - - - - PGP Hash - - - - - - - true - - - - - - - PGP Id - - - - - - - true - - - - - - - PGP Name - - - - - - - true - - - - - + + QFrame::Raised + + + + + + + + PGP Associated ID + + + + + + + Pseudonym + + + + + + + + + Nickname + + + + + + + + + + Key ID + + + + + + + true + + + + + + + PGP Hash + + + + + + + true + + + + + + + PGP Id + + + + + + + true + + + + + + + PGP Name + + + + + + + true + + + + + + + + + + QFrame::StyledPanel + + + QFrame::Raised + + + + + + TextLabel + + + + + + + RM + + + + + + + TextLabel + + + + + + + RM + + + + + + + TextLabel + + + + + + + RM + + + + + + + TextLabel + + + + + + + RM + + + + + + + TextLabel + + + + + + + RM + + + + + + + + + + TextLabel + + + + + + + Add + + + + + + + + + + - Qt::Vertical + Qt::Horizontal - 20 - 40 + 328 + 20 - - - - - - - - - PGP Associated ID - - - - - - - Pseudonym - - - - - - - Qt::Vertical - - - - 20 - 40 - - - - - + + + QDialogButtonBox::Cancel|QDialogButtonBox::Ok + + - - - - QDialogButtonBox::Cancel|QDialogButtonBox::Ok - - - - lineEdit_Nickname - lineEdit_KeyId - lineEdit_GpgHash - lineEdit_GpgId - lineEdit_GpgName radioButton_GpgId radioButton_Pseudo buttonBox diff --git a/retroshare-gui/src/gui/gxs/GxsIdChooser.cpp b/retroshare-gui/src/gui/gxs/GxsIdChooser.cpp index 9abfaf25b..a77916079 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdChooser.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdChooser.cpp @@ -67,6 +67,17 @@ bool GxsIdChooser::MakeIdDesc(const RsGxsId &id, QString &desc) if (found) { desc = QString::fromUtf8(details.mNickname.c_str()); + + std::list::iterator it; + for(it = details.mRecognTags.begin(); it != details.mRecognTags.end(); it++) + { + desc += " ("; + desc += QString::number(it->tag_class); + desc += ":"; + desc += QString::number(it->tag_type); + desc += ")"; + } + if (details.mPgpLinked) { desc += " (PGP) ["; diff --git a/retroshare-gui/src/gui/gxs/GxsIdLabel.cpp b/retroshare-gui/src/gui/gxs/GxsIdLabel.cpp index bae9f798a..12373f212 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdLabel.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdLabel.cpp @@ -71,6 +71,16 @@ static bool MakeIdDesc(const RsGxsId &id, QString &str) str = QString::fromUtf8(details.mNickname.c_str()); + std::list::iterator it; + for(it = details.mRecognTags.begin(); it != details.mRecognTags.end(); it++) + { + str += " ("; + str += QString::number(it->tag_class); + str += ":"; + str += QString::number(it->tag_type); + str += ")"; + } + bool addCode = true; if (details.mPgpLinked) { diff --git a/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp b/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp index 4b0bc039b..fcba60a52 100644 --- a/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp +++ b/retroshare-gui/src/gui/gxs/GxsIdTreeWidgetItem.cpp @@ -45,6 +45,17 @@ static bool MakeIdDesc(const RsGxsId &id, QString &str) str = QString::fromUtf8(details.mNickname.c_str()); + std::list::iterator it; + for(it = details.mRecognTags.begin(); it != details.mRecognTags.end(); it++) + { + str += " ("; + str += QString::number(it->tag_class); + str += ":"; + str += QString::number(it->tag_type); + str += ")"; + } + + bool addCode = true; if (details.mPgpLinked) { diff --git a/retroshare-gui/src/retroshare-gui.pro b/retroshare-gui/src/retroshare-gui.pro index b4397b355..79b5d37cc 100644 --- a/retroshare-gui/src/retroshare-gui.pro +++ b/retroshare-gui/src/retroshare-gui.pro @@ -2,7 +2,7 @@ CONFIG += qt gui uic qrc resources uitools idle bitdht # Below is for GXS services. # Should be disabled for releases. -#CONFIG += gxs debug +CONFIG += gxs debug gxs { @@ -193,7 +193,10 @@ macx { gxs { LIBS += ../../supportlibs/pegmarkdown/lib/libpegmarkdown.a - LIBS += -lsqlite3 + LIBS += ../../../lib/libsqlcipher.a + #LIBS += ../../../lib/sqlcipher/.libs/libsqlcipher.a + #LIBS += -lsqlite3 + #LIBS += -lsqlcipher } From d31a34bd957cd33af562c1bd1814fc1f58912a1e Mon Sep 17 00:00:00 2001 From: drbob Date: Tue, 22 Oct 2013 03:42:20 +0000 Subject: [PATCH 12/29] hack for testing purposes. git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6869 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- retroshare-gui/src/gui/Identity/IdEditDialog.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/retroshare-gui/src/gui/Identity/IdEditDialog.cpp b/retroshare-gui/src/gui/Identity/IdEditDialog.cpp index 9593746e4..2518216d3 100644 --- a/retroshare-gui/src/gui/Identity/IdEditDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdEditDialog.cpp @@ -244,6 +244,13 @@ void IdEditDialog::checkNewTag() bool ok = tagDetails(id, name, tag, desc); ui.label_TagCheck->setText(desc); + // hack to allow add invalid tags (for testing). + if (!tag.empty()) + { + ok = true; + } + + if (mEditGroup.mRecognTags.size() >= MAX_RECOGN_TAGS) { ok = false; From a6f62caef455bd864c1d5b2bfb03f14d83319a8a Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Tue, 22 Oct 2013 19:56:01 +0000 Subject: [PATCH 13/29] -fixed an update issue with actual group item - simply didn't open grp file in "out" mode on update -made update a bit more sturdy is group not found on update - prevent client can crash librs - updated unit tests for grps and messages - removed data base removal on db reset as cannot rekey db unfortunately (as passphrase (ssl) cannot be kept in the clear in mem) git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6873 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsdataservice.cc | 67 ++++++++-------- libretroshare/src/gxs/rsgenexchange.cc | 16 +++- .../src/retroshare/rsgxsifacetypes.h | 1 + .../gxs/gen_exchange/genexchange_test.pro | 4 +- .../gxs/gen_exchange/genexchangetester.cpp | 79 ++++++++++--------- .../gxs/gen_exchange/genexchangetester.h | 2 +- .../gen_exchange/genexchangetestservice.cpp | 5 ++ .../gxs/gen_exchange/genexchangetestservice.h | 1 + .../gxs/gen_exchange/gxspublishgrouptest.cc | 69 +++++++++++++++- .../gxs/gen_exchange/gxspublishgrouptest.h | 1 + .../gxs/gen_exchange/gxspublishmsgtest.cc | 8 +- .../gxs/gen_exchange/rsgenexchange_test.cc | 10 +-- 12 files changed, 174 insertions(+), 89 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 9b7769ed8..c0f7082a7 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -663,7 +663,7 @@ int RsDataService::updateGroup(std::map &grp) if(!validSize(grpPtr)) continue; std::string grpFile = mServiceDir + "/" + grpPtr->grpId; - std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::trunc); + std::ofstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::trunc); uint32_t offset = 0; // get file offset /*! @@ -1058,34 +1058,33 @@ int RsDataService::retrieveGxsGrpMetaData(std::map::iterator mit = grp.begin(); - for(; mit != grp.end(); mit++) - { - const RsGxsGroupId& grpId = mit->first; - RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpMetaColumns, "grpId='" + grpId + "'", ""); + for(; mit != grp.end(); mit++) + { + const RsGxsGroupId& grpId = mit->first; + RetroCursor* c = mDb->sqlQuery(GRP_TABLE_NAME, grpMetaColumns, "grpId='" + grpId + "'", ""); - if(c) - { - bool valid = c->moveToFirst(); + if(c) + { + bool valid = c->moveToFirst(); - while(valid) - { - RsGxsGrpMetaData* g = locked_getGrpMeta(*c); + while(valid) + { + RsGxsGrpMetaData* g = locked_getGrpMeta(*c); - if(g) - { - grp[g->mGroupId] = g; - } - valid = c->moveToNext(); - } - delete c; - } + if(g) + { + grp[g->mGroupId] = g; + } + valid = c->moveToNext(); + } + delete c; + } - } + } - } + } return 1; @@ -1104,21 +1103,21 @@ int RsDataService::resetDataStore() std::map::iterator mit = grps.begin(); - - // remove all grp msgs files from service dir - for(; mit != grps.end(); mit++){ - std::string file = mServiceDir + "/" + mit->first; - std::string msgFile = file + "-msgs"; - remove(file.c_str()); // remove group file - remove(msgFile.c_str()); // and remove messages file - delete mit->second; - } { RsStackMutex stack(mDbMutex); - mDb->closeDb(); - } - remove(mDbName.c_str()); // remove db file + // remove all grp msgs files from service dir + for(; mit != grps.end(); mit++){ + std::string file = mServiceDir + "/" + mit->first; + std::string msgFile = file + "-msgs"; + remove(file.c_str()); // remove group file + remove(msgFile.c_str()); // and remove messages file + delete mit->second; + } + + mDb->execSQL("DROP TABLE " + MSG_TABLE_NAME); + mDb->execSQL("DROP TABLE " + GRP_TABLE_NAME); + } // recreate database initialise(); diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 54899fe0e..a41f948af 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1881,7 +1881,20 @@ void RsGenExchange::processGroupUpdatePublish() { GroupUpdatePublish& gup = *vit; const RsGxsGroupId& groupId = gup.grpItem->meta.mGroupId; - RsGxsGrpMetaData* meta = grpMeta[groupId]; + std::map::iterator mit = grpMeta.find(groupId); + + RsGxsGrpMetaData* meta = NULL; + if(mit == grpMeta.end()) + { + std::cerr << "Error! could not find meta of old group to update!" << std::endl; + mDataAccess->updatePublicRequestStatus(gup.mToken, RsTokenService::GXS_REQUEST_V2_STATUS_FAILED); + delete gup.grpItem; + continue; + }else + { + meta = mit->second; + } + gup.grpItem->meta = *meta; assignMetaUpdates(gup.grpItem->meta, gup.mUpdateMeta); @@ -1897,6 +1910,7 @@ void RsGenExchange::processGroupUpdatePublish() ggps.mStartTS = time(NULL); ggps.mLastAttemptTS = 0; ggps.mIsUpdate = true; + ggps.mToken = gup.mToken; mGrpsToPublish.push_back(ggps); }else { diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index 830df9f52..6f17d3cc5 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -46,6 +46,7 @@ public: mGroupStatus = 0; mCircleType = 0; + mAuthenFlags = 0; mPublishTs = 0; } diff --git a/libretroshare/src/tests/gxs/gen_exchange/genexchange_test.pro b/libretroshare/src/tests/gxs/gen_exchange/genexchange_test.pro index 2cc6fe806..aa67a28ed 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/genexchange_test.pro +++ b/libretroshare/src/tests/gxs/gen_exchange/genexchange_test.pro @@ -56,7 +56,9 @@ linux-* { LIBS += ../../../lib/libretroshare.a LIBS += ../../../../../libbitdht/src/lib/libbitdht.a LIBS += ../../../../../openpgpsdk/src/lib/libops.a - LIBS += -lssl -lgpgme -lupnp -lixml -lgnome-keyring -lsqlite3 -lbz2 + LIBS += -lssl -lgpgme -lupnp -lixml -lgnome-keyring -lbz2 + # We need a explicit path here, to force using the home version of sqlite3 that really encrypts the database. + LIBS += /home/crispy/Development/retroshare/sqlcipher/sqlcipher/.libs/libsqlite3.a LIBS *= -rdynamic -frtti DEFINES *= HAVE_XSS # for idle time, libx screensaver extensions DEFINES *= UBUNTU diff --git a/libretroshare/src/tests/gxs/gen_exchange/genexchangetester.cpp b/libretroshare/src/tests/gxs/gen_exchange/genexchangetester.cpp index 8eacbbc54..384eba6b3 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/genexchangetester.cpp +++ b/libretroshare/src/tests/gxs/gen_exchange/genexchangetester.cpp @@ -108,7 +108,7 @@ bool GenExchangeTest::pollForMsgAcknowledgement(uint32_t token, now = time(NULL); } return false; -} +} GenExchangeTestService* GenExchangeTest::getTestService() { @@ -155,12 +155,13 @@ void GenExchangeTest::setUp() // would be useful for genexchange services // to have a protected reset button mTestService->start(); -} +} void GenExchangeTest::breakDown() { - mTestService->join(); - clearAllData(); + mDataService->resetDataStore(); + mTestService->join(); + clearAllData(); } bool msgDataSort(const RsDummyMsg* m1, const RsDummyMsg* m2) @@ -189,7 +190,7 @@ bool GenExchangeTest::compareMsgDataMaps() } return ok; -} +} bool GenExchangeTest::compareMsgIdMaps() @@ -205,7 +206,7 @@ bool GenExchangeTest::compareMsgIdMaps() ok &= Comparison, RsGxsMessageId>::comparison(v1, v2); } return ok; -} +} bool GenExchangeTest::compareMsgMetaMaps() @@ -220,19 +221,19 @@ bool GenExchangeTest::compareMsgMetaMaps() ok &= Comparison, RsMsgMetaData>::comparison(v1, v2); } return ok; -} +} bool GenExchangeTest::compareMsgRelateIdsMap() { return false; -} +} bool GenExchangeTest::compareMsgRelatedDataMap() { return false; -} +} bool grpDataSort(const RsDummyGrp* g1, const RsDummyGrp* g2) { @@ -247,7 +248,7 @@ bool GenExchangeTest::compareGrpData() bool ok = Comparison, RsDummyGrp*>::comparison (mGrpDataIn, mGrpDataOut); return ok; -} +} bool operator<(const RsGroupMetaData& l, const RsGroupMetaData& r) { @@ -263,7 +264,7 @@ bool GenExchangeTest::compareGrpMeta() bool ok = Comparison, RsGroupMetaData>::comparison (mGrpMetaDataIn, mGrpMetaDataOut); return ok; -} +} bool GenExchangeTest::compareGrpIds() @@ -289,7 +290,7 @@ void GenExchangeTest::createGrps(uint32_t nGrps, pollForGrpAcknowledgement(token, grpId); groupId.push_back(grpId); } -} +} void GenExchangeTest::init(RsMsgMetaData& msgMetaData) const { @@ -334,7 +335,7 @@ void GenExchangeTest::init(RsDummyGrp& grpItem) const { randString(SHORT_STR, grpItem.grpData); init(grpItem.meta); -} +} void GenExchangeTest::init(RsDummyMsg& msgItem) const @@ -346,72 +347,72 @@ void GenExchangeTest::init(RsDummyMsg& msgItem) const void GenExchangeTest::storeToMsgDataOutMaps(const DummyMsgMap& msgDataOut) { mMsgDataOut.insert(msgDataOut.begin(), msgDataOut.end()); -} +} void GenExchangeTest::storeToMsgIdsOutMaps(const GxsMsgIdResult& msgIdsOut) { mMsgIdsOut.insert(msgIdsOut.begin(), msgIdsOut.end()); -} +} void GenExchangeTest::storeToMsgMetaOutMaps(const GxsMsgMetaMap& msgMetaOut) { mMsgMetaDataOut.insert(msgMetaOut.begin(), msgMetaOut.end()); -} +} void GenExchangeTest::storeToMsgDataInMaps(const DummyMsgMap& msgDataIn) { mMsgDataIn.insert(msgDataIn.begin(), msgDataIn.end()); -} +} void GenExchangeTest::storeToMsgIdsInMaps(const GxsMsgIdResult& msgIdsIn) { mMsgIdsIn.insert(msgIdsIn.begin(), msgIdsIn.end()); -} +} void GenExchangeTest::storeToMsgMetaInMaps(const GxsMsgMetaMap& msgMetaIn) { mMsgMetaDataIn.insert(msgMetaIn.begin(), msgMetaIn.end()); -} +} void GenExchangeTest::storeToGrpIdsOutList( const std::list& grpIdOut) { mGrpIdsOut.insert(mGrpIdsOut.end(), grpIdOut.begin(), grpIdOut.end()); -} +} void GenExchangeTest::storeToGrpMetaOutList( const std::list& grpMetaOut) { mGrpMetaDataOut.insert(mGrpMetaDataOut.end(), grpMetaOut.begin(), grpMetaOut.end()); -} +} void GenExchangeTest::storeToGrpDataOutList( const std::vector& grpDataOut) { mGrpDataOut.insert(mGrpDataOut.end(), grpDataOut.begin(), grpDataOut.end()); -} +} void GenExchangeTest::storeToGrpIdsInList( const std::list& grpIdIn) { mGrpIdsIn.insert(mGrpIdsIn.end(), grpIdIn.begin(), grpIdIn.end()); -} +} void GenExchangeTest::storeToGrpMetaInList( const std::list& grpMetaIn) { mGrpMetaDataIn.insert(mGrpMetaDataIn.end(), grpMetaIn.begin(), grpMetaIn.end()); -} +} void GenExchangeTest::storeToGrpDataInList( @@ -437,14 +438,14 @@ void GenExchangeTest::clearAllData() void GenExchangeTest::clearMsgDataInMap() { mMsgDataIn.clear(); -} +} void GenExchangeTest::clearMsgDataOutMap() { clearMsgDataMap(mMsgDataOut); -} +} void GenExchangeTest::clearMsgDataMap(DummyMsgMap& msgDataMap) const { @@ -459,31 +460,31 @@ void GenExchangeTest::clearMsgDataMap(DummyMsgMap& msgDataMap) const void GenExchangeTest::clearMsgMetaInMap() { mMsgMetaDataIn.clear(); -} +} void GenExchangeTest::clearMsgMetaOutMap() { mMsgMetaDataOut.clear(); -} +} void GenExchangeTest::clearMsgIdInMap() { mMsgIdsIn.clear(); -} +} void GenExchangeTest::clearMsgIdOutMap() { mMsgIdsOut.clear(); -} +} void GenExchangeTest::clearMsgRelatedIdInMap() { mMsgRelatedIdsIn.clear(); -} +} void GenExchangeTest::clearGrpDataInList() @@ -494,38 +495,38 @@ void GenExchangeTest::clearGrpDataInList() void GenExchangeTest::clearGrpDataList(std::vector& grpData) const { deleteResVector(grpData); -} +} void GenExchangeTest::clearGrpDataOutList() { clearGrpDataList(mGrpDataOut); -} +} void GenExchangeTest::clearGrpMetaInList() { mGrpMetaDataIn.clear(); -} +} void GenExchangeTest::clearGrpMetaOutList() { mGrpMetaDataOut.clear(); -} +} void GenExchangeTest::clearGrpIdInList() { mGrpIdsIn.clear(); -} +} void GenExchangeTest::clearGrpIdOutList() { mGrpIdsOut.clear(); -} - +} + bool operator ==(const RsMsgMetaData& lMeta, const RsMsgMetaData& rMeta) { @@ -584,5 +585,5 @@ bool operator ==(const RsDummyMsg& lMsg, const RsDummyMsg& rMsg) bool operator ==(const RsGxsGrpItem& lGrp, const RsGxsGrpItem& rGrp) { return false; -} +} diff --git a/libretroshare/src/tests/gxs/gen_exchange/genexchangetester.h b/libretroshare/src/tests/gxs/gen_exchange/genexchangetester.h index 882a846a2..13bb1cf6e 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/genexchangetester.h +++ b/libretroshare/src/tests/gxs/gen_exchange/genexchangetester.h @@ -258,7 +258,7 @@ private: std::list mGrpMetaDataOut, mGrpMetaDataIn; std::list mGrpIdsOut, mGrpIdsIn; - DummyMsgMap mMsgDataOut, mMsgDataIn; + std::map > mMsgDataOut, mMsgDataIn; GxsMsgMetaMap mMsgMetaDataOut, mMsgMetaDataIn; GxsMsgIdResult mMsgIdsOut, mMsgIdsIn; diff --git a/libretroshare/src/tests/gxs/gen_exchange/genexchangetestservice.cpp b/libretroshare/src/tests/gxs/gen_exchange/genexchangetestservice.cpp index f2b388e71..dd9ba328a 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/genexchangetestservice.cpp +++ b/libretroshare/src/tests/gxs/gen_exchange/genexchangetestservice.cpp @@ -17,6 +17,11 @@ void GenExchangeTestService::publishDummyGrp(uint32_t &token, RsDummyGrp *grp) publishGroup(token, grp); } +void GenExchangeTestService::updateDummyGrp(uint32_t &token, RsGxsGroupUpdateMeta &updateMeta, RsDummyGrp *group) +{ + updateGroup(token, updateMeta, group); +} + void GenExchangeTestService::publishDummyMsg(uint32_t &token, RsDummyMsg *msg) { publishMsg(token, msg); diff --git a/libretroshare/src/tests/gxs/gen_exchange/genexchangetestservice.h b/libretroshare/src/tests/gxs/gen_exchange/genexchangetestservice.h index d0d5e241e..588808eb9 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/genexchangetestservice.h +++ b/libretroshare/src/tests/gxs/gen_exchange/genexchangetestservice.h @@ -15,6 +15,7 @@ public: void notifyChanges(std::vector& changes); void publishDummyGrp(uint32_t& token, RsDummyGrp* grp); + void updateDummyGrp(uint32_t &token, RsGxsGroupUpdateMeta& meta, RsDummyGrp *group); void publishDummyMsg(uint32_t& token, RsDummyMsg* msg); /*! diff --git a/libretroshare/src/tests/gxs/gen_exchange/gxspublishgrouptest.cc b/libretroshare/src/tests/gxs/gen_exchange/gxspublishgrouptest.cc index 7f4555721..2fe0d5fac 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/gxspublishgrouptest.cc +++ b/libretroshare/src/tests/gxs/gen_exchange/gxspublishgrouptest.cc @@ -7,6 +7,7 @@ #include "gxspublishgrouptest.h" #include "util/utest.h" +#include "support.h" #define POLLING_TIME_OUT 5 @@ -169,6 +170,65 @@ bool GxsPublishGroupTest::testGrpIdRetrieval() return ok; } + +bool GxsPublishGroupTest::testUpdateGroup() +{ + setUp(); + + GenExchangeTestService* testService = getTestService(); + RsTokenService* tokenService = getTokenService(); + +// create some random grps to allow msg testing + + RsDummyGrp* dgrp1 = new RsDummyGrp(); + RsDummyGrp* dgrp2 = new RsDummyGrp(); + + RsDummyGrp* dgrp2_copy = new RsDummyGrp(); + + init(*dgrp1); + init(*dgrp2); + + RsTokReqOptions opts; + opts.mReqType = 45000; + uint32_t token; + RsGxsGroupId grpId; + + std::vector groupsPublished; + std::list grpIds; + + std::string name = dgrp1->meta.mGroupName; + *dgrp2 = *dgrp1; + testService->publishDummyGrp(token, dgrp1); + bool ok = pollForGrpAcknowledgement(token, grpId); + + grpIds.push_back(grpId); + RsGxsGroupUpdateMeta updateMeta(grpId); + + updateMeta.setMetaUpdate(RsGxsGroupUpdateMeta::NAME, name); + randString(SHORT_STR, dgrp2->grpData); + dgrp2->meta.mGroupId = grpId; + *dgrp2_copy = *dgrp2; + dgrp2->grpData ="ojfosfjsofjsof"; + testService->updateDummyGrp(token, updateMeta, dgrp2); + ok &= pollForGrpAcknowledgement(token, grpId); + + groupsPublished.push_back(dgrp2_copy); + + opts.mReqType = GXS_REQUEST_TYPE_GROUP_DATA; + + tokenService->requestGroupInfo(token, 0, opts, grpIds); + + pollForToken(token, opts, true); + + + ok &= compareGrpData(); + + breakDown(); + + return ok; + +} + bool GxsPublishGroupTest::testGrpMetaRetrieval() { @@ -229,10 +289,11 @@ bool GxsPublishGroupTest::testGrpMetaRetrieval() } void GxsPublishGroupTest::runTests() { - CHECK(testGrpSubmissionRetrieval()); - CHECK(testGrpIdRetrieval()); - CHECK(testGrpMetaRetrieval()); - CHECK(testSpecificGrpRetrieval()); +// CHECK(testGrpSubmissionRetrieval()); +// CHECK(testGrpIdRetrieval()); +// CHECK(testGrpMetaRetrieval()); + // CHECK(testSpecificGrpRetrieval()); + CHECK(testUpdateGroup()); } diff --git a/libretroshare/src/tests/gxs/gen_exchange/gxspublishgrouptest.h b/libretroshare/src/tests/gxs/gen_exchange/gxspublishgrouptest.h index 9d7f685bc..a0f642828 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/gxspublishgrouptest.h +++ b/libretroshare/src/tests/gxs/gen_exchange/gxspublishgrouptest.h @@ -26,6 +26,7 @@ private: bool testSpecificGrpRetrieval(); bool testGrpIdRetrieval(); bool testGrpMetaRetrieval(); + bool testUpdateGroup(); private: diff --git a/libretroshare/src/tests/gxs/gen_exchange/gxspublishmsgtest.cc b/libretroshare/src/tests/gxs/gen_exchange/gxspublishmsgtest.cc index 16c39c6fa..adfa2026c 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/gxspublishmsgtest.cc +++ b/libretroshare/src/tests/gxs/gen_exchange/gxspublishmsgtest.cc @@ -49,10 +49,10 @@ bool GxsPublishMsgTest::testMsgSubmissionRetrieval() msgOut->meta.mMsgId = msgId.second; DummyMsgMap msgMap; - std::vector msgV; - msgV.push_back(msgOut); - msgMap[msgOut->meta.mGroupId] = msgV; - storeToMsgDataOutMaps(msgMap); + std::vector msgV; + msgV.push_back(msgOut); + msgMap[msgOut->meta.mGroupId] = msgV; + storeToMsgDataOutMaps(msgMap); RsTokReqOptions opts; diff --git a/libretroshare/src/tests/gxs/gen_exchange/rsgenexchange_test.cc b/libretroshare/src/tests/gxs/gen_exchange/rsgenexchange_test.cc index 35afd2563..8b28bad17 100644 --- a/libretroshare/src/tests/gxs/gen_exchange/rsgenexchange_test.cc +++ b/libretroshare/src/tests/gxs/gen_exchange/rsgenexchange_test.cc @@ -30,16 +30,16 @@ INITTEST(); int main() { - RsGeneralDataService* dataStore = new RsDataService("./", "testServiceDb", RS_SERVICE_TYPE_DUMMY, NULL); + RsGeneralDataService* dataStore = new RsDataService("./", "testServiceDb", RS_SERVICE_TYPE_DUMMY, NULL, ""); // we want to use default authentication which is NO authentication :) GenExchangeTestService testService(dataStore, NULL, NULL); - //GxsPublishGroupTest testGrpPublishing(&testService, dataStore); - //testGrpPublishing.runTests(); + GxsPublishGroupTest testGrpPublishing(&testService, dataStore); + testGrpPublishing.runTests(); - GxsPublishMsgTest testMsgPublishing(&testService, dataStore); - testMsgPublishing.runTests(); + //GxsPublishMsgTest testMsgPublishing(&testService, dataStore); + //testMsgPublishing.runTests(); FINALREPORT("RsGenExchangeTest"); From f2884d580f110a9bfa18c8b959f0a97e866a4d98 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Tue, 29 Oct 2013 21:29:20 +0000 Subject: [PATCH 14/29] added author pull code -- problem, still need to figure out how to get peer auth req should go to git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6881 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgxsnetservice.cc | 39 +++++++++++++++++++++++ libretroshare/src/gxs/rsgxsnetservice.h | 8 +++-- libretroshare/src/gxs/rsnxs.h | 19 ++--------- libretroshare/src/services/p3idservice.cc | 30 ++++++++++++++++- libretroshare/src/services/p3idservice.h | 11 +++++++ 5 files changed, 87 insertions(+), 20 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index c1690e96a..537fb094c 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -766,6 +766,8 @@ void RsGxsNetService::run(){ // vetting of id and circle info runVetting(); + processExplicitGroupRequests(); + } } @@ -2063,3 +2065,40 @@ void RsGxsNetService::setSyncAge(uint32_t age) } +int RsGxsNetService::requestGrp(const std::list& grpId, const std::string& peerId) +{ + RsStackMutex stack(mNxsMutex); + mExplicitRequest[peerId].assign(grpId.begin(), grpId.end()); + return 1; +} + +void RsGxsNetService::processExplicitGroupRequests() +{ + RsStackMutex stack(mNxsMutex); + + std::map >::const_iterator cit = mExplicitRequest.begin(); + + for(; cit != mExplicitRequest.end(); cit++) + { + const std::string& peerId = cit->first; + const std::list& groupIdList = cit->second; + + std::list grpSyncItems; + std::list::const_iterator git = groupIdList.begin(); + uint32_t transN = locked_getTransactionId(); + for(; git != groupIdList.end(); git++) + { + RsNxsSyncGrpItem* item = new RsNxsSyncGrpItem(mServType); + item->grpId = *git; + item->PeerId(peerId); + item->flag = RsNxsSyncGrpItem::FLAG_REQUEST; + item->transactionNumber = transN; + grpSyncItems.push_back(item); + } + + if(!grpSyncItems.empty()) + locked_pushGrpTransactionFromList(grpSyncItems, peerId, transN); + } + + mExplicitRequest.clear(); +} diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 6c37d34a0..4cb16a2c3 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -115,7 +115,7 @@ public: * @param msgId the messages to retrieve * @return request token to be redeemed */ - int requestMsg(const std::string& msgId, uint8_t hops){ return 0;} + int requestMsg(const RsGxsGrpMsgIdPair& msgId){ return 0;} /*! * Request for this group is sent through to peers on your network @@ -123,7 +123,7 @@ public: * @param enabled set to false to disable pause, and true otherwise * @return request token to be redeemed */ - int requestGrp(const std::list& grpId, uint8_t hops){ return 0;} + int requestGrp(const std::list& grpId, const std::string& peerId); /* p3Config methods */ @@ -322,6 +322,8 @@ private: bool locked_canReceive(const RsGxsGrpMetaData * const grpMeta, const std::string& peerId); + void processExplicitGroupRequests(); + private: typedef std::vector GrpFragments; @@ -422,6 +424,8 @@ private: // need to be verfied std::vector mPendingResp; std::vector mPendingCircleVets; + + std::map > mExplicitRequest; }; #endif // RSGXSNETSERVICE_H diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 0c201a737..e909994a2 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -68,21 +68,6 @@ public: */ virtual void setSyncAge(uint32_t age) = 0; - /*! - * Explicitly requests all the groups contained by a peer - * Circumvents polling of peers for message - * @param peerId id of peer - */ - virtual void requestGroupsOfPeer(const std::string& peerId) = 0; - - /*! - * get messages of a peer for a given group id, this circumvents the normal - * polling of peers for messages of given group id - * @param peerId Id of peer - * @param grpId id of group to request messages for - */ - virtual void requestMessagesOfPeer(const std::string& peerId, const std::string& grpId) = 0; - /*! * Initiates a search through the network * This returns messages which contains the search terms set in RsGxsSearch @@ -116,7 +101,7 @@ public: * @param msgId the messages to retrieve * @return request token to be redeemed */ - virtual int requestMsg(const std::string& msgId, uint8_t hops) = 0; + virtual int requestMsg(const RsGxsGrpMsgIdPair& msgId) = 0; /*! * Request for this group is sent through to peers on your network @@ -124,7 +109,7 @@ public: * @param enabled set to false to disable pause, and true otherwise * @return request token to be redeemed */ - virtual int requestGrp(const std::list& grpId, uint8_t hops) = 0; + virtual int requestGrp(const std::list& grpId, const std::string& peerId) = 0; }; diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 5864ae32d..c63dd524b 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -140,7 +140,7 @@ p3IdService::p3IdService(RsGeneralDataService *gds, RsNetworkExchangeService *ne : RsGxsIdExchange(gds, nes, new RsGxsIdSerialiser(), RS_SERVICE_GXSV2_TYPE_GXSID, idAuthenPolicy()), RsIdentity(this), GxsTokenQueue(this), RsTickEvent(), mIdMtx("p3IdService"), mPublicKeyCache(DEFAULT_MEM_CACHE_SIZE, "GxsIdPublicKeyCache"), - mPrivateKeyCache(DEFAULT_MEM_CACHE_SIZE, "GxsIdPrivateKeyCache") + mPrivateKeyCache(DEFAULT_MEM_CACHE_SIZE, "GxsIdPrivateKeyCache"), mNes(nes) { mBgSchedule_Mode = 0; mBgSchedule_Active = false; @@ -1366,6 +1366,9 @@ bool p3IdService::cache_start_load() RsGenExchange::getTokenService()->requestGroupInfo(token, ansType, opts, groupIds); GxsTokenQueue::queueRequest(token, GXSIDREQ_CACHELOAD); + std::set groupIdSet; + groupIdSet.insert(groupIds.begin(), groupIds.end()); + mGroupsToCache.insert(std::make_pair(token, groupIdSet)); } return 1; } @@ -1401,10 +1404,29 @@ bool p3IdService::cache_load_for_token(uint32_t token) std::cerr << std::endl; #endif // DEBUG_IDS + + { + // remove identities that are present + RsStackMutex stack(mIdMtx); + mGroupsToCache[token].erase(item->meta.mGroupId); + } + /* cache the data */ cache_store(item); delete item; } + + { + // now store identities that aren't present + RsStackMutex stack(mIdMtx); + const std::set& groupIdSet = mGroupsToCache[token]; + + if(!groupIdSet.empty()) + mGroupNotPresent[token].assign(groupIdSet.begin(), groupIdSet.end()); + + mGroupsToCache.erase(token); + } + } else { @@ -3601,7 +3623,13 @@ std::ostream &operator<<(std::ostream &out, const RsGxsIdOpinion &opinion) +void p3IdService::checkPeerForIdentities() +{ + RsStackMutex stack(mIdMtx); + // crud, i needed peers instead! + mGroupNotPresent.clear(); +} // Overloaded from GxsTokenQueue for Request callbacks. diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index a42712ed4..6270ee9e4 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -358,6 +358,12 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); void loadRecognKeys(); + /************************************************************************ + * for getting identities that are not present + * + */ + void checkPeerForIdentities(); + /* MUTEX PROTECTED DATA (mIdMtx - maybe should use a 2nd?) */ bool checkRecognSignature_locked(std::string encoded, RSA &key, std::string signature); @@ -413,6 +419,11 @@ std::string genRandomId(int len = 20); std::vector mGroupChange; std::vector mMsgChange; + private: + + std::map > mGroupsToCache; + std::map > mGroupNotPresent; + RsNetworkExchangeService* mNes; }; #endif // P3_IDENTITY_SERVICE_HEADER From 812ddef40e5e95b8f2d35c771d4dcb3a2885084e Mon Sep 17 00:00:00 2001 From: drbob Date: Thu, 31 Oct 2013 23:39:05 +0000 Subject: [PATCH 15/29] added missing mutex lock. git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6885 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/services/p3idservice.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index c63dd524b..d86a4fe23 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -1368,6 +1368,8 @@ bool p3IdService::cache_start_load() GxsTokenQueue::queueRequest(token, GXSIDREQ_CACHELOAD); std::set groupIdSet; groupIdSet.insert(groupIds.begin(), groupIds.end()); + + RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ mGroupsToCache.insert(std::make_pair(token, groupIdSet)); } return 1; From 67c55991d7b418a1938c57f413f448278d9920fc Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 3 Nov 2013 23:46:34 +0000 Subject: [PATCH 16/29] decided to continue propagating peer details in - gxsnet now has a ctor option to disable auto group sync git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6888 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgenexchange.cc | 2 + libretroshare/src/gxs/rsgxsnetservice.cc | 24 ++++--- libretroshare/src/gxs/rsgxsnetservice.h | 3 +- libretroshare/src/rsserver/rsinit.cc | 4 +- libretroshare/src/services/p3idservice.cc | 87 ++++++++++++++++++----- libretroshare/src/services/p3idservice.h | 11 ++- 6 files changed, 96 insertions(+), 35 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index a41f948af..ffa2a3000 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -820,6 +820,7 @@ int RsGenExchange::validateMsg(RsNxsMsg *msg, const uint32_t& grpFlag, RsTlvSecu }else { std::list peers; + peers.push_back(msg->PeerId()); mGixs->requestKey(metaData.mAuthorId, peers); return VALIDATE_FAIL_TRY_LATER; } @@ -893,6 +894,7 @@ int RsGenExchange::validateGrp(RsNxsGrp* grp, RsTlvSecurityKeySet& grpKeySet) }else { std::list peers; + peers.push_back(grp->PeerId()); mGixs->requestKey(metaData.mAuthorId, peers); return VALIDATE_FAIL_TRY_LATER; } diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 537fb094c..afab99ba0 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -36,16 +36,16 @@ #define GIXS_CUT_OFF 0 #define SYNC_PERIOD 12 // in microseconds every 10 seconds (1 second for testing) -#define TRANSAC_TIMEOUT 5 // 5 seconds +#define TRANSAC_TIMEOUT 10 // 10 seconds const uint32_t RsGxsNetService::FRAGMENT_SIZE = 150000; RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, - RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, RsGixsReputation* reputations, RsGcxs* circles) + RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, RsGixsReputation* reputations, RsGcxs* circles, bool grpAutoSync) : p3Config(servType), p3ThreadedService(servType), mTransactionTimeOut(TRANSAC_TIMEOUT), mServType(servType), mDataStore(gds), mTransactionN(0), mObserver(nxsObs), mNxsMutex("RsGxsNetService"), mNetMgr(netMgr), mSYNC_PERIOD(SYNC_PERIOD), - mSyncTs(0), mReputations(reputations), mCircles(circles) + mSyncTs(0), mReputations(reputations), mCircles(circles), mGrpAutoSync(grpAutoSync) { addSerialType(new RsNxsSerialiser(mServType)); @@ -85,13 +85,16 @@ void RsGxsNetService::syncWithPeers() std::set::iterator sit = peers.begin(); - // for now just grps - for(; sit != peers.end(); sit++) + if(mGrpAutoSync) { - RsNxsSyncGrp *grp = new RsNxsSyncGrp(mServType); - grp->clear(); - grp->PeerId(*sit); - sendItem(grp); + // for now just grps + for(; sit != peers.end(); sit++) + { + RsNxsSyncGrp *grp = new RsNxsSyncGrp(mServType); + grp->clear(); + grp->PeerId(*sit); + sendItem(grp); + } } #ifdef GXS_ENABLE_SYNC_MSGS @@ -773,8 +776,7 @@ void RsGxsNetService::run(){ bool RsGxsNetService::locked_checkTransacTimedOut(NxsTransaction* tr) { - //return tr->mTimeOut < ((uint32_t) time(NULL)); - return false; + return tr->mTimeOut < ((uint32_t) time(NULL)); } void RsGxsNetService::processTransactions(){ diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 4cb16a2c3..33d1c9d50 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -73,7 +73,7 @@ public: * arrive */ RsGxsNetService(uint16_t servType, RsGeneralDataService* gds, RsNxsNetMgr* netMgr, - RsNxsObserver* nxsObs = NULL, RsGixsReputation* repuations = NULL, RsGcxs* circles = NULL); + RsNxsObserver* nxsObs = NULL, RsGixsReputation* repuations = NULL, RsGcxs* circles = NULL, bool grpAutoSync = true); virtual ~RsGxsNetService(); @@ -420,6 +420,7 @@ private: RsGcxs* mCircles; RsGixsReputation* mReputations; + bool mGrpAutoSync; // need to be verfied std::vector mPendingResp; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 37e9598f9..e4f58908f 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -2268,7 +2268,8 @@ int RsServer::StartupRetroShare() // create GXS photo service RsGxsNetService* gxsid_ns = new RsGxsNetService( RS_SERVICE_GXSV2_TYPE_GXSID, gxsid_ds, nxsMgr, - mGxsIdService, mGxsIdService, mGxsCircles); + mGxsIdService, mGxsIdService, mGxsCircles, + false); // don't synchronise group automatic (need explicit group request) /**** GxsCircle service ****/ @@ -2653,7 +2654,6 @@ int RsServer::StartupRetroShare() createThread(*gxsforums_ns); createThread(*gxschannels_ns); - #endif // RS_ENABLE_GXS ftserver->StartupThreads(); diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index d86a4fe23..f358cf561 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -112,6 +112,7 @@ RsIdentity *rsIdentity = NULL; #define GXSID_EVENT_DUMMY_PGPID 0x2002 #define GXSID_EVENT_DUMMY_UNKNOWN_PGPID 0x2003 #define GXSID_EVENT_DUMMY_PSEUDOID 0x2004 +#define GXSID_EVENT_REQUEST_IDS 0x2005 /* delays */ @@ -411,9 +412,26 @@ bool p3IdService::requestKey(const RsGxsId &id, const std::list &peers) { if (haveKey(id)) return true; + else + { + if(isPendingNetworkRequest(id)) + return true; + } + + return cache_request_load(id); } +bool p3IdService::isPendingNetworkRequest(const RsGxsId& gxsId) const +{ + // if ids has beens confirmed as not physically present return + // immediately, id will be removed from list if found by auto nxs net search + if(mIdsNotPresent.find(gxsId) != mIdsNotPresent.end()) + return true; + + return false; +} + int p3IdService::getKey(const RsGxsId &id, RsTlvSecurityKey &key) { RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ @@ -1298,7 +1316,7 @@ bool p3IdService::cache_store(const RsGxsIdGroupItem *item) #define MIN_CYCLE_GAP 2 -bool p3IdService::cache_request_load(const RsGxsId &id) +bool p3IdService::cache_request_load(const RsGxsId &id, const std::list& peers) { #ifdef DEBUG_IDS std::cerr << "p3IdService::cache_request_load(" << id << ")"; @@ -1307,7 +1325,7 @@ bool p3IdService::cache_request_load(const RsGxsId &id) { RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ - mCacheLoad_ToCache.push_back(id); + mCacheLoad_ToCache.insert(std::make_pair(id, peers)); } if (RsTickEvent::event_count(GXSID_EVENT_CACHELOAD) > 0) @@ -1339,16 +1357,17 @@ bool p3IdService::cache_start_load() RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ /* now we process the modGroupList -> a map so we can use it easily later, and create id list too */ - std::list::iterator it; + std::map >::iterator it; for(it = mCacheLoad_ToCache.begin(); it != mCacheLoad_ToCache.end(); it++) { #ifdef DEBUG_IDS - std::cerr << "p3IdService::cache_start_load() GroupId: " << *it; + std::cerr << "p3IdService::cache_start_load() GroupId: " << it->first; std::cerr << std::endl; #endif // DEBUG_IDS - groupIds.push_back(*it); // might need conversion? + groupIds.push_back(it->first); // might need conversion? } + // mPendingCache.insert(mCacheLoad_ToCache.begin(), mCacheLoad_ToCache.end()); mCacheLoad_ToCache.clear(); } @@ -1365,12 +1384,7 @@ bool p3IdService::cache_start_load() uint32_t token = 0; RsGenExchange::getTokenService()->requestGroupInfo(token, ansType, opts, groupIds); - GxsTokenQueue::queueRequest(token, GXSIDREQ_CACHELOAD); - std::set groupIdSet; - groupIdSet.insert(groupIds.begin(), groupIds.end()); - - RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ - mGroupsToCache.insert(std::make_pair(token, groupIdSet)); + GxsTokenQueue::queueRequest(token, GXSIDREQ_CACHELOAD); } return 1; } @@ -1410,7 +1424,7 @@ bool p3IdService::cache_load_for_token(uint32_t token) { // remove identities that are present RsStackMutex stack(mIdMtx); - mGroupsToCache[token].erase(item->meta.mGroupId); + mPendingCache.erase(item->meta.mGroupId); } /* cache the data */ @@ -1420,13 +1434,13 @@ bool p3IdService::cache_load_for_token(uint32_t token) { // now store identities that aren't present + RsStackMutex stack(mIdMtx); - const std::set& groupIdSet = mGroupsToCache[token]; + mIdsNotPresent.insert(mPendingCache.begin(), mPendingCache.end()); + mPendingCache.clear(); - if(!groupIdSet.empty()) - mGroupNotPresent[token].assign(groupIdSet.begin(), groupIdSet.end()); - - mGroupsToCache.erase(token); + if(!mIdsNotPresent.empty()) + schedule_now(GXSID_EVENT_REQUEST_IDS); } } @@ -1440,6 +1454,39 @@ bool p3IdService::cache_load_for_token(uint32_t token) return true; } +void p3IdService::requestIdsFromNet() +{ + RsStackMutex stack(mIdMtx); + + std::map >::const_iterator cit; + std::map > requests; + + // transform to appropriate structure ( > map) to make request to nes + for(cit = mIdsNotPresent.begin(); cit != mIdsNotPresent.end(); cit++) + { + { +#ifdef DEBUG_IDS + std::cerr << "p3IdService::requestIdsFromNet() Id not found, deferring for net request: "; + std::cerr << cit->first; + std::cerr << std::endl; +#endif // DEBUG_IDS + + } + + const std::list& peers = cit->second; + std::list::const_iterator cit2; + for(cit2 = peers.begin(); cit2 != peers.end(); cit2++) + requests[*cit2].push_back(cit->first); + } + + std::map >::const_iterator cit2; + + for(cit2 = requests.begin(); cit2 != requests.end(); cit2++) + mNes->requestGrp(cit2->second, cit2->first); + + mIdsNotPresent.clear(); +} + bool p3IdService::cache_update_if_cached(const RsGxsId &id, std::string serviceString) { /* if these entries are cached - update with new info */ @@ -2483,7 +2530,7 @@ bool p3IdService::recogn_process() bool isDone = false; { RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ - if (!mRecognGroupsToProcess.empty()) + if (!mRecognGroupsToProcess.empty() && !mGroupsToProcess.empty()) { item = mRecognGroupsToProcess.front(); mGroupsToProcess.pop_front(); @@ -3735,6 +3782,10 @@ void p3IdService::handle_event(uint32_t event_type, const std::string &elabel) case GXSID_EVENT_DUMMY_PSEUDOID: generateDummy_UnknownPseudo(); break; + case GXSID_EVENT_REQUEST_IDS: + requestIdsFromNet(); + break; + default: /* error */ diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index 6270ee9e4..f15a0230d 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -284,16 +284,20 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); */ int cache_tick(); - bool cache_request_load(const RsGxsId &id); + bool cache_request_load(const RsGxsId &id, const std::list& peers = std::list()); bool cache_start_load(); bool cache_load_for_token(uint32_t token); bool cache_store(const RsGxsIdGroupItem *item); bool cache_update_if_cached(const RsGxsId &id, std::string serviceString); + bool isPendingNetworkRequest(const RsGxsId& gxsId) const; + void requestIdsFromNet(); + // Mutex protected. - std::list mCacheLoad_ToCache; + //std::list mCacheLoad_ToCache; + std::map > mCacheLoad_ToCache, mPendingCache; // Switching to RsMemCache for Key Caching. RsMemCache mPublicKeyCache; @@ -421,8 +425,9 @@ std::string genRandomId(int len = 20); private: - std::map > mGroupsToCache; + std::map > mIdsPendingCache; std::map > mGroupNotPresent; + std::map > mIdsNotPresent; RsNetworkExchangeService* mNes; }; From cf8a63888b2ddc062bedf799183aee9067958c40 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Mon, 4 Nov 2013 14:09:32 +0000 Subject: [PATCH 17/29] some fixes - id pull now works (Peer A, Peer B test) git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6889 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgixs.h | 2 +- libretroshare/src/gxs/rsgxsnetservice.cc | 9 +++++++-- libretroshare/src/gxs/rsgxsnetutils.cc | 10 ++++++---- libretroshare/src/gxs/rsgxsnetutils.h | 2 +- libretroshare/src/rsserver/rsinit.cc | 1 + libretroshare/src/services/p3idservice.cc | 23 +++++++++++++++++++---- libretroshare/src/services/p3idservice.h | 8 +++++++- 7 files changed, 42 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/gxs/rsgixs.h b/libretroshare/src/gxs/rsgixs.h index 304609711..cd3bddf9a 100644 --- a/libretroshare/src/gxs/rsgixs.h +++ b/libretroshare/src/gxs/rsgixs.h @@ -176,7 +176,7 @@ class RsGixsReputation public: // get Reputation. virtual bool haveReputation(const RsGxsId &id) = 0; - virtual bool loadReputation(const RsGxsId &id) = 0; + virtual bool loadReputation(const RsGxsId &id, const std::list& peers) = 0; virtual bool getReputation(const RsGxsId &id, GixsReputation &rep) = 0; }; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index afab99ba0..872a1777d 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -1263,6 +1263,9 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) MsgAuthorV toVet; + std::list peers; + peers.push_back(tr->mTransaction->PeerId()); + for(; llit != msgItemL.end(); llit++) { RsNxsSyncMsgItem*& syncItem = *llit; @@ -1289,7 +1292,7 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) else { // preload for speed - mReputations->loadReputation(syncItem->authorId); + mReputations->loadReputation(syncItem->authorId, peers); MsgAuthEntry entry; entry.mAuthorId = syncItem->authorId; entry.mGrpId = syncItem->grpId; @@ -1380,6 +1383,8 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) uint32_t transN = locked_getTransactionId(); GrpAuthorV toVet; + std::list peers; + peers.push_back(tr->mTransaction->PeerId()); for(; llit != grpItemL.end(); llit++) { @@ -1413,7 +1418,7 @@ void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) else { // preload reputation for later - mReputations->loadReputation(grpSyncItem->authorId); + mReputations->loadReputation(grpSyncItem->authorId, peers); GrpAuthEntry entry; entry.mAuthorId = grpSyncItem->authorId; entry.mGrpId = grpSyncItem->grpId; diff --git a/libretroshare/src/gxs/rsgxsnetutils.cc b/libretroshare/src/gxs/rsgxsnetutils.cc index f61ee033c..e5fb94447 100644 --- a/libretroshare/src/gxs/rsgxsnetutils.cc +++ b/libretroshare/src/gxs/rsgxsnetutils.cc @@ -47,14 +47,16 @@ bool AuthorPending::expired() const } bool AuthorPending::getAuthorRep(GixsReputation& rep, - const std::string& authorId) + const std::string& authorId, const std::string& peerId) { if(mRep->haveReputation(authorId)) { return mRep->getReputation(authorId, rep); } - mRep->loadReputation(authorId); + std::list peers; + peers.push_back(peerId); + mRep->loadReputation(authorId, peers); return false; } @@ -94,7 +96,7 @@ bool MsgRespPending::accepted() if(!entry.mPassedVetting) { GixsReputation rep; - if(getAuthorRep(rep, entry.mAuthorId)) + if(getAuthorRep(rep, entry.mAuthorId, mPeerId)) { if(rep.score > mCutOff) { @@ -129,7 +131,7 @@ bool GrpRespPending::accepted() { GixsReputation rep; - if(getAuthorRep(rep, entry.mAuthorId)) + if(getAuthorRep(rep, entry.mAuthorId, mPeerId)) { if(rep.score > mCutOff) { diff --git a/libretroshare/src/gxs/rsgxsnetutils.h b/libretroshare/src/gxs/rsgxsnetutils.h index 1bd70b9b9..b93283aac 100644 --- a/libretroshare/src/gxs/rsgxsnetutils.h +++ b/libretroshare/src/gxs/rsgxsnetutils.h @@ -141,7 +141,7 @@ protected: * @param authorId reputation to get * @return true if successfully retrieve repution */ - bool getAuthorRep(GixsReputation& rep, const std::string& authorId); + bool getAuthorRep(GixsReputation& rep, const std::string& authorId, const std::string& peerId); private: diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index e4f58908f..3c8f54155 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -2271,6 +2271,7 @@ int RsServer::StartupRetroShare() mGxsIdService, mGxsIdService, mGxsCircles, false); // don't synchronise group automatic (need explicit group request) + mGxsIdService->setNes(gxsid_ns); /**** GxsCircle service ****/ diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index f358cf561..a518b6c19 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -164,6 +164,11 @@ p3IdService::p3IdService(RsGeneralDataService *gds, RsNetworkExchangeService *ne loadRecognKeys(); } +void p3IdService::setNes(RsNetworkExchangeService *nes) +{ + RsStackMutex stack(mIdMtx); + mNes = nes; +} uint32_t p3IdService::idAuthenPolicy() { @@ -419,7 +424,7 @@ bool p3IdService::requestKey(const RsGxsId &id, const std::list &peers) } - return cache_request_load(id); + return cache_request_load(id, peers); } bool p3IdService::isPendingNetworkRequest(const RsGxsId& gxsId) const @@ -473,11 +478,18 @@ bool p3IdService::haveReputation(const RsGxsId &id) return haveKey(id); } -bool p3IdService::loadReputation(const RsGxsId &id) +bool p3IdService::loadReputation(const RsGxsId &id, const std::list& peers) { if (haveKey(id)) return true; - return cache_request_load(id); + else + { + if(isPendingNetworkRequest(id)) + return true; + } + + + return cache_request_load(id, peers); } bool p3IdService::getReputation(const RsGxsId &id, GixsReputation &rep) @@ -1367,7 +1379,7 @@ bool p3IdService::cache_start_load() groupIds.push_back(it->first); // might need conversion? } - // mPendingCache.insert(mCacheLoad_ToCache.begin(), mCacheLoad_ToCache.end()); + mPendingCache.insert(mCacheLoad_ToCache.begin(), mCacheLoad_ToCache.end()); mCacheLoad_ToCache.clear(); } @@ -1482,7 +1494,10 @@ void p3IdService::requestIdsFromNet() std::map >::const_iterator cit2; for(cit2 = requests.begin(); cit2 != requests.end(); cit2++) + { + if(mNes) mNes->requestGrp(cit2->second, cit2->first); + } mIdsNotPresent.clear(); } diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index f15a0230d..8cf81c606 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -196,6 +196,12 @@ static uint32_t idAuthenPolicy(); virtual void service_tick(); // needed for background processing. + /*! + * Design hack, id service must be constructed first as it + * is need for construction of subsequent net services + */ + void setNes(RsNetworkExchangeService* nes); + /* General Interface is provided by RsIdentity / RsGxsIfaceImpl. */ /* Data Specific Interface */ @@ -258,7 +264,7 @@ virtual int getPrivateKey(const RsGxsId &id, RsTlvSecurityKey &key); // get Reputation. virtual bool haveReputation(const RsGxsId &id); -virtual bool loadReputation(const RsGxsId &id); +virtual bool loadReputation(const RsGxsId &id, const std::list& peers); virtual bool getReputation(const RsGxsId &id, GixsReputation &rep); From 6871f9875aca048f68bf426d952d9461a8f2096d Mon Sep 17 00:00:00 2001 From: drbob Date: Tue, 5 Nov 2013 23:33:12 +0000 Subject: [PATCH 18/29] Fixed up bugs in Recogn System - basics work now. - added missing decrement to DataCount in rsmemcache::erase() - added missing note_event_locked() in rstickevent ... can now use previous tick info, - revamped recogn scheduling. on demand rather than periodic. new function recogn_schedule(). - deferred loading of pgpIdList for pgphashes, until we know it will be needed. git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6893 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/services/p3idservice.cc | 52 +++++++++++++++++++---- libretroshare/src/services/p3idservice.h | 1 + libretroshare/src/util/rsmemcache.h | 7 +++ libretroshare/src/util/rstickevent.cc | 1 + 4 files changed, 52 insertions(+), 9 deletions(-) diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index a518b6c19..9524cb48b 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -148,7 +148,6 @@ p3IdService::p3IdService(RsGeneralDataService *gds, RsNetworkExchangeService *ne // Kick off Cache Testing, + Others. RsTickEvent::schedule_in(GXSID_EVENT_PGPHASH, PGPHASH_PERIOD); - RsTickEvent::schedule_in(GXSID_EVENT_RECOGN, RECOGN_PERIOD); RsTickEvent::schedule_in(GXSID_EVENT_REPUTATION, REPUTATION_PERIOD); RsTickEvent::schedule_now(GXSID_EVENT_CACHEOWNIDS); @@ -1232,6 +1231,7 @@ bool p3IdService::cache_process_recogntaginfo(const RsGxsIdGroupItem *item, std: std::cerr << std::endl; #endif // DEBUG_RECOGN + recogn_schedule(); } return true; @@ -2054,8 +2054,6 @@ bool p3IdService::pgphash_start() std::cerr << std::endl; #endif // DEBUG_IDS - getPgpIdList(); - // ACTUALLY only need summary - but have written code for data. // Also need to use opts.groupFlags to filter stuff properly to REALID's only. // TODO @@ -2089,7 +2087,7 @@ bool p3IdService::pgphash_handlerequest(uint32_t token) // We Will do this later! std::vector groups; - std::vector groupsToProcess; + bool groupsToProcess = false; bool ok = getGroupData(token, groups); if(ok) @@ -2162,6 +2160,7 @@ bool p3IdService::pgphash_handlerequest(uint32_t token) RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ mGroupsToProcess.push_back(*vit); + groupsToProcess = true; } } else @@ -2170,6 +2169,12 @@ bool p3IdService::pgphash_handlerequest(uint32_t token) std::cerr << std::endl; } + if (groupsToProcess) + { + // update PgpIdList -> if there are groups to process. + getPgpIdList(); + } + // Schedule Processing. RsTickEvent::schedule_in(GXSID_EVENT_PGPHASH_PROC, PGPHASH_PROC_PERIOD); return true; @@ -2431,6 +2436,37 @@ void calcPGPHash(const RsGxsId &id, const PGPFingerprintType &pgp, GxsIdPgpHash * Info to be stored in GroupServiceString + Cache. **/ +bool p3IdService::recogn_schedule() +{ + std::cerr << "p3IdService::recogn_schedule()"; + std::cerr << std::endl; + + int32_t age = 0; + int32_t next_event = 0; + + if (RsTickEvent::event_count(GXSID_EVENT_RECOGN) > 0) + { + std::cerr << "p3IdService::recogn_schedule() Skipping GXSIS_EVENT_RECOGN already scheduled"; + std::cerr << std::endl; + return false; + } + + if (RsTickEvent::prev_event_ago(GXSID_EVENT_RECOGN, age)) + { + std::cerr << "p3IdService::recogn_schedule() previous event " << age << " secs ago"; + std::cerr << std::endl; + + next_event = RECOGN_PERIOD - age; + if (next_event < 0) + { + next_event = 0; + } + } + + RsTickEvent::schedule_in(GXSID_EVENT_RECOGN, next_event); + return true; +} + bool p3IdService::recogn_start() { @@ -2446,9 +2482,7 @@ bool p3IdService::recogn_start() return false; } - // SCHEDULE NEXT ONE. - RsTickEvent::schedule_in(GXSID_EVENT_RECOGN, RECOGN_PERIOD); - + // NEXT EVENT is scheduled via recogn_schedule. #ifdef DEBUG_RECOGN std::cerr << "p3IdService::recogn_start() making request"; @@ -2545,10 +2579,10 @@ bool p3IdService::recogn_process() bool isDone = false; { RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ - if (!mRecognGroupsToProcess.empty() && !mGroupsToProcess.empty()) + if (!mRecognGroupsToProcess.empty()) { item = mRecognGroupsToProcess.front(); - mGroupsToProcess.pop_front(); + mRecognGroupsToProcess.pop_front(); #ifdef DEBUG_RECOGN std::cerr << "p3IdService::recogn_process() Popped Group: " << item->meta.mGroupId; diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index 8cf81c606..48f8ebf9b 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -356,6 +356,7 @@ virtual void handle_event(uint32_t event_type, const std::string &elabel); * recogn processing. * */ + bool recogn_schedule(); bool recogn_start(); bool recogn_handlerequest(uint32_t token); bool recogn_process(); diff --git a/libretroshare/src/util/rsmemcache.h b/libretroshare/src/util/rsmemcache.h index 0f5fb966f..08a4e3a9a 100644 --- a/libretroshare/src/util/rsmemcache.h +++ b/libretroshare/src/util/rsmemcache.h @@ -204,6 +204,7 @@ template bool RsMemCache::erase(const Key &k // remove from lru. mDataMap.erase(it); update_lrumap(key, old_ts, new_ts); + mDataCount--; mStats_access++; return true; @@ -362,6 +363,12 @@ template bool RsMemCache::resize() // ERROR. std::cerr << "RsMemCache::resize() CONSISTENCY ERROR"; std::cerr << std::endl; + std::cerr << "\tmDataMap.size() = " << mDataMap.size(); + std::cerr << std::endl; + std::cerr << "\tmLruMap.size() = " << mLruMap.size(); + std::cerr << std::endl; + std::cerr << "\tmDataCount = " << mDataCount; + std::cerr << std::endl; } if (mDataCount > mMaxSize) diff --git a/libretroshare/src/util/rstickevent.cc b/libretroshare/src/util/rstickevent.cc index 2ebbeec91..69781a4f2 100644 --- a/libretroshare/src/util/rstickevent.cc +++ b/libretroshare/src/util/rstickevent.cc @@ -80,6 +80,7 @@ void RsTickEvent::tick_events() mEvents.erase(it); count_adjust_locked(event_type, -1); + note_event_locked(event_type); } } From 2e886bf443128e5d8b625ca372065917ac7e6a0c Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 10 Nov 2013 22:27:50 +0000 Subject: [PATCH 19/29] Add gxs update items for nxs sync optimisation git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6901 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- .../src/serialiser/rsgxsupdateitems.cc | 348 ++++++++++++++++++ .../src/serialiser/rsgxsupdateitems.h | 105 ++++++ 2 files changed, 453 insertions(+) create mode 100644 libretroshare/src/serialiser/rsgxsupdateitems.cc create mode 100644 libretroshare/src/serialiser/rsgxsupdateitems.h diff --git a/libretroshare/src/serialiser/rsgxsupdateitems.cc b/libretroshare/src/serialiser/rsgxsupdateitems.cc new file mode 100644 index 000000000..f99c7459a --- /dev/null +++ b/libretroshare/src/serialiser/rsgxsupdateitems.cc @@ -0,0 +1,348 @@ +/* + * libretroshare/src/serialiser: rsgxsupdateitems.h + * + * RetroShare Serialiser. + * + * Copyright 2012 Christopher Evi-Parker + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include "rsgxsupdateitems.h" + + + + +RsGxsGrpUpdateItem::RsGxsGrpUpdateItem() { +} + +RsGxsGrpUpdateItem::~RsGxsGrpUpdateItem() { +} + +void RsGxsGrpUpdateItem::clear() +{ + grpUpdateTS = 0; + peerId.clear(); +} + +std::ostream& RsGxsGrpUpdateItem::print(std::ostream& out, uint16_t indent) +{ + +} + +RsGxsMsgUpdateItem::RsGxsMsgUpdateItem() +{ +} + +RsGxsMsgUpdateItem::~RsGxsMsgUpdateItem() { +} + +void RsGxsMsgUpdateItem::clear() +{ + msgUpdateTS.clear(); + peerId.clear(); +} + +std::ostream& RsGxsMsgUpdateItem::print(std::ostream& out, uint16_t indent) +{ +} + +uint32_t RsGxsUpdateSerialiser::size(RsItem* item) +{ + RsGxsMsgUpdateItem* mui = NULL; + RsGxsGrpUpdateItem* gui = NULL; + + if((mui = dynamic_cast(item)) != NULL) + { + return sizeGxsMsgUpdate(mui); + }else if(( gui = dynamic_cast(item)) != NULL){ + return sizeGxsGrpUpdate(gui); + }else + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::size(): Could not find appropriate size function" + << std::endl; +#endif + return 0; + } +} + +bool RsGxsUpdateSerialiser::serialise(RsItem* item, void* data, + uint32_t* size) +{ +} + +RsItem* RsGxsUpdateSerialiser::deserialise(void* data, uint32_t* size) +{ + +#ifdef RSSERIAL_DEBUG + std::cerr << "RsNxsSerialiser::deserialise()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (SERVICE_TYPE != getRsItemService(rstype))) + { + return NULL; /* wrong type */ + } + + switch(getRsItemSubType(rstype)) + { + + case RS_PKT_SUBTYPE_GXS_MSG_UPDATE: + return deserialGxsMsgUpdate(data, size); + case RS_PKT_SUBTYPE_GXS_GRP_UPDATE: + return deserialGxsGrpUpddate(data, size); + default: + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialise() : data has no type" + << std::endl; +#endif + return NULL; + + } + } +} + +uint32_t RsGxsUpdateSerialiser::sizeGxsGrpUpdate(RsGxsGrpUpdateItem* item) +{ + uint32_t s = 8; // header size + s += GetTlvStringSize(item->peerId); + s += 4; + return s; +} + +bool RsGxsUpdateSerialiser::serialiseGxsGrpUpdate(RsGxsGrpUpdateItem* item, + void* data, uint32_t* size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsGrpUpdate()" << std::endl; +#endif + + uint32_t tlvsize = sizeGxsGrpUpdate(item); + uint32_t offset = 0; + + if(*size < tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsGrpUpdate() size do not match" << std::endl; +#endif + return false; + } + + *size = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize); + + /* skip the header */ + offset += 8; + + /* RsNxsSyncm */ + + + ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, item->peerId); + ok &= setRawUInt32(data, *size, &offset, item->grpUpdateTS); + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsGrpUpdate() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsGrpUpdate() NOK" << std::endl; + } +#endif + + return ok; +} + +RsGxsGrpUpdateItem* RsGxsUpdateSerialiser::deserialGxsGrpUpddate(void* data, + uint32_t* size) +{ + +} + +uint32_t RsGxsUpdateSerialiser::sizeGxsMsgUpdate(RsGxsMsgUpdateItem* item) +{ + uint32_t s = 8; // header size + s += GetTlvStringSize(item->peerId); + + const std::map& msgUpdateTS = item->msgUpdateTS; + std::map::const_iterator cit = msgUpdateTS.begin(); + + for(; cit != msgUpdateTS.end(); cit++) + { + s += GetTlvStringSize(cit->first); + s += 4; + } + + s += 4; // number of map items + + return s; +} + +bool RsGxsUpdateSerialiser::serialiseGxsMsgUpdate(RsGxsMsgUpdateItem* item, + void* data, uint32_t* size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsMsgUpdate()" << std::endl; +#endif + + uint32_t tlvsize = sizeGxsMsgUpdate(item); + uint32_t offset = 0; + + if(*size < tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsMsgUpdate() size do not match" << std::endl; +#endif + return false; + } + + *size = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize); + + /* skip the header */ + offset += 8; + + /* RsNxsSyncm */ + + + ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, item->peerId); + + const std::map& msgUpdateTS = item->msgUpdateTS; + std::map::const_iterator cit = msgUpdateTS.begin(); + + uint32_t numItems = msgUpdateTS.size(); + ok &= setRawUInt32(data, *size, &offset, numItems); + + for(; cit != msgUpdateTS.end(); cit++) + { + ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, cit->first); + ok &= setRawUInt32(data, *size, &offset, cit->second); + } + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsMsgUpdate() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsMsgUpdate() NOK" << std::endl; + } +#endif + + return ok; +} + +RsGxsMsgUpdateItem* RsGxsUpdateSerialiser::deserialGxsMsgUpdate(void* data, + uint32_t* size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsMsgUpdate()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + uint32_t offset = 0; + + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (SERVICE_TYPE != getRsItemService(rstype)) || + (RS_PKT_SUBTYPE_NXS_SYNC_MSG != getRsItemSubType(rstype))) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsMsgUpdate() FAIL wrong type" << std::endl; +#endif + return NULL; /* wrong type */ + } + + if (*size < rssize) /* check size */ + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsMsgUpdate() FAIL wrong size" << std::endl; +#endif + return NULL; /* not enough data */ + } + + /* set the packet length */ + *size = rssize; + + bool ok = true; + + RsGxsMsgUpdateItem* item = new RsGxsMsgUpdateItem(getRsItemService(rstype)); + + /* skip the header */ + offset += 8; + + ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_HASH_SHA1, item->peerId); + uint32_t numUpdateItems; + ok &= getRawUInt32(data, *size, &offset, &(numUpdateItems)); + std::map& msgUpdateItem = item->grpUpdateTS; + std::string grpId; + uint32_t updateTS; + for(uint32_t i; i < numUpdateItems; i++) + { + ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, grpId); + + if(!ok) + break; + + ok &= getRawUInt32(data, *size, &offset, &(updateTS)); + + if(!ok) + break; + + msgUpdateItem.insert(std::make_pair(grpId, updateTS)); + } + + if (offset != rssize) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsMsgUpdate() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsMsgUpdate() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} diff --git a/libretroshare/src/serialiser/rsgxsupdateitems.h b/libretroshare/src/serialiser/rsgxsupdateitems.h new file mode 100644 index 000000000..4b16bcb1b --- /dev/null +++ b/libretroshare/src/serialiser/rsgxsupdateitems.h @@ -0,0 +1,105 @@ +#ifndef RSGXSUPDATEITEMS_H_ +#define RSGXSUPDATEITEMS_H_ + +/* + * libretroshare/src/serialiser: rsgxsupdateitems.h + * + * RetroShare Serialiser. + * + * Copyright 2012 Christopher Evi-Parker + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + + +#include + +#include "serialiser/rsserviceids.h" +#include "serialiser/rsserial.h" +#include "serialiser/rstlvbase.h" +#include "serialiser/rstlvtypes.h" +#include "serialiser/rstlvkeys.h" +#include "gxs/rsgxsdata.h" + + +const uint8_t RS_PKT_SUBTYPE_GXS_GRP_UPDATE = 0x0001; +const uint8_t RS_PKT_SUBTYPE_GXS_MSG_UPDATE = 0x0002; + +class RsGxsGrpUpdateItem : public RsItem { +public: + RsGxsGrpUpdateItem(); + virtual ~RsGxsGrpUpdateItem(); + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + std::string peerId; + uint32_t grpUpdateTS; +}; + +class RsGxsMsgUpdateItem : public RsItem +{ +public: + RsGxsMsgUpdateItem(); + virtual ~RsGxsMsgUpdateItem(); + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + std::string peerId; + std::map msgUpdateTS; +}; + + + +class RsGxsUpdateSerialiser : public RsSerialType +{ +public: + + RsGxsUpdateSerialiser(uint16_t servtype) : + RsSerialType(RS_PKT_VERSION_SERVICE, servtype), SERVICE_TYPE(servtype) { return; } + + virtual ~RsGxsUpdateSerialiser() { return; } + + virtual uint32_t size(RsItem *item); + virtual bool serialise(RsItem *item, void *data, uint32_t *size); + virtual RsItem* deserialise(void *data, uint32_t *size); + +private: + + + /* for RS_PKT_SUBTYPE_GRP_UPDATE_ITEM */ + + virtual uint32_t sizeGxsGrpUpdate(RsGxsGrpUpdateItem* item); + virtual bool serialiseGxsGrpUpdate(RsGxsGrpUpdateItem *item, void *data, uint32_t *size); + virtual RsGxsGrpUpdateItem* deserialGxsGrpUpddate(void *data, uint32_t *size); + + /* for RS_PKT_SUBTYPE_GXS_ */ + + virtual uint32_t sizeGxsMsgUpdate(RsGxsMsgUpdateItem* item); + virtual bool serialiseGxsMsgUpdate(RsGxsMsgUpdateItem *item, void *data, uint32_t *size); + virtual RsGxsMsgUpdateItem* deserialGxsMsgUpdate(void *data, uint32_t *size); + +private: + + const uint16_t SERVICE_TYPE; +}; + + + +#endif /* RSGXSUPDATEITEMS_H_ */ From f74aacd759e04099afa6389e9c332bec1189fb4c Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sat, 16 Nov 2013 13:40:04 +0000 Subject: [PATCH 20/29] remaining code for nxs sync optimisation - still needs testing git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6903 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgxsnetservice.cc | 246 +++++++++++- libretroshare/src/gxs/rsgxsnetservice.h | 20 + libretroshare/src/libretroshare.pro | 6 +- .../src/serialiser/rsgxsupdateitems.cc | 363 +++++++++++++++++- .../src/serialiser/rsgxsupdateitems.h | 56 ++- libretroshare/src/serialiser/rsnxsitems.cc | 37 +- libretroshare/src/serialiser/rsnxsitems.h | 9 +- 7 files changed, 678 insertions(+), 59 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 872a1777d..3485d2863 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -90,10 +90,21 @@ void RsGxsNetService::syncWithPeers() // for now just grps for(; sit != peers.end(); sit++) { - RsNxsSyncGrp *grp = new RsNxsSyncGrp(mServType); - grp->clear(); - grp->PeerId(*sit); - sendItem(grp); + + const std::string peerId = *sit; + + ClientGrpMap::const_iterator cit = mClientGrpUpdateMap.find(peerId); + uint32_t updateTS = 0; + if(cit != mClientGrpUpdateMap.end()) + { + const RsGxsGrpUpdateItem *gui = cit->second; + updateTS = gui->grpUpdateTS; + } + RsNxsSyncGrp *grp = new RsNxsSyncGrp(mServType); + grp->clear(); + grp->PeerId(*sit); + grp->updateTS = updateTS; + sendItem(grp); } } @@ -111,7 +122,9 @@ void RsGxsNetService::syncWithPeers() RsGxsGrpMetaData* meta = mit->second; if(meta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) + { grpIds.push_back(mit->first); + } delete meta; } @@ -125,13 +138,36 @@ void RsGxsNetService::syncWithPeers() std::vector::iterator vit = grpIds.begin(); + // now see if you have an updateTS so optimise whether you need + // to get a new list of peer data + RsGxsMsgUpdateItem* mui = NULL; + + ClientMsgMap::const_iterator cit = mClientMsgUpdateMap.find(*sit); + + if(cit != mClientMsgUpdateMap.end()) + { + mui = cit->second; + } + for(; vit != grpIds.end(); vit++) { - RsNxsSyncMsg* msg = new RsNxsSyncMsg(mServType); - msg->clear(); - msg->PeerId(*sit); - msg->grpId = *vit; - sendItem(msg); + uint32_t updateTS = 0; + if(mui) + { + std::map::const_iterator cit2 = mui->msgUpdateTS.find(*vit); + + if(cit2 != mui->msgUpdateTS.end()) + { + updateTS = cit2->second; + } + } + + RsNxsSyncMsg* msg = new RsNxsSyncMsg(mServType); + msg->clear(); + msg->PeerId(*sit); + msg->grpId = *vit; + msg->updateTS = updateTS; + sendItem(msg); } } #endif @@ -526,20 +562,101 @@ void RsGxsNetService::collateMsgFragments(MsgFragments fragments, std::map& load) +class StoreHere { - return false; +public: + + StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap cmm, + RsGxsNetService::ServerMsgMap& smm, + RsGxsServerGrpUpdateItem*& sgm) : mClientGrpMap(cgm), mClientMsgMap(cmm), + mServerMsgMap(smm), mServerGrpUpdateItem(sgm) + {} + + void operator() (RsItem* item) + { + RsGxsMsgUpdateItem* mui; + RsGxsGrpUpdateItem* gui; + RsGxsServerGrpUpdateItem* gsui; + RsGxsServerMsgUpdateItem* msui; + + if((mui = dynamic_cast(item)) != NULL) + mClientMsgMap.insert(std::make_pair(mui->peerId, mui)); + else if((gui = dynamic_cast(item)) != NULL) + mClientGrpMap.insert(std::make_pair(gui->peerId, gui)); + else if((msui = dynamic_cast(item)) != NULL) + mServerMsgMap.insert(std::make_pair(msui->grpId, msui)); + else if((gsui = dynamic_cast(item)) != NULL) + { + if(mServerGrpUpdateItem) + { + mServerGrpUpdateItem = gsui; + } + else + { +#ifdef NXS_NET_DEBUG + std::cerr << "Error! More than one server group update item exists!" << std::endl; +#endif + delete gsui; + } + } + else + { + std::cerr << "Type not expected!" << std::endl; + } + + } + +private: + + RsGxsNetService::ClientGrpMap& mClientGrpMap; + RsGxsNetService::ClientMsgMap& mClientMsgMap; + RsGxsNetService::ServerMsgMap& mServerMsgMap; + RsGxsServerGrpUpdateItem*& mServerGrpUpdateItem; + +}; + +bool RsGxsNetService::loadList(std::list &load) +{ + std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, + mServerMsgUpdateMap, mGrpServerUpdateItem)); + return true; } +#include + +template +struct get_second : public std::unary_function +{ + RsItem* operator()(const typename UpdateMap::value_type& value) const + { + return value.second; + } +}; + bool RsGxsNetService::saveList(bool& cleanup, std::list& save) { - return false; + RsStackMutex stack(mNxsMutex); + + // hardcore templates + std::transform(mClientGrpUpdateMap.begin(), mClientGrpUpdateMap.end(), + std::back_inserter(save), get_second()); + + std::transform(mClientMsgUpdateMap.begin(), mClientMsgUpdateMap.end(), + std::back_inserter(save), get_second()); + + std::transform(mServerMsgUpdateMap.begin(), mServerMsgUpdateMap.end(), + std::back_inserter(save), get_second()); + + save.push_back(mGrpServerUpdateItem); } RsSerialiser *RsGxsNetService::setupSerialiser() { - return NULL; + + RsSerialiser *rss = new RsSerialiser; + rss->addSerialType(new RsGxsUpdateSerialiser(mServType)); + + return rss; } void RsGxsNetService::recvNxsItemQueue(){ @@ -1039,25 +1156,53 @@ void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) // notify listener of grps mObserver->notifyNewGroups(grps); + // now note this as the latest you've received from this peer + std::string peerFrom = tr->mTransaction->PeerId(); + uint32_t updateTS = tr->mTransaction->updateTS; + + ClientGrpMap::iterator it = mClientGrpUpdateMap.find(peerFrom); + + RsGxsGrpUpdateItem* item = NULL; + + if(it != mClientGrpUpdateMap.end()) + { + item = it->second; + }else + { + item = new RsGxsGrpUpdateItem(mServType); + } + + item->grpUpdateTS = updateTS; + item->peerId = peerFrom; + + mClientGrpUpdateMap.insert( + std::make_pair(peerFrom, item)); + + // as a grp list server also note this is the latest item you have + mGrpServerUpdateItem->grpUpdateTS = updateTS; }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) { std::vector msgs; + std::string grpId; while(tr->mItems.size() > 0) { RsNxsMsg* msg = dynamic_cast(tr->mItems.front()); if(msg) { - tr->mItems.pop_front(); - msgs.push_back(msg); + if(grpId.empty()) + grpId = msg->grpId; + + tr->mItems.pop_front(); + msgs.push_back(msg); } else { #ifdef NXS_NET_DEBUG - std::cerr << "RsGxsNetService::processCompletedTransactions(): item did not caste to msg" - << std::endl; + std::cerr << "RsGxsNetService::processCompletedTransactions(): item did not caste to msg" + << std::endl; #endif } } @@ -1081,6 +1226,10 @@ void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) // notify listener of msgs mObserver->notifyNewMessages(msgs); + // now note that this is the latest you've received from this peer + // for the grp id + locked_doMsgUpdateWork(tr->mTransaction, grpId); + } }else if(tr->mFlag == NxsTransaction::FLAG_STATE_FAILED){ // don't do anything transaction will simply be cleaned @@ -1088,6 +1237,46 @@ void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) return; } +void RsGxsNetService::locked_doMsgUpdateWork(const RsNxsTransac *nxsTrans, const std::string &grpId) +{ + + // firts check if peer exists + const std::string& peerFrom = nxsTrans->PeerId(); + + ClientMsgMap::iterator it = mClientMsgUpdateMap.find(peerFrom); + + RsGxsMsgUpdateItem* mui = NULL; + + // now update the peer's entry for this grp id + if(it != mClientMsgUpdateMap.end()) + { + mui = it->second; + } + else + { + mui = new RsGxsMsgUpdateItem(mServType); + mClientMsgUpdateMap.insert(std::make_pair(peerFrom, mui)); + } + + mui->msgUpdateTS[grpId] = nxsTrans->updateTS; + mui->peerId = peerFrom; + + ServerMsgMap::iterator mit = mServerMsgUpdateMap.find(grpId); + RsGxsServerMsgUpdateItem* msui = NULL; + if(mit != mServerMsgUpdateMap.end()) + { + msui = mit->second; + } + else + { + msui = new RsGxsServerMsgUpdateItem(mServType); + mServerMsgUpdateMap.insert(std::make_pair(grpId, msui)); + } + + msui->grpId = grpId; + msui->msgUpdateTS = nxsTrans->updateTS; +} + void RsGxsNetService::locked_processCompletedOutgoingTrans(NxsTransaction* tr) { uint16_t flag = tr->mTransaction->transactFlag; @@ -1128,6 +1317,7 @@ void RsGxsNetService::locked_processCompletedOutgoingTrans(NxsTransaction* tr) std::cerr << "complete Sending Grp Data, transN: " << tr->mTransaction->transactionNumber << std::endl; #endif + }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) { #ifdef NXS_NET_DEBUG @@ -1169,7 +1359,7 @@ void RsGxsNetService::locked_pushMsgTransactionFromList( newTrans->mTimeOut = time(NULL) + mTransactionTimeOut; // create transaction copy with your id to indicate // its an outgoing transaction - newTrans->mTransaction = new RsNxsTransac(*transac); + newTrans->mTransaction = new RsNxsTransac(*transac); newTrans->mTransaction->PeerId(mOwnId); sendItem(transac); { @@ -1250,7 +1440,7 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) msgIdSet.insert((*vit)->mMsgId); delete(*vit); } - msgMetaV.clear(); + msgMetaV.clear(); // get unique id for this transaction uint32_t transN = locked_getTransactionId(); @@ -1516,6 +1706,7 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) ntr->transactionNumber = transN; ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_GRPS; + ntr->updateTS = time(NULL); ntr->nItems = grps.size(); ntr->PeerId(tr->mTransaction->PeerId()); @@ -1697,6 +1888,7 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) ntr->transactionNumber = transN; ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_MSGS; + ntr->updateTS = time(NULL); ntr->nItems = msgSize; ntr->PeerId(peerId); @@ -1779,6 +1971,10 @@ void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) std::string peer = item->PeerId(); + // don't sync if you have no new updates for this peer + if(item->updateTS >= mGrpServerUpdateItem->grpUpdateTS && item->updateTS != 0) + return; + std::map grp; mDataStore->retrieveGxsGrpMetaData(grp); @@ -1893,6 +2089,16 @@ void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsg* item) const std::string& peer = item->PeerId(); + ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId); + + if(cit != mServerMsgUpdateMap.end()) + { + const RsGxsServerMsgUpdateItem *msui = cit->second; + + if(item->updateTS > msui->msgUpdateTS && item->updateTS != 0) + return; + } + GxsMsgMetaResult metaResult; GxsMsgReq req; diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 33d1c9d50..0f7f95774 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -34,6 +34,7 @@ #include "rsnxsobserver.h" #include "pqi/p3linkmgr.h" #include "serialiser/rsnxsitems.h" +#include "serialiser/rsgxsupdateitems.h" #include "rsgxsnetutils.h" #include "pqi/p3cfgmgr.h" #include "rsgixs.h" @@ -323,6 +324,8 @@ private: bool locked_canReceive(const RsGxsGrpMetaData * const grpMeta, const std::string& peerId); void processExplicitGroupRequests(); + + void locked_doMsgUpdateWork(const RsNxsTransac* nxsTrans, const std::string& grpId); private: @@ -427,6 +430,23 @@ private: std::vector mPendingCircleVets; std::map > mExplicitRequest; + + // nxs sync optimisation + // can pull dynamically the latest timestamp for each message + +public: + + typedef std::map ClientMsgMap; + typedef std::map ServerMsgMap; + typedef std::map ClientGrpMap; + +private: + + ClientMsgMap mClientMsgUpdateMap; + ServerMsgMap mServerMsgUpdateMap; + ClientGrpMap mClientGrpUpdateMap; + + RsGxsServerGrpUpdateItem* mGrpServerUpdateItem; }; #endif // RSGXSNETSERVICE_H diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 42fc6e8f8..c3c3ca174 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -388,7 +388,8 @@ HEADERS += serialiser/rsbaseitems.h \ serialiser/rsbanlistitems.h \ serialiser/rsbwctrlitems.h \ serialiser/rsgxsrecognitems.h \ - serialiser/rstunnelitems.h + serialiser/rstunnelitems.h \ + serialiser/rsgxsupdateitems.h HEADERS += services/p3channels.h \ services/p3chatservice.h \ @@ -532,7 +533,8 @@ SOURCES += serialiser/rsbaseitems.cc \ serialiser/rsbanlistitems.cc \ serialiser/rsbwctrlitems.cc \ serialiser/rsgxsrecognitems.cc \ - serialiser/rstunnelitems.cc + serialiser/rstunnelitems.cc \ + serialiser/rsgxsupdateitems.cc SOURCES += services/p3channels.cc \ services/p3chatservice.cc \ diff --git a/libretroshare/src/serialiser/rsgxsupdateitems.cc b/libretroshare/src/serialiser/rsgxsupdateitems.cc index f99c7459a..fdccdbeae 100644 --- a/libretroshare/src/serialiser/rsgxsupdateitems.cc +++ b/libretroshare/src/serialiser/rsgxsupdateitems.cc @@ -24,15 +24,11 @@ */ #include "rsgxsupdateitems.h" +#include "rsbaseserial.h" -RsGxsGrpUpdateItem::RsGxsGrpUpdateItem() { -} - -RsGxsGrpUpdateItem::~RsGxsGrpUpdateItem() { -} void RsGxsGrpUpdateItem::clear() { @@ -45,12 +41,7 @@ std::ostream& RsGxsGrpUpdateItem::print(std::ostream& out, uint16_t indent) } -RsGxsMsgUpdateItem::RsGxsMsgUpdateItem() -{ -} -RsGxsMsgUpdateItem::~RsGxsMsgUpdateItem() { -} void RsGxsMsgUpdateItem::clear() { @@ -66,12 +57,20 @@ uint32_t RsGxsUpdateSerialiser::size(RsItem* item) { RsGxsMsgUpdateItem* mui = NULL; RsGxsGrpUpdateItem* gui = NULL; + RsGxsServerGrpUpdateItem* gsui = NULL; + RsGxsServerMsgUpdateItem* msui = NULL; if((mui = dynamic_cast(item)) != NULL) { return sizeGxsMsgUpdate(mui); }else if(( gui = dynamic_cast(item)) != NULL){ return sizeGxsGrpUpdate(gui); + }else if((gsui = dynamic_cast(item)) != NULL) + { + return sizeGxsServerGrpUpdate(gsui); + }else if((msui = dynamic_cast(item)) != NULL) + { + return sizeGxsServerMsgUpdate(msui); }else { #ifdef RSSERIAL_DEBUG @@ -85,19 +84,41 @@ uint32_t RsGxsUpdateSerialiser::size(RsItem* item) bool RsGxsUpdateSerialiser::serialise(RsItem* item, void* data, uint32_t* size) { + RsGxsMsgUpdateItem* mui; + RsGxsGrpUpdateItem* gui; + RsGxsServerGrpUpdateItem* gsui; + RsGxsServerMsgUpdateItem* msui; + + if((mui = dynamic_cast(item)) != NULL) + return serialiseGxsMsgUpdate(mui, data, size); + else if((gui = dynamic_cast(item)) != NULL) + return serialiseGxsGrpUpdate(gui, data, size); + else if((msui = dynamic_cast(item)) != NULL) + return serialiseGxsServerMsgUpdate(msui, data, size); + else if((gsui = dynamic_cast(item)) != NULL) + return serialiseGxsServerGrpUpdate(gsui, data, size); + else + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialise() item does not caste to known type" + << std::endl; +#endif + + return false; + } } RsItem* RsGxsUpdateSerialiser::deserialise(void* data, uint32_t* size) { #ifdef RSSERIAL_DEBUG - std::cerr << "RsNxsSerialiser::deserialise()" << std::endl; + std::cerr << "RsGxsUpdateSerialiser::deserialise()" << std::endl; #endif /* get the type and size */ uint32_t rstype = getRsItemId(data); if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || - (SERVICE_TYPE != getRsItemService(rstype))) + (SERVICE_TYPE != getRsItemService(rstype))) { return NULL; /* wrong type */ } @@ -109,6 +130,10 @@ RsItem* RsGxsUpdateSerialiser::deserialise(void* data, uint32_t* size) return deserialGxsMsgUpdate(data, size); case RS_PKT_SUBTYPE_GXS_GRP_UPDATE: return deserialGxsGrpUpddate(data, size); + case RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE: + return deserialGxsServerGrpUpddate(data, size); + case RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE: + return deserialGxsServerMsgUpdate(data, size); default: { #ifdef RSSERIAL_DEBUG @@ -129,6 +154,13 @@ uint32_t RsGxsUpdateSerialiser::sizeGxsGrpUpdate(RsGxsGrpUpdateItem* item) return s; } +uint32_t RsGxsUpdateSerialiser::sizeGxsServerGrpUpdate(RsGxsServerGrpUpdateItem* item) +{ + uint32_t s = 8; // header size + s += 4; // time stamp + return s; +} + bool RsGxsUpdateSerialiser::serialiseGxsGrpUpdate(RsGxsGrpUpdateItem* item, void* data, uint32_t* size) { @@ -155,7 +187,7 @@ bool RsGxsUpdateSerialiser::serialiseGxsGrpUpdate(RsGxsGrpUpdateItem* item, /* skip the header */ offset += 8; - /* RsNxsSyncm */ + /* RsGxsGrpUpdateItem */ ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, item->peerId); @@ -178,10 +210,182 @@ bool RsGxsUpdateSerialiser::serialiseGxsGrpUpdate(RsGxsGrpUpdateItem* item, return ok; } +bool RsGxsUpdateSerialiser::serialiseGxsServerGrpUpdate(RsGxsServerGrpUpdateItem* item, + void* data, uint32_t* size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsServerGrpUpdate()" << std::endl; +#endif + + uint32_t tlvsize = sizeGxsServerGrpUpdate(item); + uint32_t offset = 0; + + if(*size < tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsServerGrpUpdate() size do not match" << std::endl; +#endif + return false; + } + + *size = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize); + + /* skip the header */ + offset += 8; + + /* RsGxsServerGrpUpdateItem */ + + ok &= setRawUInt32(data, *size, &offset, item->grpUpdateTS); + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsServerGrpUpdate() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsServerGrpUpdate() NOK" << std::endl; + } +#endif + + return ok; +} + RsGxsGrpUpdateItem* RsGxsUpdateSerialiser::deserialGxsGrpUpddate(void* data, uint32_t* size) { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerGrpUpdate()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + uint32_t offset = 0; + + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (SERVICE_TYPE != getRsItemService(rstype)) || + (RS_PKT_SUBTYPE_GXS_GRP_UPDATE != getRsItemSubType(rstype))) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsGrpUpdate() FAIL wrong type" << std::endl; +#endif + return NULL; /* wrong type */ + } + + if (*size < rssize) /* check size */ + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsGrpUpdate() FAIL wrong size" << std::endl; +#endif + return NULL; /* not enough data */ + } + + /* set the packet length */ + *size = rssize; + + bool ok = true; + + RsGxsGrpUpdateItem* item = new RsGxsGrpUpdateItem(getRsItemService(rstype)); + + /* skip the header */ + offset += 8; + + ok &= getRawUInt32(data, *size, &offset, &(item->grpUpdateTS)); + ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, item->peerId); + + if (offset != rssize) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxxGrpUpdate() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsGrpUpdate() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} + +RsGxsServerGrpUpdateItem* RsGxsUpdateSerialiser::deserialGxsServerGrpUpddate(void* data, + uint32_t* size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerGrpUpdate()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + uint32_t offset = 0; + + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (SERVICE_TYPE != getRsItemService(rstype)) || + (RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE != getRsItemSubType(rstype))) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerGrpUpdate() FAIL wrong type" << std::endl; +#endif + return NULL; /* wrong type */ + } + + if (*size < rssize) /* check size */ + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerGrpUpdate() FAIL wrong size" << std::endl; +#endif + return NULL; /* not enough data */ + } + + /* set the packet length */ + *size = rssize; + + bool ok = true; + + RsGxsServerGrpUpdateItem* item = new RsGxsServerGrpUpdateItem(getRsItemService(rstype)); + + /* skip the header */ + offset += 8; + + ok &= getRawUInt32(data, *size, &offset, &(item->grpUpdateTS)); + + if (offset != rssize) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerGrpUpdate() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerGrpUpdate() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; } uint32_t RsGxsUpdateSerialiser::sizeGxsMsgUpdate(RsGxsMsgUpdateItem* item) @@ -203,6 +407,15 @@ uint32_t RsGxsUpdateSerialiser::sizeGxsMsgUpdate(RsGxsMsgUpdateItem* item) return s; } +uint32_t RsGxsUpdateSerialiser::sizeGxsServerMsgUpdate(RsGxsServerMsgUpdateItem* item) +{ + uint32_t s = 8; // header size + s += GetTlvStringSize(item->grpId); + s += 4; // grp TS + + return s; +} + bool RsGxsUpdateSerialiser::serialiseGxsMsgUpdate(RsGxsMsgUpdateItem* item, void* data, uint32_t* size) { @@ -229,7 +442,7 @@ bool RsGxsUpdateSerialiser::serialiseGxsMsgUpdate(RsGxsMsgUpdateItem* item, /* skip the header */ offset += 8; - /* RsNxsSyncm */ + /* RsGxsMsgUpdateItem */ ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, item->peerId); @@ -263,6 +476,55 @@ bool RsGxsUpdateSerialiser::serialiseGxsMsgUpdate(RsGxsMsgUpdateItem* item, return ok; } +bool RsGxsUpdateSerialiser::serialiseGxsServerMsgUpdate(RsGxsServerMsgUpdateItem* item, + void* data, uint32_t* size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsServerMsgUpdate()" << std::endl; +#endif + + uint32_t tlvsize = sizeGxsServerMsgUpdate(item); + uint32_t offset = 0; + + if(*size < tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsServerMsgUpdate() size do not match" << std::endl; +#endif + return false; + } + + *size = tlvsize; + + bool ok = true; + + ok &= setRsItemHeader(data, tlvsize, item->PacketId(), tlvsize); + + /* skip the header */ + offset += 8; + + /* RsNxsSyncm */ + + + ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId); + ok &= setRawUInt32(data, *size, &offset, item->msgUpdateTS); + + if(offset != tlvsize){ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsServerMsgUpdate() FAIL Size Error! " << std::endl; +#endif + ok = false; + } + +#ifdef RSSERIAL_DEBUG + if (!ok) + { + std::cerr << "RsGxsUpdateSerialiser::serialiseGxsServerMsgUpdate() NOK" << std::endl; + } +#endif + + return ok; +} + RsGxsMsgUpdateItem* RsGxsUpdateSerialiser::deserialGxsMsgUpdate(void* data, uint32_t* size) { @@ -278,7 +540,7 @@ RsGxsMsgUpdateItem* RsGxsUpdateSerialiser::deserialGxsMsgUpdate(void* data, if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || (SERVICE_TYPE != getRsItemService(rstype)) || - (RS_PKT_SUBTYPE_NXS_SYNC_MSG != getRsItemSubType(rstype))) + (RS_PKT_SUBTYPE_GXS_MSG_UPDATE != getRsItemSubType(rstype))) { #ifdef RSSERIAL_DEBUG std::cerr << "RsGxsUpdateSerialiser::deserialGxsMsgUpdate() FAIL wrong type" << std::endl; @@ -304,10 +566,10 @@ RsGxsMsgUpdateItem* RsGxsUpdateSerialiser::deserialGxsMsgUpdate(void* data, /* skip the header */ offset += 8; - ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_HASH_SHA1, item->peerId); + ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, item->peerId); uint32_t numUpdateItems; ok &= getRawUInt32(data, *size, &offset, &(numUpdateItems)); - std::map& msgUpdateItem = item->grpUpdateTS; + std::map& msgUpdateItem = item->msgUpdateTS; std::string grpId; uint32_t updateTS; for(uint32_t i; i < numUpdateItems; i++) @@ -346,3 +608,70 @@ RsGxsMsgUpdateItem* RsGxsUpdateSerialiser::deserialGxsMsgUpdate(void* data, return item; } + +RsGxsServerMsgUpdateItem* RsGxsUpdateSerialiser::deserialGxsServerMsgUpdate(void* data, + uint32_t* size) +{ +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerMsgUpdate()" << std::endl; +#endif + /* get the type and size */ + uint32_t rstype = getRsItemId(data); + uint32_t rssize = getRsItemSize(data); + + uint32_t offset = 0; + + + if ((RS_PKT_VERSION_SERVICE != getRsItemVersion(rstype)) || + (SERVICE_TYPE != getRsItemService(rstype)) || + (RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE != getRsItemSubType(rstype))) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerMsgUpdate() FAIL wrong type" << std::endl; +#endif + return NULL; /* wrong type */ + } + + if (*size < rssize) /* check size */ + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerMsgUpdate() FAIL wrong size" << std::endl; +#endif + return NULL; /* not enough data */ + } + + /* set the packet length */ + *size = rssize; + + bool ok = true; + + RsGxsServerMsgUpdateItem* item = new RsGxsServerMsgUpdateItem(getRsItemService(rstype)); + + /* skip the header */ + offset += 8; + + ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId); + ok &= getRawUInt32(data, *size, &offset, &(item->msgUpdateTS)); + + if (offset != rssize) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerMsgUpdate() FAIL size mismatch" << std::endl; +#endif + /* error */ + delete item; + return NULL; + } + + if (!ok) + { +#ifdef RSSERIAL_DEBUG + std::cerr << "RsGxsUpdateSerialiser::deserialGxsServerMsgUpdate() NOK" << std::endl; +#endif + delete item; + return NULL; + } + + return item; +} + diff --git a/libretroshare/src/serialiser/rsgxsupdateitems.h b/libretroshare/src/serialiser/rsgxsupdateitems.h index 4b16bcb1b..bb6e3f9bb 100644 --- a/libretroshare/src/serialiser/rsgxsupdateitems.h +++ b/libretroshare/src/serialiser/rsgxsupdateitems.h @@ -37,13 +37,17 @@ #include "gxs/rsgxsdata.h" -const uint8_t RS_PKT_SUBTYPE_GXS_GRP_UPDATE = 0x0001; -const uint8_t RS_PKT_SUBTYPE_GXS_MSG_UPDATE = 0x0002; +const uint8_t RS_PKT_SUBTYPE_GXS_GRP_UPDATE = 0x0001; +const uint8_t RS_PKT_SUBTYPE_GXS_MSG_UPDATE = 0x0002; +const uint8_t RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE = 0x0004; +const uint8_t RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE = 0x0008; class RsGxsGrpUpdateItem : public RsItem { public: - RsGxsGrpUpdateItem(); - virtual ~RsGxsGrpUpdateItem(); + RsGxsGrpUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, + RS_PKT_SUBTYPE_GXS_GRP_UPDATE) + {} + virtual ~RsGxsGrpUpdateItem() {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -52,10 +56,24 @@ public: uint32_t grpUpdateTS; }; +class RsGxsServerGrpUpdateItem : public RsItem { +public: + RsGxsServerGrpUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, + RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE) + {} + virtual ~RsGxsServerGrpUpdateItem(); + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + uint32_t grpUpdateTS; +}; + class RsGxsMsgUpdateItem : public RsItem { public: - RsGxsMsgUpdateItem(); + RsGxsMsgUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, RS_PKT_SUBTYPE_GXS_MSG_UPDATE) + {} virtual ~RsGxsMsgUpdateItem(); virtual void clear(); @@ -65,6 +83,20 @@ public: std::map msgUpdateTS; }; +class RsGxsServerMsgUpdateItem : public RsItem +{ +public: + RsGxsServerMsgUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, + servType, RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE) + {} + virtual ~RsGxsServerMsgUpdateItem(); + + virtual void clear(); + virtual std::ostream &print(std::ostream &out, uint16_t indent); + + std::string grpId; + uint32_t msgUpdateTS; // the last time this group received a new msg +}; class RsGxsUpdateSerialiser : public RsSerialType @@ -89,12 +121,24 @@ private: virtual bool serialiseGxsGrpUpdate(RsGxsGrpUpdateItem *item, void *data, uint32_t *size); virtual RsGxsGrpUpdateItem* deserialGxsGrpUpddate(void *data, uint32_t *size); - /* for RS_PKT_SUBTYPE_GXS_ */ + /* for RS_PKT_SUBTYPE_GRP_SERVER_UPDATE_ITEM */ + + virtual uint32_t sizeGxsServerGrpUpdate(RsGxsServerGrpUpdateItem* item); + virtual bool serialiseGxsServerGrpUpdate(RsGxsServerGrpUpdateItem *item, void *data, uint32_t *size); + virtual RsGxsServerGrpUpdateItem* deserialGxsServerGrpUpddate(void *data, uint32_t *size); + + /* for RS_PKT_SUBTYPE_GXS_MSG_UPDATE_ITEM */ virtual uint32_t sizeGxsMsgUpdate(RsGxsMsgUpdateItem* item); virtual bool serialiseGxsMsgUpdate(RsGxsMsgUpdateItem *item, void *data, uint32_t *size); virtual RsGxsMsgUpdateItem* deserialGxsMsgUpdate(void *data, uint32_t *size); + /* for RS_PKT_SUBTYPE_GXS_SERVER_UPDATE_ITEM */ + + virtual uint32_t sizeGxsServerMsgUpdate(RsGxsServerMsgUpdateItem* item); + virtual bool serialiseGxsServerMsgUpdate(RsGxsServerMsgUpdateItem *item, void *data, uint32_t *size); + virtual RsGxsServerMsgUpdateItem* deserialGxsServerMsgUpdate(void *data, uint32_t *size); + private: const uint16_t SERVICE_TYPE; diff --git a/libretroshare/src/serialiser/rsnxsitems.cc b/libretroshare/src/serialiser/rsnxsitems.cc index df531ffca..108dba1d0 100644 --- a/libretroshare/src/serialiser/rsnxsitems.cc +++ b/libretroshare/src/serialiser/rsnxsitems.cc @@ -354,8 +354,9 @@ bool RsNxsSerialiser::serialiseNxsSyncGrp(RsNxsSyncGrp *item, void *data, uint32 ok &= setRawUInt32(data, *size, &offset, item->transactionNumber); ok &= setRawUInt8(data, *size, &offset, item->flag); - ok &= setRawUInt32(data, *size, &offset, item->syncAge); + ok &= setRawUInt32(data, *size, &offset, item->createdSince); ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_HASH_SHA1, item->syncHash); + ok &= setRawUInt32(data, *size, &offset, item->updateTS); if(offset != tlvsize){ #ifdef RSSERIAL_DEBUG @@ -403,7 +404,7 @@ bool RsNxsSerialiser::serialiseNxsTrans(RsNxsTransac *item, void *data, uint32_t ok &= setRawUInt32(data, *size, &offset, item->transactionNumber); ok &= setRawUInt16(data, *size, &offset, item->transactFlag); ok &= setRawUInt32(data, *size, &offset, item->nItems); - ok &= setRawUInt32(data, *size, &offset, item->timestamp); + ok &= setRawUInt32(data, *size, &offset, item->updateTS); @@ -501,7 +502,7 @@ bool RsNxsSerialiser::serialiseNxsSyncMsg(RsNxsSyncMsg *item, void *data, uint32 ok &= setRawUInt32(data, *size, &offset, item->transactionNumber); ok &= setRawUInt8(data, *size, &offset, item->flag); - ok &= setRawUInt32(data, *size, &offset, item->syncAge); + ok &= setRawUInt32(data, *size, &offset, item->createdSince); ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_HASH_SHA1, item->syncHash); ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId); @@ -710,8 +711,9 @@ RsNxsSyncGrp* RsNxsSerialiser::deserialNxsSyncGrp(void *data, uint32_t *size){ ok &= getRawUInt32(data, *size, &offset, &(item->transactionNumber)); ok &= getRawUInt8(data, *size, &offset, &(item->flag)); - ok &= getRawUInt32(data, *size, &offset, &(item->syncAge)); + ok &= getRawUInt32(data, *size, &offset, &(item->createdSince)); ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_HASH_SHA1, item->syncHash); + ok &= getRawUInt32(data, *size, &offset, &(item->updateTS)); if (offset != rssize) { @@ -846,7 +848,7 @@ RsNxsTransac* RsNxsSerialiser::deserialNxsTrans(void *data, uint32_t *size){ ok &= getRawUInt32(data, *size, &offset, &(item->transactionNumber)); ok &= getRawUInt16(data, *size, &offset, &(item->transactFlag)); ok &= getRawUInt32(data, *size, &offset, &(item->nItems)); - ok &= getRawUInt32(data, *size, &offset, &(item->timestamp)); + ok &= getRawUInt32(data, *size, &offset, &(item->updateTS)); if (offset != rssize) { @@ -984,9 +986,10 @@ RsNxsSyncMsg* RsNxsSerialiser::deserialNxsSyncMsg(void *data, uint32_t *size) ok &= getRawUInt32(data, *size, &offset, &(item->transactionNumber)); ok &= getRawUInt8(data, *size, &offset, &(item->flag)); - ok &= getRawUInt32(data, *size, &offset, &(item->syncAge)); + ok &= getRawUInt32(data, *size, &offset, &(item->createdSince)); ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_HASH_SHA1, item->syncHash); ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId); + ok &= getRawUInt32(data, *size, &offset, &(item->updateTS)); if (offset != rssize) { @@ -1057,6 +1060,7 @@ uint32_t RsNxsSerialiser::sizeNxsSyncGrp(RsNxsSyncGrp *item) s += 1; // flag s += 4; // sync age s += GetTlvStringSize(item->syncHash); + s += 4; // updateTS return s; } @@ -1086,6 +1090,7 @@ uint32_t RsNxsSerialiser::sizeNxsSyncMsg(RsNxsSyncMsg *item) s += 4; // age s += GetTlvStringSize(item->grpId); s += GetTlvStringSize(item->syncHash); + s += 4; // updateTS return s; } @@ -1111,7 +1116,7 @@ uint32_t RsNxsSerialiser::sizeNxsTrans(RsNxsTransac *item){ s += 4; // transaction number s += 2; // flag s += 4; // nMsgs - s += 4; // timeout + s += 4; // updateTS return s; } @@ -1141,16 +1146,18 @@ void RsNxsGrp::clear() void RsNxsSyncGrp::clear() { flag = 0; - syncAge = 0; + createdSince = 0; syncHash.clear(); + updateTS = 0; } void RsNxsSyncMsg::clear() { grpId.clear(); flag = 0; - syncAge = 0; + createdSince = 0; syncHash.clear(); + updateTS = 0; } void RsNxsSyncGrpItem::clear() @@ -1172,6 +1179,7 @@ void RsNxsSyncMsgItem::clear() void RsNxsTransac::clear(){ transactFlag = 0; nItems = 0; + updateTS = 0; timestamp = 0; transactionNumber = 0; } @@ -1185,10 +1193,11 @@ std::ostream& RsNxsSyncGrp::print(std::ostream &out, uint16_t indent) printIndent(out , int_Indent); out << "Hash: " << syncHash << std::endl; printIndent(out , int_Indent); - out << "Sync Age: " << syncAge << std::endl; + out << "Sync Age: " << createdSince << std::endl; printIndent(out , int_Indent); out << "flag" << flag << std::endl; - + printIndent(out , int_Indent); + out << "updateTS" << updateTS << std::endl; printRsItemEnd(out ,"RsNxsSyncGrp", indent); @@ -1217,11 +1226,13 @@ std::ostream& RsNxsSyncMsg::print(std::ostream &out, uint16_t indent) printIndent(out , int_Indent); out << "GrpId: " << grpId << std::endl; printIndent(out , int_Indent); - out << "syncAge: " << syncAge << std::endl; + out << "createdSince: " << createdSince << std::endl; printIndent(out , int_Indent); out << "syncHash: " << syncHash << std::endl; printIndent(out , int_Indent); out << "flag: " << flag << std::endl; + printIndent(out , int_Indent); + out << "updateTS: " << updateTS << std::endl; printRsItemEnd(out, "RsNxsSyncMsg", indent); return out; @@ -1316,6 +1327,8 @@ std::ostream& RsNxsTransac::print(std::ostream &out, uint16_t indent){ printIndent(out , int_Indent); out << "timeout: " << timestamp << std::endl; printIndent(out , int_Indent); + out << "updateTS: " << updateTS << std::endl; + printIndent(out , int_Indent); out << "transactionNumber: " << transactionNumber << std::endl; printIndent(out , int_Indent); diff --git a/libretroshare/src/serialiser/rsnxsitems.h b/libretroshare/src/serialiser/rsnxsitems.h index c15b0ce44..b7a6c2fec 100644 --- a/libretroshare/src/serialiser/rsnxsitems.h +++ b/libretroshare/src/serialiser/rsnxsitems.h @@ -100,7 +100,8 @@ public: virtual std::ostream &print(std::ostream &out, uint16_t indent); uint8_t flag; // advises whether to use sync hash - uint32_t syncAge; // how far back to sync data + uint32_t createdSince; // how far back to sync data + uint32_t updateTS; // time of last group update std::string syncHash; // use to determine if changes that have occured since last hash @@ -146,6 +147,9 @@ public: uint16_t transactFlag; uint32_t nItems; + uint32_t updateTS; + + // not serialised uint32_t timestamp; }; @@ -235,7 +239,8 @@ public: std::string grpId; uint8_t flag; - uint32_t syncAge; + uint32_t createdSince; + uint32_t updateTS; // time of last update std::string syncHash; }; From 6cbb3a65f8918b42e2983b73c4d3f574a7073709 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 17 Nov 2013 23:15:39 +0000 Subject: [PATCH 21/29] fixed linker issue, and bad initialisation of pointer bug git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6904 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgxsnetservice.cc | 5 +++- .../src/serialiser/rsgxsupdateitems.cc | 27 +++++++++++++++++++ .../src/serialiser/rsgxsupdateitems.h | 6 ++--- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 3485d2863..877d437b4 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -45,7 +45,7 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, : p3Config(servType), p3ThreadedService(servType), mTransactionTimeOut(TRANSAC_TIMEOUT), mServType(servType), mDataStore(gds), mTransactionN(0), mObserver(nxsObs), mNxsMutex("RsGxsNetService"), mNetMgr(netMgr), mSYNC_PERIOD(SYNC_PERIOD), - mSyncTs(0), mReputations(reputations), mCircles(circles), mGrpAutoSync(grpAutoSync) + mSyncTs(0), mReputations(reputations), mCircles(circles), mGrpAutoSync(grpAutoSync), mGrpServerUpdateItem(NULL) { addSerialType(new RsNxsSerialiser(mServType)); @@ -1972,8 +1972,11 @@ void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) std::string peer = item->PeerId(); // don't sync if you have no new updates for this peer + if(mGrpServerUpdateItem) + { if(item->updateTS >= mGrpServerUpdateItem->grpUpdateTS && item->updateTS != 0) return; + } std::map grp; mDataStore->retrieveGxsGrpMetaData(grp); diff --git a/libretroshare/src/serialiser/rsgxsupdateitems.cc b/libretroshare/src/serialiser/rsgxsupdateitems.cc index fdccdbeae..60f637f06 100644 --- a/libretroshare/src/serialiser/rsgxsupdateitems.cc +++ b/libretroshare/src/serialiser/rsgxsupdateitems.cc @@ -51,8 +51,35 @@ void RsGxsMsgUpdateItem::clear() std::ostream& RsGxsMsgUpdateItem::print(std::ostream& out, uint16_t indent) { + } + + +void RsGxsServerMsgUpdateItem::clear() +{ + msgUpdateTS = 0; + grpId.clear(); +} + +std::ostream& RsGxsServerMsgUpdateItem::print(std::ostream& out, uint16_t indent) +{ + +} + + +void RsGxsServerGrpUpdateItem::clear() +{ + grpUpdateTS = 0; +} + +std::ostream& RsGxsServerGrpUpdateItem::print(std::ostream& out, uint16_t indent) +{ + +} + + + uint32_t RsGxsUpdateSerialiser::size(RsItem* item) { RsGxsMsgUpdateItem* mui = NULL; diff --git a/libretroshare/src/serialiser/rsgxsupdateitems.h b/libretroshare/src/serialiser/rsgxsupdateitems.h index bb6e3f9bb..763c51516 100644 --- a/libretroshare/src/serialiser/rsgxsupdateitems.h +++ b/libretroshare/src/serialiser/rsgxsupdateitems.h @@ -61,7 +61,7 @@ public: RsGxsServerGrpUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE) {} - virtual ~RsGxsServerGrpUpdateItem(); + virtual ~RsGxsServerGrpUpdateItem() {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -74,7 +74,7 @@ class RsGxsMsgUpdateItem : public RsItem public: RsGxsMsgUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, RS_PKT_SUBTYPE_GXS_MSG_UPDATE) {} - virtual ~RsGxsMsgUpdateItem(); + virtual ~RsGxsMsgUpdateItem() {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); @@ -89,7 +89,7 @@ public: RsGxsServerMsgUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE) {} - virtual ~RsGxsServerMsgUpdateItem(); + virtual ~RsGxsServerMsgUpdateItem() {} virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); From 24839ee2373e09ab25e29f8f7a2f9500621db493 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sat, 23 Nov 2013 23:39:55 +0000 Subject: [PATCH 22/29] decided to use polling and a database trigger to maintain consistentency of last group post for database sync - db reset done, you will lose your current gxs data git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6906 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsdataservice.cc | 22 +++++- libretroshare/src/gxs/rsgenexchange.cc | 3 + libretroshare/src/gxs/rsgxsdata.cc | 2 + libretroshare/src/gxs/rsgxsdata.h | 6 +- libretroshare/src/gxs/rsgxsnetservice.cc | 94 +++++++++++++++--------- libretroshare/src/gxs/rsgxsnetservice.h | 2 + libretroshare/src/rsserver/rsinit.cc | 2 +- 7 files changed, 92 insertions(+), 39 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index c0f7082a7..132e366bd 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -34,6 +34,8 @@ #define MSG_TABLE_NAME std::string("MESSAGES") #define GRP_TABLE_NAME std::string("GROUPS") +#define GRP_LAST_POST_UPDATE_TRIGGER std::string("LAST_POST_UPDATE") + // generic #define KEY_NXS_FILE std::string("nxsFile") @@ -48,6 +50,7 @@ #define KEY_NXS_META std::string("meta") #define KEY_NXS_SERV_STRING std::string("serv_str") #define KEY_NXS_HASH std::string("hash") +#define KEY_RECV_TS std::string("recv_time_stamp") // grp table columns @@ -111,6 +114,7 @@ #define COL_GRP_INTERN_CIRCLE 18 #define COL_GRP_ORIGINATOR 19 #define COL_GRP_AUTHEN_FLAGS 20 +#define COL_GRP_RECV_TS 21 // msg col numbers @@ -122,6 +126,7 @@ #define COL_THREAD_ID 11 #define COL_MSG_NAME 12 #define COL_MSG_SERV_STRING 13 +#define COL_MSG_RECV_TS 14 // generic meta shared col numbers #define COL_GRP_ID 0 @@ -154,7 +159,7 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d msgMetaColumns.push_back(KEY_SIGN_SET); msgMetaColumns.push_back(KEY_NXS_IDENTITY); msgMetaColumns.push_back(KEY_NXS_HASH); msgMetaColumns.push_back(KEY_MSG_ID); msgMetaColumns.push_back(KEY_ORIG_MSG_ID); msgMetaColumns.push_back(KEY_MSG_STATUS); msgMetaColumns.push_back(KEY_CHILD_TS); msgMetaColumns.push_back(KEY_MSG_PARENT_ID); msgMetaColumns.push_back(KEY_MSG_THREAD_ID); - msgMetaColumns.push_back(KEY_MSG_NAME); msgMetaColumns.push_back(KEY_NXS_SERV_STRING); + msgMetaColumns.push_back(KEY_MSG_NAME); msgMetaColumns.push_back(KEY_NXS_SERV_STRING); msgMetaColumns.push_back(KEY_RECV_TS); // for retrieving actual data msgColumns.push_back(KEY_GRP_ID); msgColumns.push_back(KEY_NXS_FILE); msgColumns.push_back(KEY_NXS_FILE_OFFSET); @@ -168,7 +173,7 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d grpMetaColumns.push_back(KEY_GRP_LAST_POST); grpMetaColumns.push_back(KEY_ORIG_GRP_ID); grpMetaColumns.push_back(KEY_NXS_SERV_STRING); grpMetaColumns.push_back(KEY_GRP_SIGN_FLAGS); grpMetaColumns.push_back(KEY_GRP_CIRCLE_ID); grpMetaColumns.push_back(KEY_GRP_CIRCLE_TYPE); grpMetaColumns.push_back(KEY_GRP_INTERNAL_CIRCLE); grpMetaColumns.push_back(KEY_GRP_ORIGINATOR); - grpMetaColumns.push_back(KEY_GRP_AUTHEN_FLAGS); + grpMetaColumns.push_back(KEY_GRP_AUTHEN_FLAGS); grpMetaColumns.push_back(KEY_RECV_TS); // for retrieving actual grp data grpColumns.push_back(KEY_GRP_ID); grpColumns.push_back(KEY_NXS_FILE); grpColumns.push_back(KEY_NXS_FILE_OFFSET); @@ -212,6 +217,7 @@ void RsDataService::initialise(){ KEY_MSG_NAME + " TEXT," + KEY_NXS_SERV_STRING + " TEXT," + KEY_NXS_HASH + " TEXT," + + KEY_RECV_TS + " INT," + KEY_NXS_FILE_LEN + " INT);"); // create table for grp data @@ -240,8 +246,15 @@ void RsDataService::initialise(){ KEY_GRP_INTERNAL_CIRCLE + " TEXT," + KEY_GRP_ORIGINATOR + " TEXT," + KEY_NXS_HASH + " TEXT," + + KEY_RECV_TS + " INT," + KEY_SIGN_SET + " BLOB);"); + mDb->execSQL("CREATE TRIGGER " + GRP_LAST_POST_UPDATE_TRIGGER + + "UPDATE OF " + KEY_RECV_TS + " ON " + MSG_TABLE_NAME + + std::string("BEGIN ") + + "UPDATE " + GRP_TABLE_NAME + "SET " + KEY_GRP_LAST_POST + "= new." + + KEY_RECV_TS + ";" + + std::string("END;")); } RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c) @@ -293,6 +306,7 @@ RsGxsGrpMetaData* RsDataService::locked_getGrpMeta(RetroCursor &c) c.getString(COL_GRP_INTERN_CIRCLE, grpMeta->mInternalCircle); c.getString(COL_GRP_ORIGINATOR, grpMeta->mOriginator); grpMeta->mAuthenFlags = c.getInt32(COL_GRP_AUTHEN_FLAGS); + grpMeta->mRecvTS = c.getInt32(COL_GRP_RECV_TS); if(ok) @@ -379,6 +393,7 @@ RsGxsMsgMetaData* RsDataService::locked_getMsgMeta(RetroCursor &c) c.getString(COL_MSG_NAME, msgMeta->mMsgName); c.getString(COL_MSG_SERV_STRING, msgMeta->mServiceString); c.getString(COL_HASH, msgMeta->mHash); + msgMeta->recvTS = c.getInt32(COL_MSG_RECV_TS); offset = 0; data = (char*)c.getData(COL_SIGN_SET, data_len); @@ -491,6 +506,7 @@ int RsDataService::storeMessage(std::map &msg) cv.put(KEY_GRP_ID, msgMetaPtr->mGroupId); cv.put(KEY_NXS_SERV_STRING, msgMetaPtr->mServiceString); cv.put(KEY_NXS_HASH, msgMetaPtr->mHash); + cv.put(KEY_RECV_TS, (int32_t)msgMetaPtr->recvTS); char signSetData[msgMetaPtr->signSet.TlvSize()]; @@ -598,6 +614,7 @@ int RsDataService::storeGroup(std::map &grp) cv.put(KEY_GRP_ORIGINATOR, grpMetaPtr->mOriginator); cv.put(KEY_GRP_AUTHEN_FLAGS, (int32_t)grpMetaPtr->mAuthenFlags); cv.put(KEY_NXS_HASH, grpMetaPtr->mHash); + cv.put(KEY_RECV_TS, (int32_t)grpMetaPtr->mRecvTS); if(! (grpMetaPtr->mAuthorId.empty()) ){ cv.put(KEY_NXS_IDENTITY, grpMetaPtr->mAuthorId); @@ -1117,6 +1134,7 @@ int RsDataService::resetDataStore() mDb->execSQL("DROP TABLE " + MSG_TABLE_NAME); mDb->execSQL("DROP TABLE " + GRP_TABLE_NAME); + mDb->execSQL("DROP TRIGGER " + GRP_LAST_POST_UPDATE_TRIGGER); } // recreate database diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index ffa2a3000..316832b90 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1789,6 +1789,7 @@ void RsGenExchange::publishMsgs() msg->metaData->mMsgStatus = GXS_SERV::GXS_MSG_STATUS_UNPROCESSED | GXS_SERV::GXS_MSG_STATUS_UNREAD; msgId = msg->msgId; grpId = msg->grpId; + msg->metaData->recvTS = time(NULL); computeHash(msg->msg, msg->metaData->mHash); mDataAccess->addMsgData(msg); msgChangeMap[grpId].push_back(msgId); @@ -2079,6 +2080,7 @@ void RsGenExchange::publishGrps() { grpId = grp->grpId; computeHash(grp->grp, grp->metaData->mHash); + grp->metaData->mRecvTS = time(NULL); if(ggps.mIsUpdate) mDataAccess->updateGroupData(grp); @@ -2407,6 +2409,7 @@ void RsGenExchange::processRecvdGroups() // now check if group already existss if(std::find(existingGrpIds.begin(), existingGrpIds.end(), grp->grpId) == existingGrpIds.end()) { + meta->mRecvTS = time(NULL); if(meta->mCircleType == GXS_CIRCLE_TYPE_YOUREYESONLY) meta->mOriginator = grp->PeerId(); diff --git a/libretroshare/src/gxs/rsgxsdata.cc b/libretroshare/src/gxs/rsgxsdata.cc index fbccd8af2..5de64b35d 100644 --- a/libretroshare/src/gxs/rsgxsdata.cc +++ b/libretroshare/src/gxs/rsgxsdata.cc @@ -74,6 +74,7 @@ void RsGxsGrpMetaData::clear(){ mOriginator.clear(); mCircleType = 0; mAuthenFlags = 0; + mRecvTS = 0; } @@ -196,6 +197,7 @@ void RsGxsMsgMetaData::clear() mMsgFlags = 0; mMsgStatus = 0; mChildTs = 0; + recvTS = 0; } bool RsGxsMsgMetaData::serialise(void *data, uint32_t *size) diff --git a/libretroshare/src/gxs/rsgxsdata.h b/libretroshare/src/gxs/rsgxsdata.h index 9a94cc8a0..5c17a5cd4 100644 --- a/libretroshare/src/gxs/rsgxsdata.h +++ b/libretroshare/src/gxs/rsgxsdata.h @@ -70,17 +70,16 @@ public: uint32_t mPop; // HOW DO WE DO THIS NOW. uint32_t mMsgCount; // ??? - time_t mLastPost; // ??? + uint32_t mLastPost; // ??? uint32_t mGroupStatus; + uint32_t mRecvTS; std::string mOriginator; std::string mInternalCircle; std::string mHash; }; - - class RsGxsMsgMetaData { public: @@ -114,6 +113,7 @@ public: uint32_t mMsgStatus; time_t mChildTs; + uint32_t recvTS; std::string mHash; bool validated; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 877d437b4..1ec276391 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -877,6 +877,8 @@ void RsGxsNetService::run(){ Sleep((int) (timeDelta * 1000)); #endif + updateServerSyncTS(); + // process active transactions processTransactions(); @@ -891,6 +893,49 @@ void RsGxsNetService::run(){ } } +void RsGxsNetService::updateServerSyncTS() +{ + RsStackMutex stack(mNxsMutex); + + std::map gxsMap; + + // retrieve all grps and update TS + mDataStore->retrieveGxsGrpMetaData(gxsMap); + std::map::iterator mit = gxsMap.begin(); + + // as a grp list server also note this is the latest item you have + if(mGrpServerUpdateItem == NULL) + { + mGrpServerUpdateItem = new RsGxsServerGrpUpdateItem(mServType); + } + + for(; mit != gxsMap.end(); mit++) + { + const RsGxsGroupId& grpId = mit->first; + RsGxsGrpMetaData* grpMeta = mit->second; + ServerMsgMap::iterator mapIT = mServerMsgUpdateMap.find(grpId); + RsGxsServerMsgUpdateItem* msui = NULL; + + if(mapIT == mServerMsgUpdateMap.end()) + { + msui = new RsGxsServerMsgUpdateItem(mServType); + msui->grpId = grpMeta->mGroupId; + mServerMsgUpdateMap.insert(std::make_pair(msui->grpId, msui)); + }else + { + msui = mapIT->second; + } + + msui->msgUpdateTS = grpMeta->mLastPost; + + // this might be very inefficient with time + if(grpMeta->mRecvTS > mGrpServerUpdateItem->grpUpdateTS) + mGrpServerUpdateItem->grpUpdateTS = grpMeta->mRecvTS; + } + + freeAndClearContainerResource, + RsGxsGrpMetaData*>(gxsMap); +} bool RsGxsNetService::locked_checkTransacTimedOut(NxsTransaction* tr) { return tr->mTimeOut < ((uint32_t) time(NULL)); @@ -1156,30 +1201,27 @@ void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) // notify listener of grps mObserver->notifyNewGroups(grps); - // now note this as the latest you've received from this peer - std::string peerFrom = tr->mTransaction->PeerId(); - uint32_t updateTS = tr->mTransaction->updateTS; + // now note this as the latest you've received from this peer + std::string peerFrom = tr->mTransaction->PeerId(); + uint32_t updateTS = tr->mTransaction->updateTS; - ClientGrpMap::iterator it = mClientGrpUpdateMap.find(peerFrom); + ClientGrpMap::iterator it = mClientGrpUpdateMap.find(peerFrom); - RsGxsGrpUpdateItem* item = NULL; + RsGxsGrpUpdateItem* item = NULL; - if(it != mClientGrpUpdateMap.end()) - { - item = it->second; - }else - { - item = new RsGxsGrpUpdateItem(mServType); - } + if(it != mClientGrpUpdateMap.end()) + { + item = it->second; + }else + { + item = new RsGxsGrpUpdateItem(mServType); + mClientGrpUpdateMap.insert( + std::make_pair(peerFrom, item)); + } - item->grpUpdateTS = updateTS; - item->peerId = peerFrom; + item->grpUpdateTS = updateTS; + item->peerId = peerFrom; - mClientGrpUpdateMap.insert( - std::make_pair(peerFrom, item)); - - // as a grp list server also note this is the latest item you have - mGrpServerUpdateItem->grpUpdateTS = updateTS; }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) { @@ -1261,20 +1303,6 @@ void RsGxsNetService::locked_doMsgUpdateWork(const RsNxsTransac *nxsTrans, const mui->msgUpdateTS[grpId] = nxsTrans->updateTS; mui->peerId = peerFrom; - ServerMsgMap::iterator mit = mServerMsgUpdateMap.find(grpId); - RsGxsServerMsgUpdateItem* msui = NULL; - if(mit != mServerMsgUpdateMap.end()) - { - msui = mit->second; - } - else - { - msui = new RsGxsServerMsgUpdateItem(mServType); - mServerMsgUpdateMap.insert(std::make_pair(grpId, msui)); - } - - msui->grpId = grpId; - msui->msgUpdateTS = nxsTrans->updateTS; } void RsGxsNetService::locked_processCompletedOutgoingTrans(NxsTransaction* tr) diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 0f7f95774..e1c27df83 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -327,6 +327,8 @@ private: void locked_doMsgUpdateWork(const RsNxsTransac* nxsTrans, const std::string& grpId); + void updateServerSyncTS(); + private: typedef std::vector GrpFragments; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 3c8f54155..6122adc4a 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -2227,7 +2227,7 @@ int RsServer::StartupRetroShare() std::string currGxsDir = RsInitConfig::configDir + "/GXS_phase2"; #ifdef GXS_DEV_TESTNET // Different Directory for testing. - currGxsDir += "_TESTNET5"; + currGxsDir += "_TESTNET6"; #endif bool cleanUpGxsDir = false; From 1cdf329bb1203a86b92ea616f9c81530af7fdadb Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 24 Nov 2013 12:37:17 +0000 Subject: [PATCH 23/29] fix for trigger statement, and readjusted algo to use self gen-exchange generated time stamps git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6907 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsdataservice.cc | 8 +++---- libretroshare/src/gxs/rsgxsnetservice.cc | 30 +++++++++++++++++++++--- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 132e366bd..828fac742 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -250,10 +250,10 @@ void RsDataService::initialise(){ KEY_SIGN_SET + " BLOB);"); mDb->execSQL("CREATE TRIGGER " + GRP_LAST_POST_UPDATE_TRIGGER + - "UPDATE OF " + KEY_RECV_TS + " ON " + MSG_TABLE_NAME + - std::string("BEGIN ") + - "UPDATE " + GRP_TABLE_NAME + "SET " + KEY_GRP_LAST_POST + "= new." - + KEY_RECV_TS + ";" + " UPDATE OF " + KEY_RECV_TS + " ON " + MSG_TABLE_NAME + + std::string(" BEGIN ") + + " UPDATE " + GRP_TABLE_NAME + " SET " + KEY_GRP_LAST_POST + "= new." + + KEY_RECV_TS + " WHERE " + KEY_GRP_ID + "=old." + KEY_GRP_ID + ";" + std::string("END;")); } diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 1ec276391..648d83a8e 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -868,6 +868,7 @@ void RsGxsNetService::run(){ double timeDelta = 0.2; + int updateCounter = 0; while(isRunning()){ @@ -877,7 +878,13 @@ void RsGxsNetService::run(){ Sleep((int) (timeDelta * 1000)); #endif - updateServerSyncTS(); + if(updateCounter == 3) + { + updateServerSyncTS(); + updateCounter = 0; + } + else + updateCounter++; // process active transactions processTransactions(); @@ -1729,12 +1736,15 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) return; } + uint32_t updateTS = 0; + if(mGrpServerUpdateItem) + updateTS = mGrpServerUpdateItem->grpUpdateTS; RsNxsTransac* ntr = new RsNxsTransac(mServType); ntr->transactionNumber = transN; ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_GRPS; - ntr->updateTS = time(NULL); + ntr->updateTS = updateTS; ntr->nItems = grps.size(); ntr->PeerId(tr->mTransaction->PeerId()); @@ -1850,12 +1860,17 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) return; } + std::string grpId = ""; + for(;lit != tr->mItems.end(); lit++) { RsNxsSyncMsgItem* item = dynamic_cast(*lit); if (item) { msgIds[item->grpId].push_back(item->msgId); + + if(grpId.empty()) + grpId = item->grpId; } else { @@ -1912,11 +1927,18 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) return; } + uint32_t updateTS = 0; + + ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(grpId); + + if(cit != mServerMsgUpdateMap.end()) + updateTS = cit->second->msgUpdateTS; + RsNxsTransac* ntr = new RsNxsTransac(mServType); ntr->transactionNumber = transN; ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_MSGS; - ntr->updateTS = time(NULL); + ntr->updateTS = updateTS; ntr->nItems = msgSize; ntr->PeerId(peerId); @@ -2003,7 +2025,9 @@ void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) if(mGrpServerUpdateItem) { if(item->updateTS >= mGrpServerUpdateItem->grpUpdateTS && item->updateTS != 0) + { return; + } } std::map grp; From db30ea3ad43750a99e4ee346a9be2a477631baea Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 24 Nov 2013 12:38:21 +0000 Subject: [PATCH 24/29] forgot serialiser bad init git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6908 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/serialiser/rsgxsupdateitems.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/serialiser/rsgxsupdateitems.h b/libretroshare/src/serialiser/rsgxsupdateitems.h index 763c51516..77e3b2476 100644 --- a/libretroshare/src/serialiser/rsgxsupdateitems.h +++ b/libretroshare/src/serialiser/rsgxsupdateitems.h @@ -46,7 +46,7 @@ class RsGxsGrpUpdateItem : public RsItem { public: RsGxsGrpUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, RS_PKT_SUBTYPE_GXS_GRP_UPDATE) - {} + {clear();} virtual ~RsGxsGrpUpdateItem() {} virtual void clear(); @@ -60,7 +60,7 @@ class RsGxsServerGrpUpdateItem : public RsItem { public: RsGxsServerGrpUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE) - {} + { clear();} virtual ~RsGxsServerGrpUpdateItem() {} virtual void clear(); @@ -73,7 +73,7 @@ class RsGxsMsgUpdateItem : public RsItem { public: RsGxsMsgUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, RS_PKT_SUBTYPE_GXS_MSG_UPDATE) - {} + { clear();} virtual ~RsGxsMsgUpdateItem() {} virtual void clear(); @@ -88,7 +88,7 @@ class RsGxsServerMsgUpdateItem : public RsItem public: RsGxsServerMsgUpdateItem(uint16_t servType) : RsItem(RS_PKT_VERSION_SERVICE, servType, RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE) - {} + { clear();} virtual ~RsGxsServerMsgUpdateItem() {} virtual void clear(); From 34604577717d8fe3627368bc0ba8142ab7febfe3 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Mon, 25 Nov 2013 22:28:17 +0000 Subject: [PATCH 25/29] nxs grp sync optimisation now working - msg still needs a bit of work - bug fix in grp sync found (reputation logic wrong) - bug in nxssyncmsg serialisation from recent work git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6909 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgxsnetservice.cc | 56 ++++++++++++++-------- libretroshare/src/gxs/rsgxsnetservice.h | 3 ++ libretroshare/src/serialiser/rsnxsitems.cc | 1 + 3 files changed, 41 insertions(+), 19 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 648d83a8e..e7643c7e1 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -1503,7 +1503,7 @@ void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) GixsReputation rep; mReputations->getReputation(syncItem->authorId, rep); - if(rep.score > GIXS_CUT_OFF) + if(rep.score > GIXS_CUT_OFF || syncItem->authorId.empty()) { RsNxsSyncMsgItem* msgItem = new RsNxsSyncMsgItem(mServType); msgItem->grpId = grpId; @@ -2014,21 +2014,31 @@ void RsGxsNetService::locked_pushGrpRespFromList(std::list& respList locked_addTransaction(tr); } +bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncGrp *item) +{ + // don't sync if you have no new updates for this peer + if(mGrpServerUpdateItem) + { + if(item->updateTS >= mGrpServerUpdateItem->grpUpdateTS && item->updateTS != 0) + { + return false; + } + } + + return true; +} + void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) { RsStackMutex stack(mNxsMutex); + if(!locked_CanReceiveUpdate(item)) + return; + std::string peer = item->PeerId(); - // don't sync if you have no new updates for this peer - if(mGrpServerUpdateItem) - { - if(item->updateTS >= mGrpServerUpdateItem->grpUpdateTS && item->updateTS != 0) - { - return; - } - } + std::map grp; mDataStore->retrieveGxsGrpMetaData(grp); @@ -2083,6 +2093,8 @@ void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) return; } + + bool RsGxsNetService::canSendGrpId(const std::string& sslId, RsGxsGrpMetaData& grpMeta, std::vector& toVet) { // first do the simple checks @@ -2138,22 +2150,28 @@ bool RsGxsNetService::canSendGrpId(const std::string& sslId, RsGxsGrpMetaData& g return true; } +bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncMsg *item) +{ + ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId); + + if(cit != mServerMsgUpdateMap.end()) + { + const RsGxsServerMsgUpdateItem *msui = cit->second; + + if(item->updateTS >= msui->msgUpdateTS && item->updateTS != 0) + return false; + } + return true; +} void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsg* item) { RsStackMutex stack(mNxsMutex); + if(!locked_CanReceiveUpdate(item)) + return; + const std::string& peer = item->PeerId(); - ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId); - - if(cit != mServerMsgUpdateMap.end()) - { - const RsGxsServerMsgUpdateItem *msui = cit->second; - - if(item->updateTS > msui->msgUpdateTS && item->updateTS != 0) - return; - } - GxsMsgMetaResult metaResult; GxsMsgReq req; diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index e1c27df83..d72773f91 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -329,6 +329,9 @@ private: void updateServerSyncTS(); + bool locked_CanReceiveUpdate(const RsNxsSyncGrp* item); + bool locked_CanReceiveUpdate(const RsNxsSyncMsg* item); + private: typedef std::vector GrpFragments; diff --git a/libretroshare/src/serialiser/rsnxsitems.cc b/libretroshare/src/serialiser/rsnxsitems.cc index 108dba1d0..f855a57c6 100644 --- a/libretroshare/src/serialiser/rsnxsitems.cc +++ b/libretroshare/src/serialiser/rsnxsitems.cc @@ -505,6 +505,7 @@ bool RsNxsSerialiser::serialiseNxsSyncMsg(RsNxsSyncMsg *item, void *data, uint32 ok &= setRawUInt32(data, *size, &offset, item->createdSince); ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_HASH_SHA1, item->syncHash); ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId); + ok &= setRawUInt32(data, *size, &offset, item->updateTS); if(offset != tlvsize){ #ifdef RSSERIAL_DEBUG From e50f0175720969196a8819f7742071ad6b358bb5 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Wed, 27 Nov 2013 21:54:05 +0000 Subject: [PATCH 26/29] Fixes of msg sync optimisation - msg sync opt now working git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6913 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsdataservice.cc | 4 ++-- libretroshare/src/gxs/rsgenexchange.cc | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 828fac742..4ca2aaa1f 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -250,10 +250,10 @@ void RsDataService::initialise(){ KEY_SIGN_SET + " BLOB);"); mDb->execSQL("CREATE TRIGGER " + GRP_LAST_POST_UPDATE_TRIGGER + - " UPDATE OF " + KEY_RECV_TS + " ON " + MSG_TABLE_NAME + + " INSERT ON " + MSG_TABLE_NAME + std::string(" BEGIN ") + " UPDATE " + GRP_TABLE_NAME + " SET " + KEY_GRP_LAST_POST + "= new." - + KEY_RECV_TS + " WHERE " + KEY_GRP_ID + "=old." + KEY_GRP_ID + ";" + + KEY_RECV_TS + " WHERE " + KEY_GRP_ID + "=new." + KEY_GRP_ID + ";" + std::string("END;")); } diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 316832b90..d10740051 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -2303,6 +2303,7 @@ void RsGenExchange::processRecvdMessages() if(validated_entry != mMsgPendingValidate.end()) mMsgPendingValidate.erase(validated_entry); computeHash(msg->msg, meta->mHash); + meta->recvTS = time(NULL); } } else From 972674173d176fb140f03943a0eac82ac7437589 Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Mon, 9 Dec 2013 22:24:49 +0000 Subject: [PATCH 27/29] added tests for gxs update items - need to fix to allow saving sync config git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6936 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- .../tests/serialiser/rsgxsupdateitem_test.cc | 110 ++++++++++++++++++ .../tests/serialiser/rsgxsupdateitem_test.h | 25 ++++ 2 files changed, 135 insertions(+) create mode 100644 libretroshare/src/tests/serialiser/rsgxsupdateitem_test.cc create mode 100644 libretroshare/src/tests/serialiser/rsgxsupdateitem_test.h diff --git a/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.cc b/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.cc new file mode 100644 index 000000000..395e5b7c3 --- /dev/null +++ b/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.cc @@ -0,0 +1,110 @@ +/* + * rsgxsupdateitem_test.cc + * + * Created on: 9 Dec 2013 + * Author: crispy + */ + +#include "support.h" +#include "rsgxsupdateitem_test.h" + +INITTEST(); + +RsSerialType* init_item(RsGxsGrpUpdateItem& i) +{ + i.clear(); + i.grpUpdateTS = rand()%2424; + randString(SHORT_STR, i.peerId); + return new RsGxsUpdateSerialiser(0); +} + +RsSerialType* init_item(RsGxsMsgUpdateItem& i) +{ + i.clear(); + randString(SHORT_STR, i.peerId); + int numUpdates = rand()%123; + + std::string peer; + for(int j=0; j < numUpdates; j++) + { + randString(SHORT_STR, peer); + i.msgUpdateTS.insert(std::make_pair(peer, rand()%45)); + } + + return new RsGxsUpdateSerialiser(0); +} + +RsSerialType* init_item(RsGxsServerGrpUpdateItem& i) +{ + i.clear(); + i.grpUpdateTS = rand()%2424; + + return new RsGxsUpdateSerialiser(0); +} + +RsSerialType* init_item(RsGxsServerMsgUpdateItem& i) +{ + i.clear(); + randString(SHORT_STR, i.grpId); + i.msgUpdateTS = rand()%4252; + return new RsGxsUpdateSerialiser(0); +} + +bool operator ==(const RsGxsGrpUpdateItem& l, const RsGxsGrpUpdateItem& r) +{ + bool ok = l.grpUpdateTS == r.grpUpdateTS; + ok &= l.peerId == r.peerId; + + return ok; +} + +bool operator ==(const RsGxsMsgUpdateItem& l, const RsGxsMsgUpdateItem& r) +{ + bool ok = l.peerId == r.peerId; + + const std::map& lUp = l.msgUpdateTS, rUp = r.msgUpdateTS; + + ok &= lUp.size() == rUp.size(); + + std::map::const_iterator lit = lUp.begin(), rit; + + for(; lit != lUp.end(); lit++) + { + std::string key = lit->first; + if((rit = rUp.find(key)) != rUp.end()) + ok &= lit->second == rit->second; + else + return false; + } + + return ok; +} + +bool operator ==(const RsGxsServerGrpUpdateItem& l, + const RsGxsServerGrpUpdateItem& r) +{ + return l.grpUpdateTS == r.grpUpdateTS; +} + +bool operator ==(const RsGxsServerMsgUpdateItem& l, + const RsGxsServerMsgUpdateItem& r) +{ + bool ok = l.grpId == r.grpId; + ok &= l.msgUpdateTS == r.msgUpdateTS; + return ok; +} + + +int main() +{ + std::cerr << "RsGxsUpdateItem Tests" << std::endl; + + test_RsItem(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsGxsGrpUpdateItem"); + test_RsItem(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsGxsMsgUpdateItem"); + test_RsItem(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsGxsServerGrpUpdateItem"); + test_RsItem(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); REPORT("Serialise/Deserialise RsGxsServerMsgUpdateItem"); + + FINALREPORT("RsGxsUpdateItem Tests"); + + return TESTRESULT(); +} diff --git a/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.h b/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.h new file mode 100644 index 000000000..57a686241 --- /dev/null +++ b/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.h @@ -0,0 +1,25 @@ +/* + * rsgxsupdateitem_test.h + * + * Created on: 9 Dec 2013 + * Author: crispy + */ + +#ifndef RSGXSUPDATEITEM_TEST_H_ +#define RSGXSUPDATEITEM_TEST_H_ + +#include "serialiser/rsgxsupdateitems.h" +#include "support.h" + + +RsSerialType* init_item(RsGxsGrpUpdateItem& i); +RsSerialType* init_item(RsGxsMsgUpdateItem& i); +RsSerialType* init_item(RsGxsServerGrpUpdateItem& i); +RsSerialType* init_item(RsGxsServerMsgUpdateItem& i); + +bool operator==(const RsGxsGrpUpdateItem& l, const RsGxsGrpUpdateItem& r); +bool operator==(const RsGxsMsgUpdateItem& l, const RsGxsMsgUpdateItem& r); +bool operator==(const RsGxsServerGrpUpdateItem& l, const RsGxsServerGrpUpdateItem& r); +bool operator==(const RsGxsServerMsgUpdateItem& l, const RsGxsServerMsgUpdateItem& r); + +#endif /* RSGXSUPDATEITEM_TEST_H_ */ From dab782fbaa4be31db93e513dc16ca3dd754abbaa Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 15 Dec 2013 20:11:39 +0000 Subject: [PATCH 28/29] Fixes for gxsupdate item serialisation - now works git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6947 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/serialiser/rsgxsupdateitems.cc | 6 +++--- .../src/tests/serialiser/rsgxsupdateitem_test.cc | 8 ++++---- libretroshare/src/tests/serialiser/rsnxsitems_test.cc | 8 ++++---- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/libretroshare/src/serialiser/rsgxsupdateitems.cc b/libretroshare/src/serialiser/rsgxsupdateitems.cc index 60f637f06..26bcff9c3 100644 --- a/libretroshare/src/serialiser/rsgxsupdateitems.cc +++ b/libretroshare/src/serialiser/rsgxsupdateitems.cc @@ -325,8 +325,8 @@ RsGxsGrpUpdateItem* RsGxsUpdateSerialiser::deserialGxsGrpUpddate(void* data, /* skip the header */ offset += 8; - ok &= getRawUInt32(data, *size, &offset, &(item->grpUpdateTS)); ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, item->peerId); + ok &= getRawUInt32(data, *size, &offset, &(item->grpUpdateTS)); if (offset != rssize) { @@ -482,7 +482,7 @@ bool RsGxsUpdateSerialiser::serialiseGxsMsgUpdate(RsGxsMsgUpdateItem* item, for(; cit != msgUpdateTS.end(); cit++) { - ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_PEERID, cit->first); + ok &= SetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, cit->first); ok &= setRawUInt32(data, *size, &offset, cit->second); } @@ -599,7 +599,7 @@ RsGxsMsgUpdateItem* RsGxsUpdateSerialiser::deserialGxsMsgUpdate(void* data, std::map& msgUpdateItem = item->msgUpdateTS; std::string grpId; uint32_t updateTS; - for(uint32_t i; i < numUpdateItems; i++) + for(uint32_t i = 0; i < numUpdateItems; i++) { ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, grpId); diff --git a/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.cc b/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.cc index 395e5b7c3..aa858daf7 100644 --- a/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.cc +++ b/libretroshare/src/tests/serialiser/rsgxsupdateitem_test.cc @@ -15,7 +15,7 @@ RsSerialType* init_item(RsGxsGrpUpdateItem& i) i.clear(); i.grpUpdateTS = rand()%2424; randString(SHORT_STR, i.peerId); - return new RsGxsUpdateSerialiser(0); + return new RsGxsUpdateSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); } RsSerialType* init_item(RsGxsMsgUpdateItem& i) @@ -31,7 +31,7 @@ RsSerialType* init_item(RsGxsMsgUpdateItem& i) i.msgUpdateTS.insert(std::make_pair(peer, rand()%45)); } - return new RsGxsUpdateSerialiser(0); + return new RsGxsUpdateSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); } RsSerialType* init_item(RsGxsServerGrpUpdateItem& i) @@ -39,7 +39,7 @@ RsSerialType* init_item(RsGxsServerGrpUpdateItem& i) i.clear(); i.grpUpdateTS = rand()%2424; - return new RsGxsUpdateSerialiser(0); + return new RsGxsUpdateSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); } RsSerialType* init_item(RsGxsServerMsgUpdateItem& i) @@ -47,7 +47,7 @@ RsSerialType* init_item(RsGxsServerMsgUpdateItem& i) i.clear(); randString(SHORT_STR, i.grpId); i.msgUpdateTS = rand()%4252; - return new RsGxsUpdateSerialiser(0); + return new RsGxsUpdateSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); } bool operator ==(const RsGxsGrpUpdateItem& l, const RsGxsGrpUpdateItem& r) diff --git a/libretroshare/src/tests/serialiser/rsnxsitems_test.cc b/libretroshare/src/tests/serialiser/rsnxsitems_test.cc index 16cc020a9..b2dab65e9 100644 --- a/libretroshare/src/tests/serialiser/rsnxsitems_test.cc +++ b/libretroshare/src/tests/serialiser/rsnxsitems_test.cc @@ -39,7 +39,7 @@ RsSerialType* init_item(RsNxsSyncGrp& rsg) { rsg.clear(); rsg.flag = RsNxsSyncGrp::FLAG_USE_SYNC_HASH; - rsg.syncAge = rand()%2423; + rsg.createdSince = rand()%2423; randString(3124,rsg.syncHash); return new RsNxsSerialiser(RS_SERVICE_TYPE_PLUGIN_SIMPLE_FORUM); @@ -50,7 +50,7 @@ RsSerialType* init_item(RsNxsSyncMsg& rsgm) rsgm.clear(); rsgm.flag = RsNxsSyncMsg::FLAG_USE_SYNC_HASH; - rsgm.syncAge = rand()%24232; + rsgm.createdSince = rand()%24232; rsgm.transactionNumber = rand()%23; randString(SHORT_STR, rsgm.grpId); randString(SHORT_STR, rsgm.syncHash); @@ -120,7 +120,7 @@ bool operator==(const RsNxsSyncGrp& l, const RsNxsSyncGrp& r) if(l.syncHash != r.syncHash) return false; if(l.flag != r.flag) return false; - if(l.syncAge != r.syncAge) return false; + if(l.createdSince != r.createdSince) return false; if(l.transactionNumber != r.transactionNumber) return false; return true; @@ -130,7 +130,7 @@ bool operator==(const RsNxsSyncMsg& l, const RsNxsSyncMsg& r) { if(l.flag != r.flag) return false; - if(l.syncAge != r.syncAge) return false; + if(l.createdSince != r.createdSince) return false; if(l.syncHash != r.syncHash) return false; if(l.grpId != r.grpId) return false; if(l.transactionNumber != r.transactionNumber) return false; From cc7569e3966dd431a7230b42c45168e6b91d5e9f Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Fri, 20 Dec 2013 14:48:32 +0000 Subject: [PATCH 29/29] - enabled config saving of sync information - additional fixes for sync reloading - confirmed sync now working with A B test git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs_finale@6951 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsgxsnetservice.cc | 32 +++++++++++++++++++++--- libretroshare/src/rsserver/rsinit.cc | 13 ++++++++++ 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index e7643c7e1..00fd02422 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -566,7 +566,7 @@ class StoreHere { public: - StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap cmm, + StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap& cmm, RsGxsNetService::ServerMsgMap& smm, RsGxsServerGrpUpdateItem*& sgm) : mClientGrpMap(cgm), mClientMsgMap(cmm), mServerMsgMap(smm), mServerGrpUpdateItem(sgm) @@ -587,7 +587,7 @@ public: mServerMsgMap.insert(std::make_pair(msui->grpId, msui)); else if((gsui = dynamic_cast(item)) != NULL) { - if(mServerGrpUpdateItem) + if(mServerGrpUpdateItem == NULL) { mServerGrpUpdateItem = gsui; } @@ -619,6 +619,7 @@ bool RsGxsNetService::loadList(std::list &load) { std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mGrpServerUpdateItem)); + return true; } @@ -648,6 +649,9 @@ bool RsGxsNetService::saveList(bool& cleanup, std::list& save) std::back_inserter(save), get_second()); save.push_back(mGrpServerUpdateItem); + + cleanup = false; + return true; } RsSerialiser *RsGxsNetService::setupSerialiser() @@ -916,6 +920,8 @@ void RsGxsNetService::updateServerSyncTS() mGrpServerUpdateItem = new RsGxsServerGrpUpdateItem(mServType); } + bool change = false; + for(; mit != gxsMap.end(); mit++) { const RsGxsGroupId& grpId = mit->first; @@ -933,15 +939,27 @@ void RsGxsNetService::updateServerSyncTS() msui = mapIT->second; } - msui->msgUpdateTS = grpMeta->mLastPost; + if(grpMeta->mLastPost > msui->msgUpdateTS ) + { + change = true; + msui->msgUpdateTS = grpMeta->mLastPost; + } // this might be very inefficient with time if(grpMeta->mRecvTS > mGrpServerUpdateItem->grpUpdateTS) + { mGrpServerUpdateItem->grpUpdateTS = grpMeta->mRecvTS; + change = true; + } } + // actual change in config settings, then save configuration + if(change) + IndicateConfigChanged(); + freeAndClearContainerResource, RsGxsGrpMetaData*>(gxsMap); + } bool RsGxsNetService::locked_checkTransacTimedOut(NxsTransaction* tr) { @@ -1229,6 +1247,8 @@ void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) item->grpUpdateTS = updateTS; item->peerId = peerFrom; + IndicateConfigChanged(); + }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) { @@ -1310,6 +1330,7 @@ void RsGxsNetService::locked_doMsgUpdateWork(const RsNxsTransac *nxsTrans, const mui->msgUpdateTS[grpId] = nxsTrans->updateTS; mui->peerId = peerFrom; + IndicateConfigChanged(); } void RsGxsNetService::locked_processCompletedOutgoingTrans(NxsTransaction* tr) @@ -2159,7 +2180,12 @@ bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncMsg *item) const RsGxsServerMsgUpdateItem *msui = cit->second; if(item->updateTS >= msui->msgUpdateTS && item->updateTS != 0) + { +#ifdef NXS_NET_DEBUG + std::cerr << "RsGxsNetService::locked_CanReceiveUpdate(): Msgs up to date" << std::endl; +#endif return false; + } } return true; } diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 6122adc4a..c5e797f1a 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -2502,6 +2502,19 @@ int RsServer::StartupRetroShare() mConfigMgr->addConfiguration("bitdht.cfg", mBitDht); #endif +#ifdef RS_ENABLE_GXS + + mConfigMgr->addConfiguration("identity.cfg", gxsid_ns); + mConfigMgr->addConfiguration("gxsforums.cfg", gxsforums_ns); + mConfigMgr->addConfiguration("gxschannels.cfg", gxschannels_ns); + mConfigMgr->addConfiguration("gxscircles.cfg", gxscircles_ns); + mConfigMgr->addConfiguration("posted.cfg", posted_ns); + mConfigMgr->addConfiguration("wire.cfg", wire_ns); + mConfigMgr->addConfiguration("wiki.cfg", wiki_ns); + mConfigMgr->addConfiguration("photo.cfg", photo_ns); + +#endif + mPluginsManager->addConfigurations(mConfigMgr) ; ftserver->addConfiguration(mConfigMgr);