From bcf9f443b41bbd68a6ceea03574754f0e6b93a3a Mon Sep 17 00:00:00 2001 From: chrisparker126 Date: Sun, 2 Dec 2012 19:40:17 +0000 Subject: [PATCH] added option to RsTokReqOptions for group subscription filter added code for rank calculation best,top, and newest (not enabled, not working yet...) added variables for circles to grp meta type and modified storage and serialisation accordingly git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs-b1@5930 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- libretroshare/src/gxs/rsdataservice.cc | 27 +- libretroshare/src/gxs/rsgenexchange.cc | 5 +- libretroshare/src/gxs/rsgenexchange.h | 2 +- libretroshare/src/gxs/rsgxsdata.cc | 15 +- libretroshare/src/gxs/rsgxsdata.h | 7 +- libretroshare/src/gxs/rsgxsdataaccess.cc | 104 ++++- libretroshare/src/gxs/rsgxsdataaccess.h | 27 ++ libretroshare/src/gxs/rstokenservice.h | 3 +- libretroshare/src/retroshare/rsposted.h | 18 +- libretroshare/src/serialiser/rsgxsitems.cc | 4 + libretroshare/src/serialiser/rsgxsitems.h | 12 + libretroshare/src/serialiser/rsposteditems.cc | 6 +- libretroshare/src/services/p3posted.cc | 424 +++++++++++++++++- libretroshare/src/services/p3posted.h | 55 ++- 14 files changed, 660 insertions(+), 49 deletions(-) diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index 7f2bb6242..075894e5b 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -51,6 +51,10 @@ #define KEY_KEY_SET std::string("keySet") #define KEY_GRP_NAME std::string("grpName") #define KEY_GRP_SIGN_FLAGS std::string("signFlags") +#define KEY_GRP_CIRCLE_ID std::string("circleId") +#define KEY_GRP_CIRCLE_TYPE std::string("circleType") +#define KEY_GRP_INTERNAL_CIRCLE std::string("internalCircle") +#define KEY_GRP_ORIGINATOR std::string("originator") // grp local #define KEY_GRP_SUBCR_FLAG std::string("subscribeFlag") @@ -98,6 +102,11 @@ #define COL_ORIG_GRP_ID 12 #define COL_GRP_SERV_STRING 13 #define COL_GRP_SIGN_FLAGS 14 +#define COL_GRP_CIRCLE_ID 15 +#define COL_GRP_CIRCL_TYPE 16 +#define COL_GRP_INTERN_CIRCLE 17 +#define COL_GRP_ORIGINATOR 18 + // msg col numbers #define COL_MSG_ID 5 @@ -150,7 +159,8 @@ RsDataService::RsDataService(const std::string &serviceDir, const std::string &d grpMetaColumns.push_back(KEY_KEY_SET); grpMetaColumns.push_back(KEY_GRP_SUBCR_FLAG); grpMetaColumns.push_back(KEY_GRP_POP); grpMetaColumns.push_back(KEY_MSG_COUNT); grpMetaColumns.push_back(KEY_GRP_STATUS); grpMetaColumns.push_back(KEY_GRP_NAME); grpMetaColumns.push_back(KEY_GRP_LAST_POST); grpMetaColumns.push_back(KEY_ORIG_GRP_ID); grpMetaColumns.push_back(KEY_NXS_SERV_STRING); - grpMetaColumns.push_back(KEY_GRP_SIGN_FLAGS); + grpMetaColumns.push_back(KEY_GRP_SIGN_FLAGS); grpMetaColumns.push_back(KEY_GRP_CIRCLE_ID); grpMetaColumns.push_back(KEY_GRP_CIRCLE_TYPE); + grpMetaColumns.push_back(KEY_GRP_INTERNAL_CIRCLE); grpMetaColumns.push_back(KEY_GRP_ORIGINATOR); // for retrieving actual grp data grpColumns.push_back(KEY_GRP_ID); grpColumns.push_back(KEY_NXS_FILE); grpColumns.push_back(KEY_NXS_FILE_OFFSET); @@ -209,6 +219,10 @@ void RsDataService::initialise(){ KEY_NXS_SERV_STRING + " TEXT," + KEY_NXS_FLAGS + " INT," + KEY_GRP_SIGN_FLAGS + " INT," + + KEY_GRP_CIRCLE_ID + " TEXT," + + KEY_GRP_CIRCLE_TYPE + " INT," + + KEY_GRP_INTERNAL_CIRCLE + " TEXT," + + KEY_GRP_ORIGINATOR + " TEXT," + KEY_SIGN_SET + " BLOB);"); } @@ -235,7 +249,7 @@ RsGxsGrpMetaData* RsDataService::getGrpMeta(RetroCursor &c) c.getString(COL_GRP_NAME, grpMeta->mGroupName); c.getString(COL_ORIG_GRP_ID, grpMeta->mOrigGrpId); c.getString(COL_GRP_SERV_STRING, grpMeta->mServiceString); - grpMeta->mSignFlags = c.getInt32(COL_GRP_SIGN_FLAGS); + grpMeta->mSignFlags = c.getInt32(COL_GRP_SIGN_FLAGS); grpMeta->mPublishTs = c.getInt32(COL_TIME_STAMP); grpMeta->mGroupFlags = c.getInt32(COL_NXS_FLAGS); @@ -256,6 +270,11 @@ RsGxsGrpMetaData* RsDataService::getGrpMeta(RetroCursor &c) grpMeta->mLastPost = c.getInt32(COL_GRP_LAST_POST); grpMeta->mGroupStatus = c.getInt32(COL_GRP_STATUS); + c.getString(COL_GRP_CIRCLE_ID, grpMeta->mCircleId); + grpMeta->mCircleType = c.getInt32(COL_GRP_CIRCL_TYPE); + c.getString(COL_GRP_INTERN_CIRCLE, grpMeta->mInternalCircle); + c.getString(COL_GRP_ORIGINATOR, grpMeta->mOriginator); + if(ok) return grpMeta; @@ -539,6 +558,10 @@ int RsDataService::storeGroup(std::map &grp) cv.put(KEY_NXS_FLAGS, (int32_t)grpMetaPtr->mGroupFlags); cv.put(KEY_TIME_STAMP, (int32_t)grpMetaPtr->mPublishTs); cv.put(KEY_GRP_SIGN_FLAGS, (int32_t)grpMetaPtr->mSignFlags); + cv.put(KEY_GRP_CIRCLE_ID, grpMetaPtr->mCircleId); + cv.put(KEY_GRP_CIRCLE_TYPE, (int32_t)grpMetaPtr->mCircleType); + cv.put(KEY_GRP_INTERNAL_CIRCLE, grpMetaPtr->mInternalCircle); + cv.put(KEY_GRP_ORIGINATOR, grpMetaPtr->mOriginator); if(! (grpMetaPtr->mAuthorId.empty()) ){ cv.put(KEY_NXS_IDENTITY, grpMetaPtr->mAuthorId); diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 6a662cc72..2aef41746 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1570,6 +1570,7 @@ void RsGenExchange::processRecvdMessages() RsGxsMsgMetaData* meta = new RsGxsMsgMetaData(); bool ok = meta->deserialise(msg->meta.bin_data, &(msg->meta.bin_len)); + if(ok) { std::map::iterator mit = grpMetas.find(msg->grpId); @@ -1578,7 +1579,9 @@ void RsGenExchange::processRecvdMessages() if(mit != grpMetas.end()){ RsGxsGrpMetaData* grpMeta = mit->second; ok = true; - //&= validateMsg(msg, grpMeta->mGroupFlags, grpMeta->keys); +// msg->metaData = meta; + // ok &= validateMsg(msg, grpMeta->mGroupFlags, grpMeta->keys); + // msg->metaData = NULL; } else ok = false; diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index 06df6ba9e..4d52786d4 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -306,7 +306,7 @@ protected: /*! * Retrieve keys for a given group, \n - * call is blocking retrieval for underlying db + * call is blocking retrieval from underlying db * @warning under normal circumstance a service should not need this * @param grpId the id of the group to retrieve keys for * @param keys this is set to the retrieved keys diff --git a/libretroshare/src/gxs/rsgxsdata.cc b/libretroshare/src/gxs/rsgxsdata.cc index 3e5ee08fa..d60e5f24a 100644 --- a/libretroshare/src/gxs/rsgxsdata.cc +++ b/libretroshare/src/gxs/rsgxsdata.cc @@ -46,7 +46,8 @@ uint32_t RsGxsGrpMetaData::serial_size() s += GetTlvStringSize(mServiceString); s += signSet.TlvSize(); s += keys.TlvSize(); - + s += 4; // for mCircleType + s += GetTlvStringSize(mCircleId); return s; } @@ -67,6 +68,10 @@ void RsGxsGrpMetaData::clear(){ mSubscribeFlags = 0; signSet.TlvClear(); keys.TlvClear(); + mCircleId.clear(); + mInternalCircle.clear(); + mOriginator.clear(); + mCircleType = 0; } @@ -99,8 +104,10 @@ bool RsGxsGrpMetaData::serialise(void *data, uint32_t &pktsize) ok &= SetTlvString(data, tlvsize, &offset, 0, mGroupName); ok &= setRawUInt32(data, tlvsize, &offset, mGroupFlags); ok &= setRawUInt32(data, tlvsize, &offset, mPublishTs); + ok &= setRawUInt32(data, tlvsize, &offset, mCircleType); ok &= SetTlvString(data, tlvsize, &offset, 0, mAuthorId); ok &= SetTlvString(data, tlvsize, &offset, 0, mServiceString); + ok &= SetTlvString(data, tlvsize, &offset, 0, mCircleId); ok &= signSet.SetTlv(data, tlvsize, &offset); ok &= keys.SetTlv(data, tlvsize, &offset); @@ -125,8 +132,10 @@ bool RsGxsGrpMetaData::deserialise(void *data, uint32_t &pktsize) ok &= GetTlvString(data, pktsize, &offset, 0, mGroupName); ok &= getRawUInt32(data, pktsize, &offset, &mGroupFlags); ok &= getRawUInt32(data, pktsize, &offset, &mPublishTs); + ok &= getRawUInt32(data, pktsize, &offset, &mCircleType); ok &= GetTlvString(data, pktsize, &offset, 0, mAuthorId); ok &= GetTlvString(data, pktsize, &offset, 0, mServiceString); + ok &= GetTlvString(data, pktsize, &offset, 0, mCircleId); ok &= signSet.GetTlv(data, pktsize, &offset); ok &= keys.GetTlv(data, pktsize, &offset); @@ -257,6 +266,10 @@ void RsGxsGrpMetaData::operator =(const RsGroupMetaData& rMeta) this->mGroupName = rMeta.mGroupName; this->mServiceString = rMeta.mServiceString; this->mSignFlags = rMeta.mSignFlags; + this->mCircleId = rMeta.mCircleId; + this->mCircleType = rMeta.mCircleType; + this->mInternalCircle = rMeta.mInternalCircle; + this->mOriginator = rMeta.mOriginator; } void RsGxsMsgMetaData::operator =(const RsMsgMetaData& rMeta) diff --git a/libretroshare/src/gxs/rsgxsdata.h b/libretroshare/src/gxs/rsgxsdata.h index 88d4044f1..9452c364b 100644 --- a/libretroshare/src/gxs/rsgxsdata.h +++ b/libretroshare/src/gxs/rsgxsdata.h @@ -27,7 +27,6 @@ */ #include - #include "serialiser/rsserial.h" #include "serialiser/rstlvtypes.h" #include "serialiser/rstlvkeys.h" @@ -59,6 +58,9 @@ public: uint32_t mSignFlags; std::string mAuthorId; + std::string mCircleId; + uint32_t mCircleType; + RsTlvKeySignatureSet signSet; RsTlvSecurityKeySet keys; @@ -74,7 +76,8 @@ public: time_t mLastPost; // ??? uint32_t mGroupStatus; - + std::string mOriginator; + std::string mInternalCircle; }; diff --git a/libretroshare/src/gxs/rsgxsdataaccess.cc b/libretroshare/src/gxs/rsgxsdataaccess.cc index af45bbdca..3a62b7559 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.cc +++ b/libretroshare/src/gxs/rsgxsdataaccess.cc @@ -771,9 +771,15 @@ void RsGxsDataAccess::processRequests() bool RsGxsDataAccess::getGroupData(GroupDataReq* req) { std::map grpData; + std::list grpIdsOut; - std::list::iterator lit = req->mGroupIds.begin(), - lit_end = req->mGroupIds.end(); + getGroupList(req->mGroupIds, req->Options, grpIdsOut); + + if(grpIdsOut.empty()) + return true; + + std::list::iterator lit = grpIdsOut.begin(), + lit_end = grpIdsOut.end(); for(; lit != lit_end; lit++) { @@ -794,9 +800,16 @@ bool RsGxsDataAccess::getGroupSummary(GroupMetaReq* req) std::map grpMeta; - std::list::const_iterator lit = req->mGroupIds.begin(); + std::list grpIdsOut; - for(; lit != req->mGroupIds.end(); lit++) + getGroupList(req->mGroupIds, req->Options, grpIdsOut); + + if(grpIdsOut.empty()) + return true; + + std::list::const_iterator lit = grpIdsOut.begin(); + + for(; lit != grpIdsOut.end(); lit++) grpMeta[*lit] = NULL; mDataStore->retrieveGxsGrpMetaData(grpMeta); @@ -811,24 +824,37 @@ bool RsGxsDataAccess::getGroupSummary(GroupMetaReq* req) bool RsGxsDataAccess::getGroupList(GroupIdReq* req) { - std::map grpMeta; + getGroupList(req->mGroupIds, req->Options, req->mGroupIdResult); - std::list::const_iterator lit = req->mGroupIds.begin(); + return true; +} - for(; lit != req->mGroupIds.end(); lit++) - grpMeta[*lit] = NULL; +bool RsGxsDataAccess::getGroupList(const std::list& grpIdsIn, const RsTokReqOptions& opts, std::list& grpIdsOut) +{ + std::map grpMeta; - mDataStore->retrieveGxsGrpMetaData(grpMeta); + std::list::const_iterator lit = grpIdsIn.begin(); - std::map::iterator mit = grpMeta.begin(); + for(; lit != grpIdsIn.end(); lit++) + grpMeta[*lit] = NULL; - for(; mit != grpMeta.end(); mit++) - { - req->mGroupIdResult.push_back(mit->first); - delete mit->second; // so wasteful!! - } + mDataStore->retrieveGxsGrpMetaData(grpMeta); - return true; + std::map::iterator mit = grpMeta.begin(); + + for(; mit != grpMeta.end(); mit++) + { + grpIdsOut.push_back(mit->first); + } + + filterGrpList(grpIdsOut, opts, grpMeta); + + for(mit = grpMeta.begin(); mit != grpMeta.end(); mit++) + { + delete mit->second; // so wasteful!! + } + + return true; } bool RsGxsDataAccess::getMsgData(MsgDataReq* req) @@ -1394,6 +1420,32 @@ void RsGxsDataAccess::filterMsgList(GxsMsgIdResult& msgIds, const RsTokReqOption } } +void RsGxsDataAccess::filterGrpList(std::list &grpIds, const RsTokReqOptions &opts, const GrpMetaFilter &meta) const +{ + std::list::iterator lit = grpIds.begin(); + + for(; lit != grpIds.end(); ) + { + GrpMetaFilter::const_iterator cit = meta.find(*lit); + + bool keep = false; + + if(cit != meta.end()) + { + keep = checkGrpFilter(opts, cit->second); + } + + if(keep) + { + lit++; + }else + { + lit = grpIds.erase(lit); + } + + } +} + bool RsGxsDataAccess::checkRequestStatus(const uint32_t& token, uint32_t& status, uint32_t& reqtype, uint32_t& anstype, time_t& ts) @@ -1516,6 +1568,26 @@ bool RsGxsDataAccess::disposeOfPublicToken(const uint32_t& token) return true; } +bool RsGxsDataAccess::checkGrpFilter(const RsTokReqOptions &opts, const RsGxsGrpMetaData *meta) const +{ + + bool subscribeMatch = false; + + if(opts.mSubscribeMask) + { + // Exact Flags match required. + if ((opts.mSubscribeMask & opts.mSubscribeFilter) == (opts.mSubscribeMask & meta->mSubscribeFlags)) + { + subscribeMatch = true; + } + } + else + { + subscribeMatch = true; + } + + return subscribeMatch; +} bool RsGxsDataAccess::checkMsgFilter(const RsTokReqOptions& opts, const RsGxsMsgMetaData* meta) const { bool statusMatch = false; diff --git a/libretroshare/src/gxs/rsgxsdataaccess.h b/libretroshare/src/gxs/rsgxsdataaccess.h index f8703477d..5f1e09975 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.h +++ b/libretroshare/src/gxs/rsgxsdataaccess.h @@ -32,6 +32,7 @@ typedef std::map< RsGxsGroupId, std::map > MsgMetaFilter; +typedef std::map< RsGxsGroupId, RsGxsGrpMetaData* > GrpMetaFilter; class RsGxsDataAccess : public RsTokenService { @@ -304,6 +305,14 @@ private: */ bool getGroupList(GroupIdReq* req); + /*! + * convenience function for filtering grpIds + * @param grpIdsIn The ids to filter with opts + * @param opts the filter options + * @param grpIdsOut grpIdsIn filtered with opts + */ + bool getGroupList(const std::list& grpIdsIn, const RsTokReqOptions& opts, std::list& grpIdsOut); + /*! * Attempts to retrieve msg id list from data store * Computationally/CPU-Bandwidth expensive @@ -358,6 +367,14 @@ private: */ void filterMsgList(GxsMsgIdResult& msgIds, const RsTokReqOptions& opts, const MsgMetaFilter& meta) const; + /*! + * This filter msgs based of options supplied (at the moment just status masks) + * @param grpIds The group ids to filter + * @param opts the request options containing mask set by user + * @param meta The accompanying meta information for group ids + */ + void filterGrpList(std::list& msgIds, const RsTokReqOptions& opts, const GrpMetaFilter& meta) const; + /*! * This applies the options to the meta to find out if the given message satisfies @@ -368,6 +385,16 @@ private: */ bool checkMsgFilter(const RsTokReqOptions& opts, const RsGxsMsgMetaData* meta) const; + /*! + * This applies the options to the meta to find out if the given group satisfies + * them + * @param opts options containing filters to check + * @param meta meta containing currently defined options for group + * @return true if group meta passes all options + */ + bool checkGrpFilter(const RsTokReqOptions& opts, const RsGxsGrpMetaData* meta) const; + + /*! * This is a filter method which applies the request options to the list of ids * requested diff --git a/libretroshare/src/gxs/rstokenservice.h b/libretroshare/src/gxs/rstokenservice.h index c2e768ade..655666104 100644 --- a/libretroshare/src/gxs/rstokenservice.h +++ b/libretroshare/src/gxs/rstokenservice.h @@ -79,6 +79,7 @@ RsTokReqOptions() { mOptions = 0; mStatusFilter = 0; mStatusMask = 0; mSubscribeFilter = 0; + mSubscribeMask = 0; mMsgFlagMask = 0; mMsgFlagFilter = 0; mBefore = 0; mAfter = 0; mReqType = 0; } @@ -95,7 +96,7 @@ uint32_t mMsgFlagMask, mMsgFlagFilter; uint32_t mReqType; -uint32_t mSubscribeFilter; // Only for Groups. +uint32_t mSubscribeFilter, mSubscribeMask; // Only for Groups. // Time range... again applied after Options. time_t mBefore; diff --git a/libretroshare/src/retroshare/rsposted.h b/libretroshare/src/retroshare/rsposted.h index 3d0eb20fd..2094ce89b 100644 --- a/libretroshare/src/retroshare/rsposted.h +++ b/libretroshare/src/retroshare/rsposted.h @@ -87,6 +87,7 @@ class RsPosted : public RsGxsIfaceImpl static const uint32_t FLAG_MSGTYPE_POST; static const uint32_t FLAG_MSGTYPE_VOTE; static const uint32_t FLAG_MSGTYPE_COMMENT; + static const uint32_t FLAG_MSGTYPE_MASK; RsPosted(RsGenExchange* gxs) : RsGxsIfaceImpl(gxs) { return; } @@ -138,17 +139,20 @@ class RsPostedPost std::string mNotes; }; +class RsGxsPostedVoteItem; class RsPostedVote { - public: - RsPostedVote() - { - mMeta.mMsgFlags = RsPosted::FLAG_MSGTYPE_VOTE; - return; - } +public: - RsMsgMetaData mMeta; + RsPostedVote(const RsGxsPostedVoteItem&); + RsPostedVote() + { + mMeta.mMsgFlags = RsPosted::FLAG_MSGTYPE_VOTE; + return; + } + uint8_t mDirection; + RsMsgMetaData mMeta; }; class RsGxsPostedCommentItem; diff --git a/libretroshare/src/serialiser/rsgxsitems.cc b/libretroshare/src/serialiser/rsgxsitems.cc index 9f62c6da1..00fc2b100 100644 --- a/libretroshare/src/serialiser/rsgxsitems.cc +++ b/libretroshare/src/serialiser/rsgxsitems.cc @@ -42,6 +42,10 @@ this->mGroupName = rGxsMeta.mGroupName; this->mServiceString = rGxsMeta.mServiceString; this->mSignFlags = rGxsMeta.mSignFlags; + this->mCircleId = rGxsMeta.mCircleId; + this->mCircleType = rGxsMeta.mCircleType; + this->mInternalCircle = rGxsMeta.mInternalCircle; + this->mOriginator = rGxsMeta.mOriginator; } diff --git a/libretroshare/src/serialiser/rsgxsitems.h b/libretroshare/src/serialiser/rsgxsitems.h index 2726e21b9..346cc14a1 100644 --- a/libretroshare/src/serialiser/rsgxsitems.h +++ b/libretroshare/src/serialiser/rsgxsitems.h @@ -34,6 +34,7 @@ class RsGxsGrpMetaData; class RsGxsMsgMetaData; + class RsGroupMetaData { public: @@ -48,6 +49,7 @@ public: mLastPost = 0; mGroupStatus = 0; + mCircleType = 0; //mPublishTs = 0; } @@ -62,6 +64,10 @@ public: time_t mPublishTs; // Mandatory. std::string mAuthorId; // Optional. + // for circles + std::string mCircleId; + uint32_t mCircleType; + // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG. uint32_t mSubscribeFlags; @@ -72,6 +78,12 @@ public: uint32_t mGroupStatus; std::string mServiceString; // Service Specific Free-Form extra storage. + std::string mOriginator; + std::string mInternalCircle; + + + + }; diff --git a/libretroshare/src/serialiser/rsposteditems.cc b/libretroshare/src/serialiser/rsposteditems.cc index 5edc827ec..99280b634 100644 --- a/libretroshare/src/serialiser/rsposteditems.cc +++ b/libretroshare/src/serialiser/rsposteditems.cc @@ -25,6 +25,7 @@ */ #include "serialiser/rsposteditems.h" +#include "rsbaseserial.h" uint32_t RsGxsPostedSerialiser::size(RsItem *item) @@ -154,6 +155,7 @@ uint32_t RsGxsPostedSerialiser::sizeGxsPostedVoteItem(RsGxsPostedVoteItem* item) RsPostedVote& v = item->mVote; uint32_t s = 8; + s += 1; // for vote direction return s; } @@ -289,7 +291,7 @@ bool RsGxsPostedSerialiser::serialiseGxsPostedVoteItem(RsGxsPostedVoteItem* item /* skip the header */ offset += 8; - /* GxsPhotoAlbumItem */ + ok &= setRawUInt32(data, tlvsize, &offset, item->mVote.mDirection); if(offset != tlvsize) { @@ -525,6 +527,8 @@ RsGxsPostedVoteItem* RsGxsPostedSerialiser::deserialiseGxsPostedVoteItem(void *d /* skip the header */ offset += 8; + ok &= getRawUInt8(data, rssize, &offset, &(item->mVote.mDirection)); + if (offset != rssize) { #ifdef GXS_POSTED_SERIAL_DEBUG diff --git a/libretroshare/src/services/p3posted.cc b/libretroshare/src/services/p3posted.cc index b67d334e7..cc9e695cc 100644 --- a/libretroshare/src/services/p3posted.cc +++ b/libretroshare/src/services/p3posted.cc @@ -1,9 +1,17 @@ + +#include +#include + #include "p3posted.h" +#include "gxs/rsgxsflags.h" #include "serialiser/rsposteditems.h" const uint32_t RsPosted::FLAG_MSGTYPE_COMMENT = 0x0001; const uint32_t RsPosted::FLAG_MSGTYPE_POST = 0x0002; const uint32_t RsPosted::FLAG_MSGTYPE_VOTE = 0x0004; +const uint32_t RsPosted::FLAG_MSGTYPE_MASK = 0x000f; + +#define POSTED_MAX_SERVICE_STRING 50 RsPosted *rsPosted = NULL; @@ -13,9 +21,17 @@ RsPostedComment::RsPostedComment(const RsGxsPostedCommentItem & item) mMeta = item.meta; } -p3Posted::p3Posted(RsGeneralDataService *gds, RsNetworkExchangeService *nes) - : RsGenExchange(gds, nes, new RsGxsPostedSerialiser(), RS_SERVICE_GXSV1_TYPE_POSTED), RsPosted(this), mPostedMutex("Posted") +RsPostedVote::RsPostedVote(const RsGxsPostedVoteItem& item) { + mDirection = item.mVote.mDirection; + mMeta = item.meta; +} + +p3Posted::p3Posted(RsGeneralDataService *gds, RsNetworkExchangeService *nes) + : RsGenExchange(gds, nes, new RsGxsPostedSerialiser(), RS_SERVICE_GXSV1_TYPE_POSTED), RsPosted(this), mPostedMutex("Posted"), + mTokenService(NULL) +{ + mTokenService = RsGenExchange::getTokenService(); } void p3Posted::notifyChanges(std::vector &changes) @@ -154,7 +170,7 @@ bool p3Posted::submitVote(uint32_t &token, RsPostedVote &vote) RsGxsPostedVoteItem* voteItem = new RsGxsPostedVoteItem(); voteItem->mVote = vote; voteItem->meta = vote.mMeta; - voteItem->meta.mMsgFlags |= FLAG_MSGTYPE_POST; + voteItem->meta.mMsgFlags |= FLAG_MSGTYPE_VOTE; RsGenExchange::publishMsg(token, voteItem); return true; @@ -221,6 +237,7 @@ void p3Posted::processMessageRanks() RsStackMutex stack(mPostedMutex); std::map::iterator mit =mPendingPostRanks.begin(); + // go through all pending posts for(; mit !=mPendingPostRanks.begin(); mit++) { uint32_t token; @@ -235,7 +252,7 @@ void p3Posted::processMessageRanks() while(true) { - uint32_t status = RsGenExchange::getTokenService()->requestStatus(token); + uint32_t status = mTokenService->requestStatus(token); if(RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE == status) @@ -259,38 +276,417 @@ void p3Posted::processMessageRanks() void p3Posted::discardCalc(const uint32_t &token) { + mTokenService->cancelRequest(token); +} +bool PostedTopScoreComp(const PostedScore& i, const PostedScore& j) +{ + if((i.upVotes + (-i.downVotes)) == (j.upVotes + (-j.downVotes))){ + return i.date < j.date; + }else + return (i.upVotes + (-i.downVotes)) < (j.upVotes + (-j.downVotes)); +} + +bool PostedNewScoreComp(const PostedScore& i, const PostedScore& j) +{ + return i.date < j.date; +} + +bool PostedBestScoreComp(const PostedScore& i, const PostedScore& j) +{ + +// n = ups + downs if n == 0: return 0 z = 1.0 +// #1.0 = 85%, 1.6 = 95% phat = float(ups) +// / n return sqrt(phat+z*z/(2*n)-z*((phat*(1-phat)+z*z/(4*n))/n))/(1+z*z/n) +// def confidence(ups, downs): if ups + downs == 0: return 0 else: +// return _confidence(ups, downs) + // very expensive!! + + static float z = 1.0; + float phat; + + float i_score; + int n = i.upVotes + (-i.downVotes); + if(n==0) + i_score = 0.; + else + { + phat = float(i.upVotes); + i_score = sqrt(phat+z*z/(2*n)-z*((phat*(1-phat)+z*z/(4*n))/n))/(1+z*z/n); + } + + float j_score; + n = j.upVotes + (-j.downVotes); + if(n==0) + j_score = 0.; + else + { + phat = float(j.upVotes); + j_score = sqrt(phat+z*z/(2*n)-z*((phat*(1-phat)+z*z/(4*n))/n))/(1+z*z/n); + } + + if(j_score == i_score) + return i.date < j.date; + else + return i_score < j_score; } void p3Posted::completePostedPostCalc(GxsPostedPostRanking *gpp) { GxsMsgMetaMap msgMetas; - getMsgMeta(gpp->reqToken, msgMetas); - GxsMsgMetaMap::iterator mit = msgMetas.begin(); - - for(; mit != msgMetas.end(); mit++ ) + if(getMsgMeta(gpp->reqToken, msgMetas)) { - RsGxsMsgMetaData* m = NULL; - //retrieveScores(m->mServiceString, upVotes, downVotes, nComments); + std::vector msgMetaV = msgMetas[gpp->grpId]; + switch(gpp->rType) + { + case NewRankType: + calcPostedPostRank(msgMetaV, gpp->rankingResult, PostedNewScoreComp); + break; + case TopRankType: + calcPostedPostRank(msgMetaV, gpp->rankingResult, PostedTopScoreComp); + break; + default: + std::cerr << "Unknown ranking tpye: " << gpp->rType << std::endl; + } + } +} - // then dependent on rank request type process for that way + +void p3Posted::calcPostedPostRank(const std::vector msgMeta, PostedRanking &ranking, + bool comp(const PostedScore &, const PostedScore &)) const +{ + + std::vector::const_iterator cit = msgMeta.begin(); + std::vector scores; + + for(; cit != msgMeta.begin(); ) + { + const RsMsgMetaData& m = *cit; + uint32_t upVotes, downVotes, nComments; + retrieveScores(m.mServiceString, upVotes, downVotes, nComments); + + PostedScore c; + c.upVotes = upVotes; + c.downVotes = downVotes; + c.date = m.mPublishTs; + scores.push_back(c); } + std::sort(scores.begin(), scores.end(), comp); + + std::vector::iterator vit = scores.begin(); + + int i = 1; + for(; vit != scores.end(); vit) + { + const PostedScore& p = *vit; + ranking.insert(std::make_pair(p.msgId, i++)); + } +} + +void p3Posted::calcPostedCommentsRank(const std::map > &msgBranches, + std::map& msgMetas, PostedRanking &ranking, bool comp(const PostedScore &, const PostedScore &)) const +{ + + std::map >::const_iterator cit = msgBranches.begin(); + + for(; cit != msgBranches.end(); cit++) + { + const std::vector& branch = cit->second; + std::vector scores; + + std::vector::const_iterator vit = branch.begin(); + + for(; vit != branch.end(); vit++) + { + + std::map::iterator mit = + msgMetas.find(*vit); + + if(mit != msgMetas.end()) + { + uint32_t upVotes, downVotes, nComments; + + const RsMsgMetaData& m = mit->second; + retrieveScores(m.mServiceString, upVotes, downVotes, nComments); + + PostedScore c; + c.upVotes = upVotes; + c.downVotes = downVotes; + c.date = m.mPublishTs; + scores.push_back(c); + } + } + + std::sort(scores.begin(), scores.end(), comp); + + std::vector::iterator cvit = scores.begin(); + + int i = 1; + for(; cvit != scores.end(); cvit) + { + const PostedScore& p = *cvit; + ranking.insert(std::make_pair(p.msgId, i++)); + } + } } -bool p3Posted::retrieveScores(const std::string &serviceString, uint32_t &upVotes, uint32_t downVotes, uint32_t nComments) +void p3Posted::completePostedCommentRanking(GxsPostedCommentRanking *gpc) { - if (2 == sscanf(serviceString.c_str(), "%d %d %d", &upVotes, &downVotes, &nComments)) + GxsMsgRelatedMetaMap msgMetas; + + if(getMsgRelatedMeta(gpc->reqToken, msgMetas)) { - return true; + + // create map of msgs + std::vector& msgV = msgMetas[gpc->msgId]; + std::map > msgBranches; + std::map remappedMsgMeta; + + std::vector::iterator vit = msgV.begin(); + + for(; vit != msgV.end(); vit++) + { + const RsMsgMetaData& m = *vit; + + if(!m.mParentId.empty()) + { + msgBranches[m.mParentId].push_back(m.mMsgId); + } + + remappedMsgMeta.insert(std::make_pair(m.mMsgId, m)); + } + + switch(gpc->rType) + { + case BestRankType: + calcPostedCommentsRank(msgBranches, remappedMsgMeta, gpc->result, PostedBestScoreComp); + break; + case TopRankType: + calcPostedCommentsRank(msgBranches, remappedMsgMeta, gpc->result, PostedTopScoreComp); + break; + case NewRankType: + calcPostedCommentsRank(msgBranches, remappedMsgMeta, gpc->result, PostedNewScoreComp); + break; + default: + std::cerr << "Unknown Rank type" << gpc->rType << std::endl; + break; + } + } +} + +bool p3Posted::retrieveScores(const std::string &serviceString, uint32_t &upVotes, uint32_t downVotes, uint32_t nComments) const +{ + if (3 == sscanf(serviceString.c_str(), "%d %d %d", &upVotes, &downVotes, &nComments)) + { + return true; } return false; } +bool p3Posted::storeScores(std::string &serviceString, uint32_t &upVotes, uint32_t downVotes, uint32_t nComments) const +{ + char line[POSTED_MAX_SERVICE_STRING]; + + bool ok = snprintf(line, POSTED_MAX_SERVICE_STRING, "%d %d %d", upVotes, downVotes, nComments) > -1; + + serviceString = line; + return ok; +} void p3Posted::processCommentRanks() { } + + +void p3Posted::updateVotes() +{ + if(!mUpdateTokenQueued) + { + mUpdateTokenQueued = true; + + switch(mUpdatePhase) + { +// case UPDATE_PHASE_GRP_REQUEST: +// { +// updateRequestGroups(mUpda); +// break; +// } +// case UPDATE_PHASE_GRP_MSG_REQUEST: +// { +// updateRequestMessages(mVoteUpdataToken); +// break; +// } +// case UPDATE_VOTE_COMMENT_REQUEST: +// { +// updateRequestVotesComments(mVoteUpdataToken); +// break; +// } +// case UPDATE_COMPLETE_UPDATE: +// { +// updateCompleteUpdate(); +// break; +// } +// default: +// break; + } + + // first get all msgs for groups for which you are subscribed to. + // then request comments for them + + } +} + +bool p3Posted::updateRequestGroups(uint32_t &token) +{ + + + RsTokReqOptions opts; + opts.mReqType = GXS_REQUEST_TYPE_GROUP_IDS; + opts.mSubscribeMask = GXS_SERV::GROUP_SUBSCRIBE_MASK; + opts.mSubscribeFilter = GXS_SERV::GROUP_SUBSCRIBE_ADMIN | + GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED; + mTokenService->requestGroupInfo(token, 0, opts); + + mUpdatePhase = UPDATE_PHASE_GRP_MSG_REQUEST; +} + +bool p3Posted::updateRequestMessages(uint32_t &token) +{ + + uint32_t status = mTokenService->requestStatus(token); + + if(status == RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE) + { + std::list grpIds; + RsGenExchange::getGroupList(token, grpIds); + RsTokReqOptions opts; + opts.mReqType = GXS_REQUEST_TYPE_MSG_IDS; + opts.mOptions = RS_TOKREQOPT_MSG_LATEST | RS_TOKREQOPT_MSG_THREAD; + mTokenService->requestMsgInfo(token, 0, opts, grpIds); + mUpdatePhase = UPDATE_VOTE_COMMENT_REQUEST; + return true; + } + else if(status == RsTokenService::GXS_REQUEST_V2_STATUS_FAILED) + { + mTokenService->cancelRequest(token); + return false; + } +} + +bool p3Posted::updateRequestVotesComments(uint32_t &token) +{ + + uint32_t status = mTokenService->requestStatus(token); + + if(status == RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE) + { + + GxsMsgIdResult result; + RsGenExchange::getMsgList(token, result); + + std::vector msgIds; + + GxsMsgIdResult::iterator mit = result.begin(); + + for(; mit != result.end(); mit++) + { + std::vector& msgIdV = mit->second; + std::vector::const_iterator cit = msgIdV.begin(); + + for(; cit != msgIdV.end(); cit++) + msgIds.push_back(std::make_pair(mit->first, *cit)); + } + + // only need ids for comments + RsTokReqOptions opts; + opts.mReqType = GXS_REQUEST_TYPE_MSG_IDS; + opts.mOptions = RS_TOKREQOPT_MSG_LATEST | RS_TOKREQOPT_MSG_PARENT; + opts.mMsgFlagMask = RsPosted::FLAG_MSGTYPE_MASK; + opts.mMsgFlagFilter = RsPosted::FLAG_MSGTYPE_COMMENT; + mTokenService->requestMsgRelatedInfo(mCommentToken, 0, opts, msgIds); + + // need actual data from votes + opts.mReqType = GXS_REQUEST_TYPE_MSG_DATA; + opts.mOptions = RS_TOKREQOPT_MSG_LATEST | RS_TOKREQOPT_MSG_PARENT; + opts.mMsgFlagMask = RsPosted::FLAG_MSGTYPE_MASK; + opts.mMsgFlagFilter = RsPosted::FLAG_MSGTYPE_VOTE; + mTokenService->requestMsgRelatedInfo(mVoteToken, 0, opts, msgIds); + + mUpdatePhase = UPDATE_COMPLETE_UPDATE; + mMsgsPendingUpdate = msgIds; + + return true; + } + else if(status == RsTokenService::GXS_REQUEST_V2_STATUS_FAILED) + { + mTokenService->cancelRequest(token); + return false; + } +} + +bool p3Posted::updateCompleteUpdate() +{ + uint32_t commentStatus = mTokenService->requestStatus(mCommentToken); + uint32_t voteStatus = mTokenService->requestStatus(mVoteToken); + + bool ready = commentStatus == RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE; + ready &= voteStatus == RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE; + + bool failed = commentStatus == RsTokenService::GXS_REQUEST_V2_STATUS_FAILED; + failed &= voteStatus == RsTokenService::GXS_REQUEST_V2_STATUS_FAILED; + + if(ready) + { + std::map > msgCommentIds; + std::map > votes; + getMsgRelatedDataT(mVoteToken, votes); + std::vector::iterator vit = mMsgsPendingUpdate.begin(); + + for(; vit != mMsgsPendingUpdate.end();vit++) + { + updateMsg(*vit, votes[*vit], msgCommentIds[*vit]); + } + mUpdatePhase = 0; + } + else if(failed) + { + mTokenService->cancelRequest(mCommentToken); + mTokenService->cancelRequest(mVoteToken); + return false; + }else + { + return true; + } +} + +bool p3Posted::updateMsg(const RsGxsGrpMsgIdPair& msgId, const std::vector &msgVotes, + const std::vector& msgCommentIds) +{ + + uint32_t nComments = msgCommentIds.size(); + uint32_t nUp = 0, nDown = 0; + + std::vector::const_iterator cit = msgVotes.begin(); + + for(; cit != msgVotes.end(); cit++) + { + const RsPostedVote& v = *cit; + + if(v.mDirection == 0) + { + nDown++; + }else + { + nUp++; + } + } + std::string servStr; + storeScores(servStr, nUp, nDown, nComments); + uint32_t token; + setMsgServiceString(token, msgId, servStr); +} + diff --git a/libretroshare/src/services/p3posted.h b/libretroshare/src/services/p3posted.h index 62d7b789b..7a5e9d42b 100644 --- a/libretroshare/src/services/p3posted.h +++ b/libretroshare/src/services/p3posted.h @@ -15,7 +15,7 @@ public: uint32_t reqToken; RsPosted::RankType rType; RsGxsGroupId grpId; - PostedRanking result; + PostedRanking rankingResult; }; class GxsPostedCommentRanking @@ -29,6 +29,19 @@ public: PostedRanking result; }; +class PostedScore { +public: + int32_t upVotes, downVotes; + time_t date; + RsGxsMessageId msgId; +}; + + +#define UPDATE_PHASE_GRP_REQUEST 1 +#define UPDATE_PHASE_GRP_MSG_REQUEST 2 +#define UPDATE_VOTE_COMMENT_REQUEST 3 +#define UPDATE_COMPLETE_UPDATE 4 + class p3Posted : public RsGenExchange, public RsPosted { public: @@ -67,21 +80,57 @@ public: private: + /* Functions for processing rankings */ + void processRankings(); void processMessageRanks(); void processCommentRanks(); void discardCalc(const uint32_t& token); void completePostedPostCalc(GxsPostedPostRanking* gpp); - bool retrieveScores(const std::string& serviceString, uint32_t& upVotes, uint32_t downVotes, uint32_t nComments); + void completePostedCommentRanking(GxsPostedCommentRanking* gpc); + bool retrieveScores(const std::string& serviceString, uint32_t& upVotes, uint32_t downVotes, uint32_t nComments) const; + bool storeScores(std::string& serviceString, uint32_t& upVotes, uint32_t downVotes, uint32_t nComments) const; + + // for posts + void calcPostedPostRank(const std::vector, PostedRanking& ranking, bool com(const PostedScore& i, const PostedScore &j)) const; + + // for comments + void calcPostedCommentsRank(const std::map >& msgBranches, std::map& msgMetas, + PostedRanking& ranking, bool com(const PostedScore& i, const PostedScore &j)) const; + + /* Functions for maintaing vote counts in meta data */ + + /*! + * Update votes should only be called when a vote comes in + * Several phases to calculating votes. + * First get all messages for groups which you are subscribed + * Then for these messages get all the votes accorded to them + * Then do the calculation and update messages + * Also stores updates for messages which have new scores + */ + void updateVotes(); + bool updateRequestGroups(uint32_t& token); + bool updateRequestMessages(uint32_t& token); + bool updateRequestVotesComments(uint32_t& token); + bool updateCompleteUpdate(); + bool updateMsg(const RsGxsGrpMsgIdPair& msgId, const std::vector& msgVotes, + const std::vector& msgCommentIds); private: + // for calculating ranks std::map mPendingPostRanks; std::map mPendingCalculationPostRanks; - std::map mPendingCommentRanks; std::map mPendingCalculationCommentRanks; + // for maintaining vote counts in msg meta + uint32_t mVoteUpdataToken, mVoteToken, mCommentToken; + bool mUpdateTokenQueued; + uint32_t mUpdatePhase; + std::vector mMsgsPendingUpdate; + + RsTokenService* mTokenService; RsMutex mPostedMutex; };