diff --git a/libretroshare/src/gxs/rsdataservice.cc b/libretroshare/src/gxs/rsdataservice.cc index fc29abdc6..739b60e5f 100644 --- a/libretroshare/src/gxs/rsdataservice.cc +++ b/libretroshare/src/gxs/rsdataservice.cc @@ -134,6 +134,7 @@ const std::string RsGeneralDataService::GRP_META_SUBSCRIBE_FLAG = KEY_GRP_SUBCR_ const std::string RsGeneralDataService::MSG_META_SERV_STRING = KEY_NXS_SERV_STRING; const std::string RsGeneralDataService::MSG_META_STATUS = KEY_MSG_STATUS; +const uint32_t RsGeneralDataService::GXS_MAX_ITEM_SIZE = 1572864; // 1.5 Mbytes RsDataService::RsDataService(const std::string &serviceDir, const std::string &dbName, uint16_t serviceType, RsGxsSearchModule *mod) @@ -453,6 +454,9 @@ int RsDataService::storeMessage(std::map &msg) RsNxsMsg* msgPtr = mit->first; RsGxsMsgMetaData* msgMetaPtr = mit->second; + // skip msg item if size if greater than + if(!validSize(msgPtr)) continue; + // create or access file in binary std::string msgFile = mServiceDir + "/" + msgPtr->grpId + "-msgs"; std::fstream ostrm(msgFile.c_str(), std::ios::binary | std::ios::app | std::ios::out); @@ -519,6 +523,13 @@ int RsDataService::storeMessage(std::map &msg) return ret; } +bool RsDataService::validSize(RsNxsMsg* msg) const +{ + if((msg->msg.TlvSize() + msg->meta.TlvSize()) <= GXS_MAX_ITEM_SIZE) return true; + + return false; +} + int RsDataService::storeGroup(std::map &grp) { @@ -536,6 +547,9 @@ int RsDataService::storeGroup(std::map &grp) RsNxsGrp* grpPtr = sit->first; RsGxsGrpMetaData* grpMetaPtr = sit->second; + // if data is larger than max item size do not add + if(!validSize(grpPtr)) continue; + std::string grpFile = mServiceDir + "/" + grpPtr->grpId; std::fstream ostrm(grpFile.c_str(), std::ios::binary | std::ios::app | std::ios::out); ostrm.seekg(0, std::ios::end); // go to end to append @@ -607,6 +621,12 @@ int RsDataService::storeGroup(std::map &grp) return ret; } +bool RsDataService::validSize(RsNxsGrp* grp) const +{ + if((grp->grp.TlvSize() + grp->meta.TlvSize()) <= GXS_MAX_ITEM_SIZE) return true; + return false; +} + int RsDataService::retrieveNxsGrps(std::map &grp, bool withMeta, bool cache){ if(grp.empty()){ diff --git a/libretroshare/src/gxs/rsdataservice.h b/libretroshare/src/gxs/rsdataservice.h index 7a8a8fb25..8bfe97856 100644 --- a/libretroshare/src/gxs/rsdataservice.h +++ b/libretroshare/src/gxs/rsdataservice.h @@ -132,6 +132,11 @@ public: */ int resetDataStore(); + + bool validSize(RsNxsMsg* msg) const; + bool validSize(RsNxsGrp* grp) const; + + private: /*! diff --git a/libretroshare/src/gxs/rsgds.h b/libretroshare/src/gxs/rsgds.h index bfcee886c..c7a5cf387 100644 --- a/libretroshare/src/gxs/rsgds.h +++ b/libretroshare/src/gxs/rsgds.h @@ -105,6 +105,8 @@ class RsGeneralDataService public: + static const uint32_t GXS_MAX_ITEM_SIZE; + static const std::string MSG_META_SERV_STRING; static const std::string MSG_META_STATUS; @@ -210,6 +212,22 @@ public: */ virtual int resetDataStore() = 0; + /*! + * Use to determine if message isn't over the storage + * limit for a single message item + * @param msg the message to check size validity + * @return whether the size of of msg is valid + */ + virtual bool validSize(RsNxsMsg* msg) const = 0 ; + + /*! + * Use to determine if group isn't over the storage limit + * for a single group item + * @param grp the group to check size validity + * @return whether the size of grp is valid for storage + */ + virtual bool validSize(RsNxsGrp* grp) const = 0 ; + }; diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 88686c681..5bd58fb66 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -160,7 +160,8 @@ bool RsGenExchange::acknowledgeTokenGrp(const uint32_t& token, return true; } -void RsGenExchange::generateGroupKeys(RsTlvSecurityKeySet& keySet, bool genPublishKeys) +void RsGenExchange::generateGroupKeys(RsTlvSecurityKeySet& privatekeySet, + RsTlvSecurityKeySet& publickeySet, bool genPublishKeys) { /* create Keys */ @@ -186,8 +187,8 @@ void RsGenExchange::generateGroupKeys(RsTlvSecurityKeySet& keySet, bool genPubli adminKey.keyFlags = RSTLV_KEY_DISTRIB_ADMIN | RSTLV_KEY_TYPE_PUBLIC_ONLY; privAdminKey.keyFlags = RSTLV_KEY_DISTRIB_ADMIN | RSTLV_KEY_TYPE_FULL; - keySet.keys[adminKey.keyId] = adminKey; - keySet.keys[privAdminKey.keyId] = privAdminKey; + publickeySet.keys[adminKey.keyId] = adminKey; + privatekeySet.keys[privAdminKey.keyId] = privAdminKey; // clean up RSA_free(rsa_admin); @@ -216,15 +217,15 @@ void RsGenExchange::generateGroupKeys(RsTlvSecurityKeySet& keySet, bool genPubli pubKey.keyFlags = RSTLV_KEY_DISTRIB_PUBLIC | RSTLV_KEY_TYPE_PUBLIC_ONLY; privPubKey.keyFlags = RSTLV_KEY_DISTRIB_PRIVATE | RSTLV_KEY_TYPE_FULL; - keySet.keys[pubKey.keyId] = pubKey; - keySet.keys[privPubKey.keyId] = privPubKey; + publickeySet.keys[pubKey.keyId] = pubKey; + privatekeySet.keys[privPubKey.keyId] = privPubKey; RSA_free(rsa_publish); RSA_free(rsa_publish_pub); } } -bool RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& keySet) +bool RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet) { std::cerr << "RsGenExchange::createGroup()"; std::cerr << std::endl; @@ -235,21 +236,13 @@ bool RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& keySet) // find private admin key RsTlvSecurityKey privAdminKey; - std::map::iterator mit = keySet.keys.begin(); + std::map::iterator mit = privateKeySet.keys.begin(); - bool privKeyFound = false; - for(; mit != keySet.keys.end(); mit++) + bool privKeyFound = false; // private admin key + for(; mit != privateKeySet.keys.end(); mit++) { RsTlvSecurityKey& key = mit->second; - // add public admin key - if((key.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) && (key.keyFlags & RSTLV_KEY_TYPE_PUBLIC_ONLY)) - meta->keys.keys.insert(std::make_pair(key.keyId, key)); - - // add public publish key - if((key.keyFlags & RSTLV_KEY_DISTRIB_PUBLIC) && (key.keyFlags & RSTLV_KEY_TYPE_PUBLIC_ONLY)) - meta->keys.keys.insert(std::make_pair(key.keyId, key)); - if((key.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) && (key.keyFlags & RSTLV_KEY_TYPE_FULL)) { privAdminKey = key; @@ -265,6 +258,8 @@ bool RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& keySet) return false; } + meta->keys = publicKeySet; // only public keys are included to be transported + // group is self signing // for the creation of group signature // only public admin and publish keys are present in meta @@ -290,9 +285,9 @@ bool RsGenExchange::createGroup(RsNxsGrp *grp, RsTlvSecurityKeySet& keySet) grp->meta.setBinData(metaData, metaDataLen); // but meta that is stored locally - // has all keys + // has private keys // nxs net transports only bin data - meta->keys = keySet; + meta->keys = privateKeySet; // clean up delete[] allGrpData; @@ -1245,7 +1240,7 @@ void RsGenExchange::publishMsgs() RsGxsMessageId msgId; RsGxsGroupId grpId = msgItem->meta.mGroupId; - bool msgDoesnExist = false; + bool msgDoesnExist = false, validSize = false; if(createOk) { @@ -1267,6 +1262,11 @@ void RsGenExchange::publishMsgs() } if(createOk && msgDoesnExist) + { + validSize = mDataStore->validSize(msg); + } + + if(createOk && msgDoesnExist && validSize) { // empty orig msg id means this is the original // msg @@ -1354,16 +1354,16 @@ void RsGenExchange::publishGrps() RsNxsGrp* grp = new RsNxsGrp(mServType); RsGxsGrpItem* grpItem = mit->second; - RsTlvSecurityKeySet keySet; - generateGroupKeys(keySet, + RsTlvSecurityKeySet privatekeySet, publicKeySet, tempKeySet; + generateGroupKeys(privatekeySet, publicKeySet, !(grpItem->meta.mGroupFlags & GXS_SERV::FLAG_PRIVACY_PUBLIC)); // find private admin key RsTlvSecurityKey privAdminKey; - std::map::iterator mit_keys = keySet.keys.begin(); + std::map::iterator mit_keys = privatekeySet.keys.begin(); bool privKeyFound = false; - for(; mit_keys != keySet.keys.end(); mit_keys++) + for(; mit_keys != privatekeySet.keys.end(); mit_keys++) { RsTlvSecurityKey& key = mit_keys->second; @@ -1386,8 +1386,14 @@ void RsGenExchange::publishGrps() ok = false; } + //tempKeySet = privatekeySet; + privatekeySet.keys.insert(publicKeySet.keys.begin(), + publicKeySet.keys.end()); + + service_CreateGroup(grpItem, privatekeySet); + //privatekeySet = tempKeySet; + - service_CreateGroup(grpItem, keySet); uint32_t size = mSerialiser->size(grpItem); char gData[size]; @@ -1407,21 +1413,28 @@ void RsGenExchange::publishGrps() *(grp->metaData) = grpItem->meta; grp->metaData->mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_ADMIN; - ok &= createGroup(grp, keySet); - - if (!ok) + if (!createGroup(grp, privatekeySet, publicKeySet)) { std::cerr << "RsGenExchange::publishGrps() !ok ERROR After createGroup" << std::endl; } + else + { - RsGxsGroupId grpId = grp->grpId; - mDataAccess->addGroupData(grp); + // ensure group size is not too large + ok &= mDataStore->validSize(grp); - std::cerr << "RsGenExchange::publishGrps() ok -> pushing to notifies" << std::endl; + if(ok) + { + RsGxsGroupId grpId = grp->grpId; + mDataAccess->addGroupData(grp); - // add to published to allow acknowledgement - mGrpNotify.insert(std::make_pair(mit->first, grpId)); - mDataAccess->updatePublicRequestStatus(mit->first, RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE); + std::cerr << "RsGenExchange::publishGrps() ok -> pushing to notifies" << std::endl; + + // add to published to allow acknowledgement + mGrpNotify.insert(std::make_pair(mit->first, grpId)); + mDataAccess->updatePublicRequestStatus(mit->first, RsTokenService::GXS_REQUEST_V2_STATUS_COMPLETE); + } + } } if(!ok) @@ -1505,16 +1518,16 @@ void RsGenExchange::createDummyGroup(RsGxsGrpItem *grpItem) bool ok = mSerialiser->serialise(grpItem, gData, &size); grp->grp.setBinData(gData, size); - RsTlvSecurityKeySet keySet; - generateGroupKeys(keySet, + RsTlvSecurityKeySet privateKeySet, publicKeySet; + generateGroupKeys(privateKeySet, publicKeySet, !(grpItem->meta.mGroupFlags & GXS_SERV::FLAG_PRIVACY_PUBLIC)); // find private admin key RsTlvSecurityKey privAdminKey; - std::map::iterator mit_keys = keySet.keys.begin(); + std::map::iterator mit_keys = privateKeySet.keys.begin(); bool privKeyFound = false; - for(; mit_keys != keySet.keys.end(); mit_keys++) + for(; mit_keys != privateKeySet.keys.end(); mit_keys++) { RsTlvSecurityKey& key = mit_keys->second; @@ -1536,7 +1549,7 @@ void RsGenExchange::createDummyGroup(RsGxsGrpItem *grpItem) ok = false; } - service_CreateGroup(grpItem, keySet); + service_CreateGroup(grpItem, privateKeySet); if(ok) { @@ -1544,7 +1557,7 @@ void RsGenExchange::createDummyGroup(RsGxsGrpItem *grpItem) grpItem->meta.mPublishTs = time(NULL); *(grp->metaData) = grpItem->meta; grp->metaData->mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_ADMIN; - createGroup(grp, keySet); + createGroup(grp, privateKeySet, publicKeySet); mDataAccess->addGroupData(grp); } diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index 42320f1c8..41d3ef1be 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -475,7 +475,7 @@ private: * Meta is serialised and stored in group at this point also * @param grp Nxs group to create */ - bool createGroup(RsNxsGrp* grp, RsTlvSecurityKeySet& keySet); + bool createGroup(RsNxsGrp* grp, RsTlvSecurityKeySet& privateKeySet, RsTlvSecurityKeySet& publicKeySet); /*! * This completes the creation of an instance on RsNxsMsg @@ -504,10 +504,11 @@ private: /*! * Generate a set of keys that can define a GXS group - * @param keySet this is set generated keys - * @param genPublicKeys should public keys also be generated + * @param privatekeySet contains private generated keys + * @param privatekeySet contains public generated keys (counterpart of private) + * @param genPublicKeys should publish key pair also be generated */ - void generateGroupKeys(RsTlvSecurityKeySet& keySet, bool genPublishKeys); + void generateGroupKeys(RsTlvSecurityKeySet& privatekeySet, RsTlvSecurityKeySet& publickeySet, bool genPublishKeys); /*! * Attempts to validate msg diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 758e34639..6ccbbab9b 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -25,6 +25,7 @@ */ #include +#include #include "rsgxsnetservice.h" #include "rsgxsflags.h" @@ -34,6 +35,7 @@ #define SYNC_PERIOD 12 // in microseconds every 10 seconds (1 second for testing) #define TRANSAC_TIMEOUT 5 // 5 seconds + const uint32_t RsGxsNetService::FRAGMENT_SIZE = 150000; RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs) @@ -129,6 +131,236 @@ void RsGxsNetService::syncWithPeers() //#endif } + +bool RsGxsNetService::fragmentMsg(RsNxsMsg& msg, MsgFragments& msgFragments) const +{ + // first determine how many fragments + uint32_t msgSize = msg.msg.TlvSize(); + uint32_t dataLeft = msgSize; + uint8_t nFragments = ceil(float(msgSize)/FRAGMENT_SIZE); + char buffer[FRAGMENT_SIZE]; + int currPos = 0; + + + for(uint8_t i=0; i < nFragments; i++) + { + RsNxsMsg* msgFrag = new RsNxsMsg(mServType); + msgFrag->grpId = msg.grpId; + msgFrag->msgId = msg.msgId; + msgFrag->meta = msg.meta; + msgFrag->pos = i; + msgFrag->count = nFragments; + uint32_t fragSize = std::min(dataLeft, FRAGMENT_SIZE); + + memcpy(buffer, ((char*)msg.msg.bin_data) + currPos, fragSize); + + currPos += fragSize; + dataLeft -= fragSize; + msgFragments.push_back(msgFrag); + } + + return true; +} + +bool RsGxsNetService::fragmentGrp(RsNxsGrp& grp, GrpFragments& grpFragments) const +{ + // first determine how many fragments + uint32_t grpSize = grp.grp.TlvSize(); + uint32_t dataLeft = grpSize; + uint8_t nFragments = ceil(float(grpSize)/FRAGMENT_SIZE); + char buffer[FRAGMENT_SIZE]; + int currPos = 0; + + + for(uint8_t i=0; i < nFragments; i++) + { + RsNxsGrp* grpFrag = new RsNxsGrp(mServType); + grpFrag->grpId = grp.grpId; + grpFrag->meta = grp.meta; + grpFrag->pos = i; + grpFrag->count = nFragments; + uint32_t fragSize = std::min(dataLeft, FRAGMENT_SIZE); + + memcpy(buffer, ((char*)grp.grp.bin_data) + currPos, fragSize); + + currPos += fragSize; + dataLeft -= fragSize; + grpFragments.push_back(grpFrag); + } + + return true; +} + +RsNxsMsg* RsGxsNetService::deFragmentMsg(MsgFragments& msgFragments) const +{ + if(msgFragments.empty()) return NULL; + + // first determine total size for binary data + MsgFragments::iterator mit = msgFragments.begin(); + uint32_t datSize = 0; + + for(; mit != msgFragments.end(); mit++) + datSize += (*mit)->msg.bin_len; + + char* data = new char[datSize]; + uint32_t currPos = 0; + + for(mit = msgFragments.begin(); mit != msgFragments.end(); mit++) + { + RsNxsMsg* msg = *mit; + memcpy(data + (currPos), msg->msg.bin_data, msg->msg.bin_len); + currPos += msg->msg.bin_len; + } + + RsNxsMsg* msg = new RsNxsMsg(mServType); + const RsNxsMsg& m = *(*(msgFragments.begin())); + msg->msg.setBinData(data, datSize); + msg->msgId = m.msgId; + msg->grpId = m.grpId; + msg->transactionNumber = m.transactionNumber; + msg->meta = m.meta; + + delete[] data; + return msg; +} + +RsNxsGrp* RsGxsNetService::deFragmentGrp(GrpFragments& grpFragments) const +{ + if(grpFragments.empty()) return NULL; + + // first determine total size for binary data + GrpFragments::iterator mit = grpFragments.begin(); + uint32_t datSize = 0; + + for(; mit != grpFragments.end(); mit++) + datSize += (*mit)->grp.bin_len; + + char* data = new char[datSize]; + uint32_t currPos = 0; + + for(mit = grpFragments.begin(); mit != grpFragments.end(); mit++) + { + RsNxsGrp* grp = *mit; + memcpy(data + (currPos), grp->grp.bin_data, grp->grp.bin_len); + currPos += grp->grp.bin_len; + } + + RsNxsGrp* grp = new RsNxsGrp(mServType); + const RsNxsGrp& g = *(*(grpFragments.begin())); + grp->grp.setBinData(data, datSize); + grp->grpId = g.grpId; + grp->transactionNumber = g.transactionNumber; + grp->meta = g.meta; + + delete[] data; + + return grp; +} + +struct GrpFragCollate +{ + RsGxsGroupId mGrpId; + GrpFragCollate(const RsGxsGroupId& grpId) : mGrpId(grpId){ } + bool operator()(RsNxsGrp* grp) { return grp->grpId == mGrpId;} +}; + +void RsGxsNetService::collateGrpFragments(GrpFragments fragments, + std::map& partFragments) const +{ + // get all unique grpIds; + GrpFragments::iterator vit = fragments.begin(); + std::set grpIds; + + for(; vit != fragments.end(); vit++) + grpIds.insert( (*vit)->grpId ); + + std::set::iterator sit = grpIds.begin(); + + for(; sit != grpIds.end(); sit++) + { + const RsGxsGroupId& grpId = *sit; + GrpFragments::iterator bound = std::partition( + fragments.begin(), fragments.end(), + GrpFragCollate(grpId)); + + // something will always be found for a group id + for(vit = fragments.begin(); vit != bound; ) + { + partFragments[grpId].push_back(*vit); + vit = fragments.erase(vit); + } + + GrpFragments& f = partFragments[grpId]; + RsNxsGrp* grp = *(f.begin()); + + // if counts of fragments is incorrect remove + // from coalescion + if(grp->count != f.size()) + { + GrpFragments::iterator vit2 = f.begin(); + + for(; vit2 != f.end(); vit2++) + delete *vit2; + + partFragments.erase(grpId); + } + } + + fragments.clear(); +} + +struct MsgFragCollate +{ + RsGxsGroupId mMsgId; + MsgFragCollate(const RsGxsMessageId& msgId) : mMsgId(msgId){ } + bool operator()(RsNxsMsg* msg) { return msg->msgId == mMsgId;} +}; + +void RsGxsNetService::collateMsgFragments(MsgFragments fragments, std::map& partFragments) const +{ + // get all unique message Ids; + MsgFragments::iterator vit = fragments.begin(); + std::set msgIds; + + for(; vit != fragments.end(); vit++) + msgIds.insert( (*vit)->msgId ); + + + std::set::iterator sit = msgIds.begin(); + + for(; sit != msgIds.end(); sit++) + { + const RsGxsMessageId& msgId = *sit; + MsgFragments::iterator bound = std::partition( + fragments.begin(), fragments.end(), + MsgFragCollate(msgId)); + + // something will always be found for a group id + for(vit = fragments.begin(); vit != bound; ) + { + partFragments[msgId].push_back(*vit); + vit = fragments.erase(vit); + } + + MsgFragments& f = partFragments[msgId]; + RsNxsMsg* msg = *(f.begin()); + + // if counts of fragments is incorrect remove + // from coalescion + if(msg->count != f.size()) + { + MsgFragments::iterator vit2 = f.begin(); + + for(; vit2 != f.end(); vit2++) + delete *vit2; + + partFragments.erase(msgId); + } + } + + fragments.clear(); +} + bool RsGxsNetService::loadList(std::list& load) { return false; diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index f555dd5d4..3b671bfef 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -133,7 +133,7 @@ class RsGxsNetService : public RsNetworkExchangeService, public p3ThreadedServic { public: - + static const uint32_t FRAGMENT_SIZE; /*! * only one observer is allowed * @param servType service type @@ -365,6 +365,60 @@ private: private: + typedef std::vector GrpFragments; + typedef std::vector MsgFragments; + + /*! + * Fragment a message into individual fragments which are at most 150kb + * @param msg message to fragment + * @param msgFragments fragmented message + * @return false if fragmentation fails true otherwise + */ + bool fragmentMsg(RsNxsMsg& msg, MsgFragments& msgFragments) const; + + /*! + * Fragment a group into individual fragments which are at most 150kb + * @param grp group to fragment + * @param grpFragments fragmented group + * @return false if fragmentation fails true other wise + */ + bool fragmentGrp(RsNxsGrp& grp, GrpFragments& grpFragments) const; + + /*! + * Fragment a message into individual fragments which are at most 150kb + * @param msg message to fragment + * @param msgFragments fragmented message + * @return NULL if not possible to reconstruct message from fragment, + * pointer to defragments nxs message is possible + */ + RsNxsMsg* deFragmentMsg(MsgFragments& msgFragments) const; + + /*! + * Fragment a group into individual fragments which are at most 150kb + * @param grp group to fragment + * @param grpFragments fragmented group + * @return NULL if not possible to reconstruct group from fragment, + * pointer to defragments nxs group is possible + */ + RsNxsGrp* deFragmentGrp(GrpFragments& grpFragments) const; + + + /*! + * Note that if all fragments for a message are not found then its fragments are dropped + * @param fragments message fragments which are not necessarily from the same message + * @param partFragments the partitioned fragments (into message ids) + */ + void collateMsgFragments(MsgFragments fragments, std::map& partFragments) const; + + /*! + * Note that if all fragments for a group are not found then its fragments are dropped + * @param fragments group fragments which are not necessarily from the same group + * @param partFragments the partitioned fragments (into message ids) + */ + void collateGrpFragments(GrpFragments fragments, std::map& partFragments) const; +private: + + /*** transactions ***/ /// active transactions diff --git a/libretroshare/src/serialiser/rsnxsitems.cc b/libretroshare/src/serialiser/rsnxsitems.cc index 448872b6a..39b7254b7 100644 --- a/libretroshare/src/serialiser/rsnxsitems.cc +++ b/libretroshare/src/serialiser/rsnxsitems.cc @@ -248,6 +248,7 @@ bool RsNxsSerialiser::serialiseNxsMsg(RsNxsMsg *item, void *data, uint32_t *size offset += 8; ok &= setRawUInt32(data, *size, &offset, item->transactionNumber); + ok &= setRawUInt8(data, *size, &offset, item->pos); ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_MSGID, item->msgId); ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_GROUPID, item->grpId); ok &= item->msg.SetTlv(data, tlvsize, &offset); @@ -299,6 +300,7 @@ bool RsNxsSerialiser::serialiseNxsGrp(RsNxsGrp *item, void *data, uint32_t *size // grp id ok &= setRawUInt32(data, *size, &offset, item->transactionNumber); + ok &= setRawUInt8(data, *size, &offset, item->pos); ok &= SetTlvString(data, tlvsize, &offset, TLV_TYPE_STR_GROUPID, item->grpId); ok &= item->grp.SetTlv(data, tlvsize, &offset); ok &= item->meta.SetTlv(data, *size, &offset); @@ -564,6 +566,7 @@ RsNxsGrp* RsNxsSerialiser::deserialNxsGrp(void *data, uint32_t *size){ offset += 8; ok &= getRawUInt32(data, *size, &offset, &(item->transactionNumber)); + ok &= getRawUInt8(data, *size, &offset, &(item->pos)); ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId); ok &= item->grp.GetTlv(data, *size, &offset); ok &= item->meta.GetTlv(data, *size, &offset); @@ -632,6 +635,7 @@ RsNxsMsg* RsNxsSerialiser::deserialNxsMsg(void *data, uint32_t *size){ offset += 8; ok &= getRawUInt32(data, *size, &offset, &(item->transactionNumber)); + ok &= getRawUInt8(data, *size, &offset, &(item->pos)); ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_MSGID, item->msgId); ok &= GetTlvString(data, *size, &offset, TLV_TYPE_STR_GROUPID, item->grpId); ok &= item->msg.GetTlv(data, *size, &offset); @@ -1015,6 +1019,7 @@ uint32_t RsNxsSerialiser::sizeNxsMsg(RsNxsMsg *item) uint32_t s = 8; //header size s += 4; // transaction number + s += 1; // pos s += GetTlvStringSize(item->grpId); s += GetTlvStringSize(item->msgId); s += item->msg.TlvSize(); @@ -1028,6 +1033,7 @@ uint32_t RsNxsSerialiser::sizeNxsGrp(RsNxsGrp *item) uint32_t s = 8; // header size s += 4; // transaction number + s += 1; // pos s += GetTlvStringSize(item->grpId); s += item->grp.TlvSize(); s += item->meta.TlvSize(); @@ -1253,6 +1259,8 @@ std::ostream& RsNxsGrp::print(std::ostream &out, uint16_t indent){ out << "grpId: " << grpId << std::endl; printIndent(out , int_Indent); out << "grp: " << std::endl; + printIndent(out , int_Indent); + out << "pos: " << pos << std::endl; grp.print(out, int_Indent); out << "meta: " << std::endl; meta.print(out, int_Indent); @@ -1269,6 +1277,8 @@ std::ostream& RsNxsMsg::print(std::ostream &out, uint16_t indent){ out << "msgId: " << msgId << std::endl; printIndent(out , int_Indent); out << "grpId: " << grpId << std::endl; + printIndent(out , int_Indent); + out << "pos: " << pos << std::endl; printIndent(out , int_Indent); out << "msg: " << std::endl; msg.print(out, indent); diff --git a/libretroshare/src/serialiser/rsnxsitems.h b/libretroshare/src/serialiser/rsnxsitems.h index 968850789..7a6770f6e 100644 --- a/libretroshare/src/serialiser/rsnxsitems.h +++ b/libretroshare/src/serialiser/rsnxsitems.h @@ -202,6 +202,8 @@ public: std::string grpId; /// group Id, needed to complete version Id (ncvi) static int refcount; RsTlvBinaryData grp; /// actual group data + uint8_t pos; /// used for splitting up grp + uint8_t count; /// number of split up messages /*! * This should contains all data @@ -284,6 +286,8 @@ public: virtual void clear(); virtual std::ostream &print(std::ostream &out, uint16_t indent); + uint8_t pos; /// used for splitting up msg + uint8_t count; /// number of split up messages std::string grpId; /// group id, forms part of version id std::string msgId; /// msg id static int refcount;