diff --git a/build_scripts/Debian+Ubuntu/changelog b/build_scripts/Debian+Ubuntu/changelog index e89d1c3ca..867933372 100644 --- a/build_scripts/Debian+Ubuntu/changelog +++ b/build_scripts/Debian+Ubuntu/changelog @@ -1,5 +1,76 @@ retroshare06 (0.6.0-1.XXXXXX~YYYYYY) YYYYYY; urgency=low + 0ff6349 thunder2 Fri, 15 Jan 2016 12:01:37 +0100 Fixed possibility of XXE injection in FeedReader plugin. + 748147f thunder2 Fri, 15 Jan 2016 12:00:02 +0100 Fixed Windows compile. + b5f6059 Cyril Soler Fri, 15 Jan 2016 10:07:21 -0500 deprecated updateClientSyncTS() which is not needed anymore + c24447f csoler Thu, 14 Jan 2016 20:11:26 -0500 fixed update of mGrpClientUpdateMap when no new group is available, avoiding unnecessary GXS grp list traffic (patch from jolavillette) + 3b18c43 csoler Wed, 13 Jan 2016 20:25:18 -0500 removed [dos] LF from rsgxsnetservice (causes merging rampage) + 086b076 csoler Wed, 13 Jan 2016 12:22:55 -0500 fixed a few serialisation issues + 96f3446 csoler Wed, 13 Jan 2016 11:13:23 -0500 added size limit to packet grouping + b56881e csoler Wed, 13 Jan 2016 10:23:21 -0500 fixed windows compilation + 4916496 csoler Wed, 13 Jan 2016 10:20:51 -0500 Merge pull request #253 from csoler/v0.6-SecurityReview + 2c019d9 csoler Wed, 13 Jan 2016 00:39:22 -0500 added load image option in forum posts when images are to be displayed + 60408b3 csoler Wed, 13 Jan 2016 00:13:16 -0500 fixed a bug in GRouterGenericDataItem::duplicate() + 2a8fc7f csoler Wed, 13 Jan 2016 00:08:27 -0500 hardened the restriction to image loading to avoid external resources in RSTextBrowser + d55993d csoler Tue, 12 Jan 2016 21:43:04 -0500 removed two instances of malloc(0) captured by new rs_malloc funtion + d13526f csoler Tue, 12 Jan 2016 21:10:11 -0500 added a new method rs_malloc that checks its arguments and prints a stacktrace on error/weird call. + Changed the code everywhere to use this instead of malloc. Removed some mallocs and replaced with RsTemporaryMemory + 9c6e7df csoler Mon, 11 Jan 2016 23:49:00 -0500 added checks after mallocs in several files + 46520b0 csoler Mon, 11 Jan 2016 20:49:26 -0500 fixed potential integer overflow / Out of bounds read in gxstunnelitems.cc + 8e666fc csoler Mon, 11 Jan 2016 20:40:57 -0500 fixed potential integer overflow / Out of bounds read in rsturtleitems.cc + 6e9d96e csoler Mon, 11 Jan 2016 20:26:19 -0500 fixed potential integer overflow / Out of bounds read in RsVOIPDataItem.cc + 98f0c10 csoler Mon, 11 Jan 2016 19:26:54 -0500 fixed potential integer overflow / Out of bounds read in GRouterItems.cc + 3094b52 csoler Mon, 11 Jan 2016 19:08:52 -0500 fixed potential integer overflow / Out of bounds read in rsbaseserial.cc. Also added a few more error output. + 8e6998b electron128 Sun, 10 Jan 2016 14:34:08 +0100 removed redundant map _lobby_ids from DistributedChatService. The same information and more is stored in the _chat_lobbys map. + _lobby_ids is unused since commit 67e1495d9a78ac259649014355cd127a2febeaaf + 09ed1ea electron128 Sun, 10 Jan 2016 14:22:06 +0100 removed getVirtualPeerId method from rsmsgs lobby interface. It is not neccesary anymore to have the virtual peer id exposed to the GUI layer. + 67e1495 electron128 Sun, 10 Jan 2016 13:29:40 +0100 use ChatId instead of virtual peer id for chat lobby popup. Removed a now unused method from rsmsgs interface and DistributedChatService. + c3d422a csoler Sat, 9 Jan 2016 14:31:20 -0500 added some tooltips on keys and certificate details + 994bfaf csoler Sat, 9 Jan 2016 13:54:24 -0500 updated TODO + e01b1a4 csoler Sat, 9 Jan 2016 11:00:57 -0500 Merge pull request #245 from csoler/v0.6-Messaging + ee84ab7 csoler Sat, 9 Jan 2016 10:58:49 -0500 fixed serialisation error (missing call) for received Distant msg hash map + b125cb1 electron128 Sat, 9 Jan 2016 12:09:43 +0100 fix the bug where a filter is applied in friends file list even if view mode is tree and search box is hidden. Problem was: + SharedFilesDialog::FilterItems() is called from multiple places. The check if we should filter or not is now in the filter function itself. + a118870 electron128 Sat, 9 Jan 2016 12:01:32 +0100 if file in friends file list is available locally or is being downloaded, then show it in red like in search. The color is hardcoded, because + it is not possible to style arbitrary QObjects with stylesheets. closes #195 + 9f4256c csoler Fri, 8 Jan 2016 23:19:45 -0500 Merge pull request #244 from csoler/v0.6-Messaging + fe02167 csoler Fri, 8 Jan 2016 23:18:45 -0500 disabled part of msg re-sending code in order to improve it later + 56a844b csoler Fri, 8 Jan 2016 23:08:14 -0500 Merge pull request #243 from csoler/v0.6-Messaging + a540ffd csoler Fri, 8 Jan 2016 22:52:01 -0500 removed two debug outputs from MessageDialog + 3f6964b csoler Fri, 8 Jan 2016 22:50:03 -0500 added back messages to sent box + 75edb46 csoler Fri, 8 Jan 2016 20:48:47 -0500 added auto-resend of unsent messages. Fixed display of From field in outbox + 0c591f0 csoler Thu, 7 Jan 2016 22:22:05 -0500 added auto re-send of distant messages after global router notified the message cannot be sent. Added a map n p3msgService to avoid + receiving multiple times the same message. + 9651f43 csoler Thu, 7 Jan 2016 21:01:50 -0500 fixed wrong usage of local client ids in global router causing distant messages to stay in outbox forever. + 81e2a59 defnax Thu, 7 Jan 2016 19:25:52 +0100 Fixing People Margins Update emotes file + 4484b08 csoler Thu, 7 Jan 2016 00:04:52 -0500 cleaned up the problem of outgoing messages being shown with the from from field. + 3d6fa0c csoler Wed, 6 Jan 2016 18:38:27 -0500 fixed bug causing turtle to keep asking for a tunnel after a distant chat is closed if not connected + 61f8e73 csoler Wed, 6 Jan 2016 18:12:15 -0500 improved text of warning against pasting pgp keys in connect wizard + 984ce2d csoler Wed, 6 Jan 2016 18:07:20 -0500 removed circles tab from IdDialog. Came from a difficult manual merging between dos and unix encoded files + 7da2ad1 csoler Wed, 6 Jan 2016 10:31:59 -0500 re-enabled previously disabled code in rsgxsnetservice, that proved useful + 6398b7b csoler Tue, 5 Jan 2016 23:00:26 -0500 added comment in rsgxsnetservice.cc + 3fa8d86 csoler Tue, 5 Jan 2016 22:53:56 -0500 removed time-stamp-ing of messages on client side in locked_genReqMsgTransaction() as it is already handled when receiving messages, + and the need to stamp non subscribed groups is not justified anymore + 678afe7 csoler Tue, 5 Jan 2016 22:27:00 -0500 fixed stupid mistake in previous commit (GXS transaction timestamp) + 7b3fd53 csoler Tue, 5 Jan 2016 19:23:59 -0500 added correct timestamp in msg list response items for GXS. To be tested. + 141f092 csoler Tue, 5 Jan 2016 14:22:27 -0500 added tooltip and fixed text in connect friend dialog + 496546e defnax Tue, 5 Jan 2016 18:25:44 +0100 Merge branch 'master' of https://github.com/RetroShare/RetroShare + 19011ba defnax Tue, 5 Jan 2016 18:23:43 +0100 clean up emotes, removed some unused. changed icons for emotes button. reduze the size of the emoticons + 9fd0fc5 csoler Tue, 5 Jan 2016 11:43:50 -0500 fixed bandwidth measurement in pqistreamer + e6cd4a4 defnax Tue, 5 Jan 2016 16:09:14 +0100 Merge pull request #240 from sehraf/pr-smiley-whitespace + 54f9912 sehraf Tue, 5 Jan 2016 14:58:41 +0100 Add whitespaces around smileys. Only replace smileys with an image when they are surrounded by whitespaces. + This fixes smileys in links and other text that happen to include the sequence for a smiley. + f1f111f csoler Mon, 4 Jan 2016 20:05:44 -0500 merged + e04b933 B.Cristea Mon, 4 Jan 2016 15:25:22 +0200 corrected compilation inst on openSUSE + fddc008 csoler Sun, 3 Jan 2016 17:31:22 -0500 merged with origin + 3d5042b csoler Sun, 3 Jan 2016 14:29:33 -0500 Merge pull request #237 from G10h4ck/master + 05fd33b Gio Sun, 3 Jan 2016 17:21:11 +0100 p3ServiceControl: Optimize checkFilter(...), solve memory leak removing mSerialiser, code cleanup + 1082a8d csoler Sat, 2 Jan 2016 18:59:05 -0500 updated ubuntu changelog + + -- Cyril Soler Thu, 14 Jan 2016 20:00:00 +0100 + +retroshare06 (0.6.0-1.20160102.1082a8d1~trusty) trusty; urgency=low + 4953324 electron128 Sat, 2 Jan 2016 14:53:24 +0100 Merge pull request #235 from PhenomRetroShare/Fix_MakeMACOS_PluginWorking c5e6f62 Phenom Sat, 2 Jan 2016 14:19:53 +0100 Make Plugin working in Mac OSX. b6b5f9c csoler Fri, 1 Jan 2016 22:36:07 -0500 fixed some of the issues reported by coverity scan (mainly uninitialised class members) diff --git a/libbitdht/src/bitdht/bdnode.cc b/libbitdht/src/bitdht/bdnode.cc index 9161c6991..a994db475 100644 --- a/libbitdht/src/bitdht/bdnode.cc +++ b/libbitdht/src/bitdht/bdnode.cc @@ -2399,6 +2399,13 @@ bdNodeNetMsg::bdNodeNetMsg(char *msg, int len, struct sockaddr_in *in_addr) :data(NULL), mSize(len), addr(*in_addr) { data = (char *) malloc(len); + + if(data == NULL) + { + std::cerr << "(EE) " << __PRETTY_FUNCTION__ << ": ERROR. cannot allocate memory for " << len << " bytes." << std::endl; + return ; + } + memcpy(data, msg, len); //print(std::cerr); } diff --git a/libbitdht/src/bitdht/bencode.c b/libbitdht/src/bitdht/bencode.c index 2a486daca..4678677f7 100644 --- a/libbitdht/src/bitdht/bencode.c +++ b/libbitdht/src/bitdht/bencode.c @@ -20,6 +20,7 @@ */ #include +#include #include /* malloc() realloc() free() strtoll() */ #include /* memset() */ #include "util/bdstring.h" @@ -111,6 +112,13 @@ static char *_be_decode_str(const char **data, long long *data_len) if (**data == ':') { char *_ret = (char *) malloc(sizeof(sllen) + len + 1); + + if(_ret == NULL) + { + std::cerr << "(EE) " << __PRETTY_FUNCTION__ << ": ERROR. cannot allocate memory for " << len+1+sizeof(sllen) << " bytes." << std::endl; + return NULL; + } + memcpy(_ret, &sllen, sizeof(sllen)); ret = _ret + sizeof(sllen); memcpy(ret, *data + 1, len); @@ -500,6 +508,12 @@ be_node *be_create_str(const char *str) int len = strlen(str); long long int sllen = len; char *_ret = (char *) malloc(sizeof(sllen) + len + 1); + + if(_ret == NULL) + { + std::cerr << "(EE) " << __PRETTY_FUNCTION__ << ": ERROR. cannot allocate memory for " << len+1+sizeof(sllen) << " bytes." << std::endl; + return NULL; + } char *ret = NULL; memcpy(_ret, &sllen, sizeof(sllen)); @@ -519,6 +533,12 @@ be_node *be_create_str_wlen(const char *str, int len) /* not including \0 */ be_node *n = be_alloc(BE_STR); long long int sllen = len; char *_ret = (char *) malloc(sizeof(sllen) + len + 1); + + if(_ret == NULL) + { + std::cerr << "(EE) " << __PRETTY_FUNCTION__ << ": ERROR. cannot allocate memory for " << len+1+sizeof(sllen) << " bytes." << std::endl; + return NULL; + } char *ret = NULL; memcpy(_ret, &sllen, sizeof(sllen)); @@ -561,6 +581,12 @@ int be_add_keypair(be_node *dict, const char *str, be_node *node) int len = strlen(str); long long int sllen = len; char *_ret = (char *) malloc(sizeof(sllen) + len + 1); + + if(_ret == NULL) + { + std::cerr << "(EE) " << __PRETTY_FUNCTION__ << ": ERROR. cannot allocate memory for " << len+1+sizeof(sllen) << " bytes." << std::endl; + return 0; + } char *ret = NULL; //fprintf(stderr, "be_add_keypair() key len = %d\n",len); diff --git a/libbitdht/src/udp/udplayer.cc b/libbitdht/src/udp/udplayer.cc index 71dfd4f9a..db048b857 100644 --- a/libbitdht/src/udp/udplayer.cc +++ b/libbitdht/src/udp/udplayer.cc @@ -66,7 +66,11 @@ class udpPacket :raddr(*addr), len(dlen) { data = malloc(len); - memcpy(data, dta, len); + + if(data != NULL) + memcpy(data, dta, len); + else + std::cerr << "(EE) error in memory allocation in " << __PRETTY_FUNCTION__ << std::endl; } ~udpPacket() @@ -242,6 +246,12 @@ void UdpLayer::recv_loop() int maxsize = 16000; void *inbuf = malloc(maxsize); + if(inbuf == NULL) + { + std::cerr << "(EE) Error in memory allocation of size " << maxsize << " in " << __PRETTY_FUNCTION__ << std::endl; + return ; + } + int status; struct timeval timeout; diff --git a/libbitdht/src/util/bdbloom.cc b/libbitdht/src/util/bdbloom.cc index 1d6566d52..36f3a3bee 100644 --- a/libbitdht/src/util/bdbloom.cc +++ b/libbitdht/src/util/bdbloom.cc @@ -100,6 +100,13 @@ int bloomFilter::setFilterBits(const std::string &hex) // convert to binary array. uint8_t *tmparray = (uint8_t *) malloc(bytes); + + if(tmparray == NULL) + { + std::cerr << "(EE) Error. Cannot allocate memory for " << bytes << " bytes in " << __PRETTY_FUNCTION__ << std::endl; + return 0; + } + uint32_t i = 0; for(i = 0; i < bytes; i++) @@ -139,6 +146,13 @@ std::string bloomFilter::getFilter() // convert to binary array. uint8_t *tmparray = (uint8_t *) malloc(bytes); + + if(tmparray == NULL) + { + std::cerr << "(EE) Error. Cannot allocate memory for " << bytes << " bytes in " << __PRETTY_FUNCTION__ << std::endl; + return std::string(); + } + int i,j; for(i = 0; i < bytes; i++) diff --git a/libretroshare/src/dbase/fimonitor.cc b/libretroshare/src/dbase/fimonitor.cc index 91a8ffb9b..c4f60ee81 100644 --- a/libretroshare/src/dbase/fimonitor.cc +++ b/libretroshare/src/dbase/fimonitor.cc @@ -29,6 +29,7 @@ #include "rsserver/p3face.h" #include "dbase/fimonitor.h" #include "util/rsdir.h" +#include "util/rsmemory.h" #include "pqi/authssl.h" #include "serialiser/rsserviceids.h" #include "retroshare/rsiface.h" @@ -128,13 +129,11 @@ HashCache::HashCache(const std::string& path) // read the binary stream into memory. // - void *buffer = malloc(file_size) ; + void *buffer = rs_malloc(file_size) ; if(buffer == NULL) - { - std::cerr << "Cannot allocate memory for reading encrypted file cache, bytes=" << file_size << std::endl; return ; - } + FILE *F = fopen( (_path+".bin").c_str(),"rb") ; if (!F) { diff --git a/libretroshare/src/ft/ftcontroller.cc b/libretroshare/src/ft/ftcontroller.cc index b58650a1a..fc66c7f2a 100644 --- a/libretroshare/src/ft/ftcontroller.cc +++ b/libretroshare/src/ft/ftcontroller.cc @@ -39,6 +39,7 @@ #include "util/rsstring.h" #endif #include "util/rsdiscspace.h" +#include "util/rsmemory.h" #include "ft/ftcontroller.h" @@ -747,8 +748,14 @@ bool ftController::copyFile(const std::string& source,const std::string& dest) size_t T=0; static const int BUFF_SIZE = 10485760 ; // 10 MB buffer to speed things up. - void *buffer = malloc(BUFF_SIZE) ; + void *buffer = rs_malloc(BUFF_SIZE) ; + if(buffer == NULL) + { + fclose (in); + fclose (out); + return false ; + } bool bRet = true; while( (s = fread(buffer,1,BUFF_SIZE,in)) > 0) diff --git a/libretroshare/src/ft/ftdatamultiplex.cc b/libretroshare/src/ft/ftdatamultiplex.cc index 05c8265ef..a30545b38 100644 --- a/libretroshare/src/ft/ftdatamultiplex.cc +++ b/libretroshare/src/ft/ftdatamultiplex.cc @@ -35,6 +35,7 @@ #include "ft/ftfileprovider.h" #include "ft/ftsearch.h" #include "util/rsdir.h" +#include "util/rsmemory.h" #include #include @@ -878,13 +879,11 @@ bool ftDataMultiplex::locked_handleServerRequest(ftFileProvider *provider, const std::cerr << "Warning: peer " << peerId << " is asking a large chunk (s=" << chunksize << ") for hash " << hash << ", filesize=" << size << ". This is unexpected." << std::endl ; return false ; } - void *data = malloc(chunksize); + void *data = rs_malloc(chunksize); if(data == NULL) - { - std::cerr << "WARNING: Could not allocate data for a chunksize of " << chunksize << std::endl ; return false ; - } + #ifdef MPLEX_DEBUG std::cerr << "ftDataMultiplex::locked_handleServerRequest()"; std::cerr << "\t peer: " << peerId << " hash: " << hash; diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index ae0c8354a..2bcb53a5f 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -465,6 +465,8 @@ RsTurtleGenericTunnelItem *ftServer::deserialiseItem(void *data,uint32_t size) c return NULL; /* wrong type */ } + try + { switch(getRsItemSubType(rstype)) { case RS_TURTLE_SUBTYPE_FILE_REQUEST : return new RsTurtleFileRequestItem(data,size) ; @@ -477,6 +479,13 @@ RsTurtleGenericTunnelItem *ftServer::deserialiseItem(void *data,uint32_t size) c default: return NULL ; } + } + catch(std::exception& e) + { + std::cerr << "(EE) deserialisation error in " << __PRETTY_FUNCTION__ << ": " << e.what() << std::endl; + + return NULL ; + } } void ftServer::addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) @@ -1093,11 +1102,10 @@ bool ftServer::sendData(const RsPeerId& peerId, const RsFileHash& hash, uint64_t item->chunk_offset = offset+baseoffset ; item->chunk_size = chunk; - item->chunk_data = malloc(chunk) ; + item->chunk_data = rs_malloc(chunk) ; if(item->chunk_data == NULL) { - std::cerr << "p3turtle: Warning: failed malloc of " << chunk << " bytes for sending data packet." << std::endl ; delete item; return false; } diff --git a/libretroshare/src/ft/ftturtlefiletransferitem.cc b/libretroshare/src/ft/ftturtlefiletransferitem.cc index 938c76796..4ec1123cc 100644 --- a/libretroshare/src/ft/ftturtlefiletransferitem.cc +++ b/libretroshare/src/ft/ftturtlefiletransferitem.cc @@ -26,6 +26,7 @@ #include #include +#include #include #include @@ -422,14 +423,24 @@ RsTurtleFileDataItem::RsTurtleFileDataItem(void *data,uint32_t pktsize) uint32_t offset = 8; // skip the header uint32_t rssize = getRsItemSize(data); - /* add mandatory parts first */ - bool ok = true ; + + if(rssize > pktsize) + ok = false ; + + /* add mandatory parts first */ ok &= getRawUInt32(data, pktsize, &offset, &tunnel_id) ; ok &= getRawUInt64(data, pktsize, &offset, &chunk_offset); ok &= getRawUInt32(data, pktsize, &offset, &chunk_size); - chunk_data = (void*)malloc(chunk_size) ; + if(chunk_size > rssize || rssize - chunk_size < offset) + throw std::runtime_error("RsTurtleFileDataItem::() error while deserializing.") ; + + chunk_data = (void*)rs_malloc(chunk_size) ; + + if(chunk_data == NULL) + throw std::runtime_error("RsTurtleFileDataItem::() cannot allocate memory.") ; + memcpy(chunk_data,(void*)((unsigned char*)data+offset),chunk_size) ; offset += chunk_size ; diff --git a/libretroshare/src/grouter/grouteritems.cc b/libretroshare/src/grouter/grouteritems.cc index b67ed71e6..cab25c43e 100644 --- a/libretroshare/src/grouter/grouteritems.cc +++ b/libretroshare/src/grouter/grouteritems.cc @@ -66,6 +66,12 @@ RsGRouterTransactionChunkItem *RsGRouterSerialiser::deserialise_RsGRouterTransac uint32_t rssize = getRsItemSize(data); bool ok = true ; + if(tlvsize < rssize) + { + std::cerr << __PRETTY_FUNCTION__ << ": wrong encoding of item size. Serialisation error!" << std::endl; + return NULL ; + } + RsGRouterTransactionChunkItem *item = new RsGRouterTransactionChunkItem() ; /* add mandatory parts first */ @@ -74,15 +80,14 @@ RsGRouterTransactionChunkItem *RsGRouterSerialiser::deserialise_RsGRouterTransac ok &= getRawUInt32(data, tlvsize, &offset, &item->chunk_size); ok &= getRawUInt32(data, tlvsize, &offset, &item->total_size); - if( NULL == (item->chunk_data = (uint8_t*)malloc(item->chunk_size))) + if(item->chunk_size > rssize || offset > rssize - item->chunk_size) // better than if(item->chunk_size + offset > rssize) { - std::cerr << __PRETTY_FUNCTION__ << ": Cannot allocate memory for chunk " << item->chunk_size << std::endl; + std::cerr << __PRETTY_FUNCTION__ << ": Cannot read beyond item size. Serialisation error!" << std::endl; delete item; return NULL ; } - if(item->chunk_size + offset > rssize) + if( NULL == (item->chunk_data = (uint8_t*)rs_malloc(item->chunk_size))) { - std::cerr << __PRETTY_FUNCTION__ << ": Cannot read beyond item size. Serialisation error!" << std::endl; delete item; return NULL ; } @@ -125,36 +130,40 @@ RsGRouterGenericDataItem *RsGRouterSerialiser::deserialise_RsGRouterGenericDataI uint32_t rssize = getRsItemSize(data); bool ok = true ; + if(pktsize < rssize) + { + std::cerr << __PRETTY_FUNCTION__ << ": wrong encoding of item size. Serialisation error!" << std::endl; + return NULL ; + } RsGRouterGenericDataItem *item = new RsGRouterGenericDataItem() ; ok &= getRawUInt64(data, pktsize, &offset, &item->routing_id); ok &= item->destination_key.deserialise(data, pktsize, offset) ; - ok &= getRawUInt32(data, pktsize, &offset, &item->service_id); - ok &= getRawUInt32(data, pktsize, &offset, &item->data_size); + ok &= getRawUInt32(data, pktsize, &offset, &item->service_id); + ok &= getRawUInt32(data, pktsize, &offset, &item->data_size); - if( NULL == (item->data_bytes = (uint8_t*)malloc(item->data_size))) + if(item->data_size > rssize || offset > rssize - item->data_size) // better than if(item->data_size + offset > rssize) { - std::cerr << __PRETTY_FUNCTION__ << ": Cannot allocate memory for chunk " << item->data_size << std::endl; + std::cerr << __PRETTY_FUNCTION__ << ": Cannot read beyond item size. Serialisation error!" << std::endl; delete item; return NULL ; } - if(item->data_size + offset > rssize) - { - std::cerr << __PRETTY_FUNCTION__ << ": Cannot read beyond item size. Serialisation error!" << std::endl; - delete item; - return NULL ; - } + if( NULL == (item->data_bytes = (uint8_t*)rs_malloc(item->data_size))) + { + delete item; + return NULL ; + } - memcpy(item->data_bytes,&((uint8_t*)data)[offset],item->data_size) ; + memcpy(item->data_bytes,&((uint8_t*)data)[offset],item->data_size) ; offset += item->data_size ; - ok &= item->signature.GetTlv(data, pktsize, &offset) ; + ok &= item->signature.GetTlv(data, pktsize, &offset) ; - ok &= getRawUInt32(data, pktsize, &offset, &item->randomized_distance); - ok &= getRawUInt32(data, pktsize, &offset, &item->flags); + ok &= getRawUInt32(data, pktsize, &offset, &item->randomized_distance); + ok &= getRawUInt32(data, pktsize, &offset, &item->flags); - if (offset != rssize || !ok) + if (offset != rssize || !ok) { std::cerr << __PRETTY_FUNCTION__ << ": error while deserialising! Item will be dropped." << std::endl; delete item; @@ -337,8 +346,19 @@ RsGRouterGenericDataItem *RsGRouterGenericDataItem::duplicate() const // then duplicate the memory chunk - item->data_bytes = (uint8_t*)malloc(data_size) ; - memcpy(item->data_bytes,data_bytes,data_size) ; + if(data_size > 0) + { + item->data_bytes = (uint8_t*)rs_malloc(data_size) ; + + if(item->data_bytes == NULL) + { + delete item ; + return NULL ; + } + memcpy(item->data_bytes,data_bytes,data_size) ; + } + else + item->data_bytes = NULL ; return item ; } diff --git a/libretroshare/src/grouter/grouteritems.h b/libretroshare/src/grouter/grouteritems.h index 574c7b3c4..ad87d1051 100644 --- a/libretroshare/src/grouter/grouteritems.h +++ b/libretroshare/src/grouter/grouteritems.h @@ -25,6 +25,8 @@ #pragma once +#include "util/rsmemory.h" + #include "serialiser/rsserial.h" #include "serialiser/rstlvkeys.h" #include "serialiser/rsserviceids.h" @@ -194,7 +196,11 @@ class RsGRouterTransactionChunkItem: public RsGRouterTransactionItem, public RsG { RsGRouterTransactionChunkItem *item = new RsGRouterTransactionChunkItem ; *item = *this ; // copy all fields - item->chunk_data = (uint8_t*)malloc(chunk_size) ; // deep copy memory chunk + item->chunk_data = (uint8_t*)rs_malloc(chunk_size) ; // deep copy memory chunk + + if(item->chunk_data == NULL) + return NULL ; + memcpy(item->chunk_data,chunk_data,chunk_size) ; return item ; } diff --git a/libretroshare/src/grouter/p3grouter.cc b/libretroshare/src/grouter/p3grouter.cc index 61721e312..e11358970 100644 --- a/libretroshare/src/grouter/p3grouter.cc +++ b/libretroshare/src/grouter/p3grouter.cc @@ -1121,13 +1121,11 @@ bool p3GRouter::locked_sendTransactionData(const RsPeerId& pid,const RsGRouterTr std::cerr << " sending to tunnel vpid " << pid << std::endl; #endif uint32_t turtle_data_size = trans_item.serial_size() ; - uint8_t *turtle_data = (uint8_t*)malloc(turtle_data_size) ; + uint8_t *turtle_data = (uint8_t*)rs_malloc(turtle_data_size) ; if(turtle_data == NULL) - { - std::cerr << " ERROR: Cannot allocate turtle data memory for size " << turtle_data_size << std::endl; return false ; - } + if(!trans_item.serialise(turtle_data,turtle_data_size)) { std::cerr << " ERROR: cannot serialise RsGRouterTransactionChunkItem." << std::endl; @@ -1304,14 +1302,13 @@ bool p3GRouter::sliceDataItem(RsGRouterAbstractMsgItem *item,std::listtotal_size = size; chunk_item->chunk_start= offset; chunk_item->chunk_size = chunk_size ; - chunk_item->chunk_data = (uint8_t*)malloc(chunk_size) ; + chunk_item->chunk_data = (uint8_t*)rs_malloc(chunk_size) ; #ifdef GROUTER_DEBUG std::cerr << " preparing to send a chunk [" << offset << " -> " << offset + chunk_size << " / " << size << "]" << std::endl; #endif if(chunk_item->chunk_data == NULL) { - std::cerr << " ERROR: Cannot allocate memory for size " << chunk_size << std::endl; delete chunk_item; throw ; } @@ -1921,7 +1918,11 @@ bool p3GRouter::sendData(const RsGxsId& destination,const GRouterServiceId& clie RsGRouterGenericDataItem *data_item = new RsGRouterGenericDataItem ; - data_item->data_bytes = (uint8_t*)malloc(data_size) ; + data_item->data_bytes = (uint8_t*)rs_malloc(data_size) ; + + if(data_item->data_bytes == NULL) + return false ; + memcpy(data_item->data_bytes,data,data_size) ; data_item->data_size = data_size ; diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index 37e33be88..8b27fe1bc 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -444,7 +444,11 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u int out_offset = 0; int max_evp_key_size = EVP_PKEY_size(public_key); - ek = (unsigned char*)malloc(max_evp_key_size); + ek = (unsigned char*)rs_malloc(max_evp_key_size); + + if(ek == NULL) + return false ; + const EVP_CIPHER *cipher = EVP_aes_128_cbc(); int cipher_block_size = EVP_CIPHER_block_size(cipher); int size_net_ekl = sizeof(net_ekl); @@ -455,13 +459,10 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u if(!EVP_SealInit(&ctx, EVP_aes_128_cbc(), &ek, &eklen, iv, &public_key, 1)) return false; // now assign memory to out accounting for data, and cipher block size, key length, and key length val - out = (uint8_t*)malloc(inlen + cipher_block_size + size_net_ekl + eklen + EVP_MAX_IV_LENGTH); + out = (uint8_t*)rs_malloc(inlen + cipher_block_size + size_net_ekl + eklen + EVP_MAX_IV_LENGTH); if(out == NULL) - { - std::cerr << "gxssecurity::encrypt(): cnnot allocate memory of size " << inlen + cipher_block_size + size_net_ekl + eklen + EVP_MAX_IV_LENGTH << " to encrypt data." << std::endl; return false ; - } net_ekl = htonl(eklen); memcpy((unsigned char*)out + out_offset, &net_ekl, size_net_ekl); @@ -540,7 +541,11 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in, EVP_CIPHER_CTX ctx; int eklen = 0, net_ekl = 0; - unsigned char *ek = (unsigned char*)malloc(EVP_PKEY_size(privateKey)); + unsigned char *ek = (unsigned char*)rs_malloc(EVP_PKEY_size(privateKey)); + + if(ek == NULL) + return false ; + unsigned char iv[EVP_MAX_IV_LENGTH]; EVP_CIPHER_CTX_init(&ctx); @@ -574,13 +579,10 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in, std::cerr << "Severe error in " << __PRETTY_FUNCTION__ << ": cannot encrypt. " << std::endl; return false ; } - out = (uint8_t*)malloc(inlen - in_offset); + out = (uint8_t*)rs_malloc(inlen - in_offset); if(out == NULL) - { - std::cerr << "gxssecurity::decrypt(): cannot allocate memory of size " << inlen - in_offset << " to decrypt data." << std::endl; return false; - } if(!EVP_OpenUpdate(&ctx, (unsigned char*) out, &out_currOffset, (unsigned char*)in + in_offset, inlen - in_offset)) { diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 36d50f83b..674e1abff 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -1,4137 +1,4163 @@ - -/* - * libretroshare/src/gxs: rsgxnetservice.cc - * - * Access to rs network and synchronisation service implementation - * - * Copyright 2012-2012 by Christopher Evi-Parker - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -// -// RsNxsItem -// | -// +-- RsNxsSyncGrp send req for group list, with time stamp of what we have -// +-- RsNxsSyncMsg -// +-- RsNxsGroupPublishKeyItem -// +-- RsNxsSyncGrpItem send individual grp info with time stamps, authors, etc. -// -// -// tick() -// | -// +----------- sharePublishKeys() -// | -// +----------- syncWithPeers() -// | | -// | +--if AutoSync--- send global UpdateTS of each peer to itself => the peer knows the last -// | | time current peer has received an updated from himself -// | | type=RsNxsSyncGrp -// | | role: advise to request grp list for mServType -// | | -// | +--Retrive all grp Id + meta -// | -// | For each peer -// | For each grp to request -// | send RsNxsSyncMsg(ServiceType, grpId, updateTS) -// | | -// | (Only send if rand() < sendingProb()) +---comes from mClientMsgUpdateMap -// | -// +----------- recvNxsItemQueue() -// | -// +------ handleRecvPublishKeys(auto*) -// | -// | -// | -// +------ handleRecvSyncGroup( RsNxsSyncGrp*) -// | - parse all subscribed groups. For each, send a RsNxsSyncGrpItem with publish TS -// | - pack into a single RsNxsTransac item -// | | -// | +---- canSendGrpId(peer, grpMeta, toVet) // determines if put in vetting list -// | | | // or sent right away -// | | +--CIRCLES_TYPE_LOCAL------- false -// | | +--CIRCLES_TYPE_PUBLIC------ true -// | | +--CIRCLES_TYPE_EXTERNAL---- mCircles->canSend(circleId, getPgpId(peerId)) -// | | +--CIRCLES_TYPE_YOUR_EYES--- internal circle stuff -// | | -// | +---- store in mPendingCircleVet ou directement locked_pushGrpRespFromList() -// | -// +------ handleRecvSyncMessage( RsNxsSyncMsg*) -// - parse msgs from group -// - send all msg IDs for this group -// data_tick() -// | -// +----------- updateServerSyncTS() -// | - retrieve all group meta data -// | - updates mServerMsgUpdateMap[grpId]=grp->mLastPostTS for all grps -// | - updates mGrpServerUpdateItem to max of all received TS -// | -// +----------- processTransactions() -// | -// +----------- processCompletedTransactions() -// | | -// | +------ locked_processCompletedIncomingTrans() -// | | | -// | | +-------- locked_genReqMsgTransaction() // request messages based on list -// | | | -// | | +-------- locked_genReqGrpTransaction() // request groups based on list -// | | | -// | | +-------- locked_genSendMsgsTransaction() // send msg list -// | | | -// | | +-------- locked_genSendGrpsTransaction() // send group list -// | | -// | +------ locked_processCompletedOutgoingTrans() -// | -// +----------- processExplicitGroupRequests() -// | - parse mExplicitRequest and for each element (containing a grpId list), -// | send the group ID (?!?!) -// | -// +----------- runVetting() -// | -// +--------- sort items from mPendingResp -// | | -// | +------ locked_createTransactionFromPending(GrpRespPending / MsgRespPending) -// | | // takes accepted transaction and adds them to the list of active trans -// | -// +--------- sort items from mPendingCircleVetting -// | -// +------ locked_createTransactionFromPending(GrpCircleIdsRequestVetting / MsgCircleIdsRequestVetting) -// // takes accepted transaction and adds them to the list of active trans -// -// Objects for time stamps -// ======================= -// -// mClientGrpUpdateMap: map< RsPeerId, TimeStamp > Time stamp of last modification of group data for that peer (in peer's clock time!) -// (Set at server side to be mGrpServerUpdateItem->grpUpdateTS) -// -// Only updated in processCompletedIncomingTransaction() from Grp list transaction. -// Used in syncWithPeers() sending in RsNxsSyncGrp once to all peers: peer will send data if -// has something new. All time comparisons are in the friends' clock time. -// -// mClientMsgUpdateMap: map< RsPeerId, map > -// -// Last msg list modification time sent by that peer Id -// Updated in processCompletedIncomingTransaction() from Grp list trans. -// Used in syncWithPeers() sending in RsNxsSyncGrp once to all peers. -// Set at server to be mServerMsgUpdateMap[grpId]->msgUpdateTS -// -// mGrpServerUpdateItem: TimeStamp Last group local modification timestamp over all groups -// -// mServerMsgUpdateMap: map< GrpId, TimeStamp > Timestamp local modification for each group (i.e. time of most recent msg / metadata update) -// -// -// Group update algorithm -// ====================== -// -// CLient Server -// ====== ====== -// -// tick() tick() -// | | -// +---- SyncWithPeers +-- recvNxsItemQueue() -// | | -// +---------------- Send global UpdateTS of each peer to itself => the peer knows +---------> +------ handleRecvSyncGroup( RsNxsSyncGrp*) -// | the last msg sent (stored in mClientGrpUpdateMap[peer_id]), | | - parse all subscribed groups. For each, send a RsNxsSyncGrpItem with publish TS -// | type=RsNxsSyncGrp | | - pack into a single RsNxsTransac item -// | role: advise to request grp list for mServType -------------------+ | -// | +--> +------ handleRecvSyncMessage( RsNxsSyncMsg*) -// +---------------- Retrieve all grp Id + meta | - parse msgs from group -// | | - send all msg IDs for this group -// +-- For each peer | -// For each grp to request | -// send RsNxsSyncMsg(ServiceType, grpId, updateTS) | -// | | -// (Only send if rand() < sendingProb()) +---comes from mClientMsgUpdateMap -----+ -// -// Suggestions -// =========== -// * handleRecvSyncGroup should use mit->second.mLastPost to limit the sending of already known data -// X * apparently mServerMsgUpdateMap is initially empty -> by default clients will always want to receive the data. -// => new peers will always send data for each group until they get an update for that group. -// X * check that there is a timestamp for unsubscribed items, otherwise we always send TS=0 and we always get them!! (in 346) -// -// -> there is not. mClientMsgUpdateMap is updated when msgs are received. -// -> 1842: leaves before asking for msg content. -// -// Proposed changes: -// - for unsubsribed groups, mClientMsgUpdateMap[peerid][grpId]=now when the group list is received => wont be asked again -// - when we subscribe, we reset the time stamp. -// -// Better change: -// - each peer sends last -// -// * the last TS method is not perfect: do new peers always receive old messages? -// -// * there's double information between mServerMsgUpdateMap first element (groupId) and second->grpId -// * processExplicitGroupRequests() seems to send the group list that it was asked for without further information. How is that useful??? -// -// * grps without messages will never be stamped because stamp happens in genReqMsgTransaction, after checking msgListL.empty() -// Problem: without msg, we cannot know the grpId!! -// -// * mClientMsgUpdateMap[peerid][grpId] is only updated when new msgs are received. Up to date groups will keep asking for lists! - - -#include -#include -#include -#include - -#include "rsgxsnetservice.h" -#include "retroshare/rsconfig.h" -#include "retroshare/rsreputations.h" -#include "retroshare/rsgxsflags.h" -#include "retroshare/rsgxscircles.h" -#include "pgp/pgpauxutils.h" - -/*** - * Use the following defines to debug: - NXS_NET_DEBUG_0 shows group update high level information - NXS_NET_DEBUG_1 shows group update low level info (including transaction details) - NXS_NET_DEBUG_2 bandwidth information - NXS_NET_DEBUG_3 publish key exchange - NXS_NET_DEBUG_4 vetting - NXS_NET_DEBUG_5 summary of transactions (useful to just know what comes in/out) - ***/ -//#define NXS_NET_DEBUG_0 1 -//#define NXS_NET_DEBUG_1 1 -//#define NXS_NET_DEBUG_2 1 -//#define NXS_NET_DEBUG_3 1 -//#define NXS_NET_DEBUG_4 1 -//#define NXS_NET_DEBUG_5 1 -//#define NXS_NET_DEBUG_6 1 - -#define GIXS_CUT_OFF 0 - -// The constant below have a direct influence on how fast forums/channels/posted/identity groups propagate and on the overloading of queues: -// -// Channels/forums will update at a rate of SYNC_PERIOD*MAX_REQLIST_SIZE/60 messages per minute. -// A large TRANSAC_TIMEOUT helps large transactions to finish before anything happens (e.g. disconnexion) or when the server has low upload bandwidth, -// but also uses more memory. -// A small value for MAX_REQLIST_SIZE is likely to help messages to propagate in a chaotic network, but will also slow them down. -// A small SYNC_PERIOD fasten message propagation, but is likely to overload the server side of transactions (e.g. overload outqueues). -// -#define SYNC_PERIOD 60 -#define MAX_REQLIST_SIZE 20 // No more than 20 items per msg request list => creates smaller transactions that are less likely to be cancelled. -#define TRANSAC_TIMEOUT 2000 // In seconds. Has been increased to avoid epidemic transaction cancelling due to overloaded outqueues. -#define SECURITY_DELAY_TO_FORCE_CLIENT_REUPDATE 3600 // force re-update if there happens to be a large delay between our server side TS and the client side TS of friends -#define REJECTED_MESSAGE_RETRY_DELAY 24*3600 // re-try rejected messages every 24hrs. Most of the time this is because the peer's reputation has changed. -#define GROUP_STATS_UPDATE_DELAY 1800 // update unsubscribed group statistics every 30 mins -#define GROUP_STATS_UPDATE_NB_PEERS 2 // update unsubscribed group statistics every 30 mins - -// Debug system to allow to print only for some IDs (group, Peer, etc) - -#if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2) || defined(NXS_NET_DEBUG_3) || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6) - -static const RsPeerId peer_to_print = RsPeerId(std::string("")) ; -static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("" )) ; // use this to allow to this group id only, or "" for all IDs -static const uint32_t service_to_print = 0 ; // use this to allow to this service id only, or 0 for all services - // warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums) - -class nullstream: public std::ostream {}; - -#if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2) || defined(NXS_NET_DEBUG_3) || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5)|| defined(NXS_NET_DEBUG_6) -static std::string nice_time_stamp(time_t now,time_t TS) -{ - if(TS == 0) - return "Never" ; - else - { - std::ostringstream s; - s << now - TS << " secs ago" ; - return s.str() ; - } -} -#endif - -static std::ostream& gxsnetdebug(const RsPeerId& peer_id,const RsGxsGroupId& grp_id,uint32_t service_type) -{ - static nullstream null ; - - if((peer_to_print.isNull() || peer_id.isNull() || peer_id == peer_to_print) - && (group_id_to_print.isNull() || grp_id.isNull() || grp_id == group_id_to_print) - && (service_to_print==0 || service_type == 0 || ((service_type >> 8)&0xffff) == service_to_print)) - return std::cerr << time(NULL) << ": " ; - else - return null ; -} - -#define GXSNETDEBUG___ gxsnetdebug(RsPeerId(),RsGxsGroupId(),mServiceInfo.mServiceType) -#define GXSNETDEBUG_P_(peer_id ) gxsnetdebug(peer_id ,RsGxsGroupId(),mServiceInfo.mServiceType) -#define GXSNETDEBUG__G( group_id) gxsnetdebug(RsPeerId(),group_id ,mServiceInfo.mServiceType) -#define GXSNETDEBUG_PG(peer_id,group_id) gxsnetdebug(peer_id ,group_id ,mServiceInfo.mServiceType) - -#endif - -const uint32_t RsGxsNetService::FRAGMENT_SIZE = 150000; - -RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, - RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, - const RsServiceInfo serviceInfo, - RsGixsReputation* reputations, RsGcxs* circles, - PgpAuxUtils *pgpUtils, bool grpAutoSync,bool msgAutoSync) - : p3ThreadedService(), p3Config(), mTransactionN(0), - mObserver(nxsObs), mDataStore(gds), mServType(servType), - mTransactionTimeOut(TRANSAC_TIMEOUT), mNetMgr(netMgr), mNxsMutex("RsGxsNetService"), - mSyncTs(0), mLastKeyPublishTs(0),mLastCleanRejectedMessages(0), mSYNC_PERIOD(SYNC_PERIOD), mCircles(circles), mReputations(reputations), - mPgpUtils(pgpUtils), - mGrpAutoSync(grpAutoSync),mAllowMsgSync(msgAutoSync), mGrpServerUpdateItem(NULL), - mServiceInfo(serviceInfo) - -{ - addSerialType(new RsNxsSerialiser(mServType)); - mOwnId = mNetMgr->getOwnId(); - mUpdateCounter = 0; -} - -RsGxsNetService::~RsGxsNetService() -{ - RS_STACK_MUTEX(mNxsMutex) ; - - for(TransactionsPeerMap::iterator it = mTransactions.begin();it!=mTransactions.end();++it) - { - for(TransactionIdMap::iterator it2 = it->second.begin();it2!=it->second.end();++it2) - delete it2->second ; - - it->second.clear() ; - } - mTransactions.clear() ; - - delete mGrpServerUpdateItem ; - - for(ClientGrpMap::iterator it = mClientGrpUpdateMap.begin();it!=mClientGrpUpdateMap.end();++it) - delete it->second ; - - mClientGrpUpdateMap.clear() ; - - for(std::map::iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();) - delete it->second ; - - mServerMsgUpdateMap.clear() ; -} - - -int RsGxsNetService::tick() -{ - // always check for new items arriving - // from peers - if(receivedItems()) - recvNxsItemQueue(); - - bool should_notify = false; - - { - RS_STACK_MUTEX(mNxsMutex) ; - - should_notify = should_notify || !mNewGroupsToNotify.empty() ; - should_notify = should_notify || !mNewMessagesToNotify.empty() ; - } - - if(should_notify) - processObserverNotifications() ; - - time_t now = time(NULL); - time_t elapsed = mSYNC_PERIOD + mSyncTs; - - if((elapsed) < now) - { - syncWithPeers(); - syncGrpStatistics(); - - mSyncTs = now; - } - - if(now > 10 + mLastKeyPublishTs) - { - sharePublishKeysPending() ; - - mLastKeyPublishTs = now ; - } - - if(now > 3600 + mLastCleanRejectedMessages) - { - mLastCleanRejectedMessages = now ; - cleanRejectedMessages() ; - } - return 1; -} - -void RsGxsNetService::processObserverNotifications() -{ -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG___ << "Processing observer notification." << std::endl; -#endif - - std::vector grps_copy ; - std::vector msgs_copy ; - - { - RS_STACK_MUTEX(mNxsMutex) ; - - grps_copy = mNewGroupsToNotify ; - msgs_copy = mNewMessagesToNotify ; - - mNewGroupsToNotify.clear() ; - mNewMessagesToNotify.clear() ; - } - - mObserver->notifyNewGroups(grps_copy); - mObserver->notifyNewMessages(msgs_copy); -} - -void RsGxsNetService::rejectMessage(const RsGxsMessageId& msg_id) -{ - RS_STACK_MUTEX(mNxsMutex) ; - - mRejectedMessages[msg_id] = time(NULL) ; -} -void RsGxsNetService::cleanRejectedMessages() -{ - RS_STACK_MUTEX(mNxsMutex) ; - time_t now = time(NULL) ; - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG___ << "Cleaning rejected messages." << std::endl; -#endif - - for(std::map::iterator it(mRejectedMessages.begin());it!=mRejectedMessages.end();) - if(it->second + REJECTED_MESSAGE_RETRY_DELAY < now) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG___ << " message id " << it->first << " should be re-tried. removing from list..." << std::endl; -#endif - - std::map::iterator tmp = it ; - ++tmp ; - mRejectedMessages.erase(it) ; - it=tmp ; - } - else - ++it ; -} - -// This class collects outgoing items due to the broadcast of Nxs messages. It computes -// a probability that can be used to temper the broadcast of items so as to match the -// residual bandwidth (difference between max allowed bandwidth and current outgoing rate. - -class NxsBandwidthRecorder -{ -public: - static const int OUTQUEUE_CUTOFF_VALUE = 500 ; - static const int BANDWIDTH_ESTIMATE_DELAY = 20 ; - - static void recordEvent(uint16_t service_type, RsItem *item) - { - RS_STACK_MUTEX(mtx) ; - - uint32_t bw = RsNxsSerialiser(service_type).size(item) ; // this is used to estimate bandwidth. - timeval tv ; - gettimeofday(&tv,NULL) ; - - // compute time(NULL) in msecs, for a more accurate bw estimate. - - uint64_t now = (uint64_t) tv.tv_sec * 1000 + tv.tv_usec/1000 ; - - total_record += bw ; - ++total_events ; - -#ifdef NXS_NET_DEBUG_2 - std::cerr << "bandwidthRecorder::recordEvent() Recording event time=" << now << ". bw=" << bw << std::endl; -#endif - - // Every 20 seconds at min, compute a new estimate of the required bandwidth. - - if(now > last_event_record + BANDWIDTH_ESTIMATE_DELAY*1000) - { - // Compute the bandwidth using recorded times, in msecs - float speed = total_record/1024.0f/(now - last_event_record)*1000.0f ; - - // Apply a small temporal convolution. - estimated_required_bandwidth = 0.75*estimated_required_bandwidth + 0.25 * speed ; - -#ifdef NXS_NET_DEBUG_2 - std::cerr << std::dec << " " << total_record << " Bytes (" << total_events << " items)" - << " received in " << now - last_event_record << " seconds. Speed: " << speed << " KBytes/sec" << std::endl; - std::cerr << " instantaneous speed = " << speed << " KB/s" << std::endl; - std::cerr << " cumulated estimated = " << estimated_required_bandwidth << " KB/s" << std::endl; -#endif - - last_event_record = now ; - total_record = 0 ; - total_events = 0 ; - } - } - - // Estimate the probability of sending an item so that the expected bandwidth matches the residual bandwidth - - static float computeCurrentSendingProbability() - { - int maxIn=50,maxOut=50; - float currIn=0,currOut=0 ; - - rsConfig->GetMaxDataRates(maxIn,maxOut) ; - rsConfig->GetCurrentDataRates(currIn,currOut) ; - - RsConfigDataRates rates ; - rsConfig->getTotalBandwidthRates(rates) ; - -#ifdef NXS_NET_DEBUG_2 - std::cerr << std::dec << std::endl; -#endif - - float outqueue_factor = 1.0f/pow( std::max(0.02f,rates.mQueueOut / (float)OUTQUEUE_CUTOFF_VALUE),5.0f) ; - float accepted_bandwidth = std::max( 0.0f, maxOut - currOut) ; - float max_bandwidth_factor = std::min( accepted_bandwidth / estimated_required_bandwidth,1.0f ) ; - - // We account for two things here: - // 1 - the required max bandwidth - // 2 - the current network overload, measured from the size of the outqueues. - // - // Only the later can limit the traffic if the internet connexion speed is responsible for outqueue overloading. - - float sending_probability = std::min(outqueue_factor,max_bandwidth_factor) ; - -#ifdef NXS_NET_DEBUG_2 - std::cerr << "bandwidthRecorder::computeCurrentSendingProbability()" << std::endl; - std::cerr << " current required bandwidth : " << estimated_required_bandwidth << " KB/s" << std::endl; - std::cerr << " max_bandwidth_factor : " << max_bandwidth_factor << std::endl; - std::cerr << " outqueue size : " << rates.mQueueOut << ", factor=" << outqueue_factor << std::endl; - std::cerr << " max out : " << maxOut << ", currOut=" << currOut << std::endl; - std::cerr << " computed probability : " << sending_probability << std::endl; -#endif - - return sending_probability ; - } - -private: - static RsMutex mtx; - static uint64_t last_event_record ; - static float estimated_required_bandwidth ; - static uint32_t total_events ; - static uint64_t total_record ; -}; - -uint32_t NxsBandwidthRecorder::total_events =0 ; // total number of events. Not used. -uint64_t NxsBandwidthRecorder::last_event_record = time(NULL) * 1000;// starting time of bw estimate period (in msec) -uint64_t NxsBandwidthRecorder::total_record =0 ; // total bytes recorded in the current time frame -float NxsBandwidthRecorder::estimated_required_bandwidth = 10.0f ;// Estimated BW for sending sync data. Set to 10KB/s, to avoid 0. -RsMutex NxsBandwidthRecorder::mtx("Bandwidth recorder") ; // Protects the recorder since bw events are collected from multiple GXS Net services - -// temporary holds a map of pointers to class T, and destroys all pointers on delete. - -template -class RsGxsMetaDataTemporaryMap: public std::map -{ -public: - virtual ~RsGxsMetaDataTemporaryMap() - { - clear() ; - } - - virtual void clear() - { - for(typename RsGxsMetaDataTemporaryMap::iterator it = this->begin();it!=this->end();++it) - if(it->second != NULL) - delete it->second ; - - std::map::clear() ; - } -}; - -void RsGxsNetService::syncWithPeers() -{ -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG___ << "RsGxsNetService::syncWithPeers() this=" << (void*)this << ". serviceInfo=" << mServiceInfo << std::endl; -#endif - - static RsNxsSerialiser ser(mServType) ; // this is used to estimate bandwidth. - - RS_STACK_MUTEX(mNxsMutex) ; - - std::set peers; - mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers); - if (peers.empty()) { - // nothing to do - return; - } - - std::set::iterator sit = peers.begin(); - - // for now just grps - for(; sit != peers.end(); ++sit) - { - - const RsPeerId peerId = *sit; - - ClientGrpMap::const_iterator cit = mClientGrpUpdateMap.find(peerId); - uint32_t updateTS = 0; - if(cit != mClientGrpUpdateMap.end()) - { - const RsGxsGrpUpdateItem *gui = cit->second; - updateTS = gui->grpUpdateTS; - } - RsNxsSyncGrp *grp = new RsNxsSyncGrp(mServType); - grp->clear(); - grp->PeerId(*sit); - grp->updateTS = updateTS; - - NxsBandwidthRecorder::recordEvent(mServType,grp) ; - -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_P_(*sit) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global group TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) to himself" << std::endl; -#endif - sendItem(grp); - } - - if(!mAllowMsgSync) - return ; - -#ifndef GXS_DISABLE_SYNC_MSGS - - typedef RsGxsMetaDataTemporaryMap GrpMetaMap; - GrpMetaMap grpMeta; - - mDataStore->retrieveGxsGrpMetaData(grpMeta); - - GrpMetaMap toRequest; - - for(GrpMetaMap::iterator mit = grpMeta.begin(); mit != grpMeta.end(); ++mit) - { - RsGxsGrpMetaData* meta = mit->second; - - // This was commented out because we want to know how many messages are available for unsubscribed groups. - - if(meta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) - { - toRequest.insert(std::make_pair(mit->first, meta)); - mit->second = NULL ; // avoids destruction ;-) - } - } - - sit = peers.begin(); - - float sending_probability = NxsBandwidthRecorder::computeCurrentSendingProbability() ; -#ifdef NXS_NET_DEBUG_2 - std::cerr << " syncWithPeers(): Sending probability = " << sending_probability << std::endl; -#endif - - // Synchronise group msg for groups which we're subscribed to - // For each peer and each group, we send to the peer the time stamp of the most - // recent modification the peer has sent. If the peer has more recent messages he will send them, because its latest - // modifications will be more recent. This ensures that we always compare timestamps all taken in the same - // computer (the peer's computer in this case) - - for(; sit != peers.end(); ++sit) - { - const RsPeerId& peerId = *sit; - - - // now see if you have an updateTS so optimise whether you need - // to get a new list of peer data - RsGxsMsgUpdateItem* mui = NULL; - - ClientMsgMap::const_iterator cit = mClientMsgUpdateMap.find(peerId); - - if(cit != mClientMsgUpdateMap.end()) - mui = cit->second; - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(peerId) << " syncing messages with peer " << peerId << std::endl; -#endif - - GrpMetaMap::const_iterator mmit = toRequest.begin(); - for(; mmit != toRequest.end(); ++mmit) - { - const RsGxsGrpMetaData* meta = mmit->second; - const RsGxsGroupId& grpId = mmit->first; - - if(!checkCanRecvMsgFromPeer(peerId, *meta)) - continue; - - // On default, the info has never been received so the TS is 0, meaning the peer has sent that it had no information. - - uint32_t updateTS = 0; - - if(mui) - { - std::map::const_iterator cit2 = mui->msgUpdateInfos.find(grpId); - - if(cit2 != mui->msgUpdateInfos.end()) - updateTS = cit2->second.time_stamp; - } - - RsNxsSyncMsg* msg = new RsNxsSyncMsg(mServType); - msg->clear(); - msg->PeerId(peerId); - msg->grpId = grpId; - msg->updateTS = updateTS; - - NxsBandwidthRecorder::recordEvent(mServType,msg) ; - - if(RSRandom::random_f32() < sending_probability) - { - sendItem(msg); -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_PG(*sit,grpId) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself" << std::endl; -#endif - } - else - { - delete msg ; -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(*sit,grpId) << " cancel RsNxsSyncMsg req (last local update TS for group+peer) for grpId=" << grpId << " to peer " << *sit << ": not enough bandwidth." << std::endl; -#endif - } - } - } - -#endif -} - -void RsGxsNetService::syncGrpStatistics() -{ - RS_STACK_MUTEX(mNxsMutex) ; - -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG___<< "Sync-ing group statistics." << std::endl; -#endif - RsGxsMetaDataTemporaryMap grpMeta; - - mDataStore->retrieveGxsGrpMetaData(grpMeta); - - std::set online_peers; - mNetMgr->getOnlineList(mServiceInfo.mServiceType, online_peers); - - // Go through group statistics and groups without information are re-requested to random peers selected - // among the ones who provided the group info. - - time_t now = time(NULL) ; - - for(std::map::const_iterator it(grpMeta.begin());it!=grpMeta.end();++it) - { - RsGroupNetworkStatsRecord& rec(mGroupNetworkStats[it->first]) ; -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG__G(it->first) << " group " << it->first ; -#endif - - if(rec.update_TS + GROUP_STATS_UPDATE_DELAY < now && rec.suppliers.size() > 0) - { -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG__G(it->first) << " needs update. Randomly asking to some friends" << std::endl; -#endif - // randomly select GROUP_STATS_UPDATE_NB_PEERS friends among the suppliers of this group - - uint32_t n = RSRandom::random_u32() % rec.suppliers.size() ; - - std::set::const_iterator rit = rec.suppliers.begin(); - for(uint32_t i=0;ifirst) << " asking friend " << peer_id << " for an update of stats for group " << it->first << std::endl; -#endif - - RsNxsSyncGrpStats *grs = new RsNxsSyncGrpStats(mServType) ; - - grs->request_type = RsNxsSyncGrpStats::GROUP_INFO_TYPE_REQUEST ; - grs->grpId = it->first ; - grs->PeerId(peer_id) ; - - sendItem(grs) ; - } - } - } -#ifdef NXS_NET_DEBUG_6 - else - GXSNETDEBUG__G(it->first) << " up to date." << std::endl; -#endif - } -} - -void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStats *grs) -{ - if(grs->request_type == RsNxsSyncGrpStats::GROUP_INFO_TYPE_REQUEST) - { -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << "Received Grp update stats Request for group " << grs->grpId << " from friend " << grs->PeerId() << std::endl; -#endif - RsGxsMetaDataTemporaryMap grpMetas; - grpMetas[grs->grpId] = NULL; - - mDataStore->retrieveGxsGrpMetaData(grpMetas); - - RsGxsGrpMetaData* grpMeta = grpMetas[grs->grpId]; - - if(grpMeta == NULL) - { -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " Group is unknown. Not reponding." << std::endl; -#endif - return ; - } - - // check if we're subscribed or not - - if(! (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )) - { -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " Group is not subscribed. Not reponding." << std::endl; -#endif - return ; - } - - // now count available messages - - GxsMsgReq reqIds; - reqIds[grs->grpId] = std::vector(); - GxsMsgMetaResult result; - -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " retrieving message information." << std::endl; -#endif - mDataStore->retrieveGxsMsgMetaData(reqIds, result); - - const std::vector& vec(result[grs->grpId]) ; - - if(vec.empty()) // that means we don't have any, or there isn't any, but since the default is always 0, no need to send. - return ; - - RsNxsSyncGrpStats *grs_resp = new RsNxsSyncGrpStats(mServType) ; - grs_resp->request_type = RsNxsSyncGrpStats::GROUP_INFO_TYPE_RESPONSE ; - grs_resp->number_of_posts = vec.size(); - grs_resp->grpId = grs->grpId; - grs_resp->PeerId(grs->PeerId()) ; - - grs_resp->last_post_TS = 0 ; - - for(uint32_t i=0;ilast_post_TS < vec[i]->mPublishTs) - grs_resp->last_post_TS = vec[i]->mPublishTs; - - delete vec[i] ; - } -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " sending back statistics item with " << vec.size() << " elements." << std::endl; -#endif - - sendItem(grs_resp) ; - } - else if(grs->request_type == RsNxsSyncGrpStats::GROUP_INFO_TYPE_RESPONSE) - { -#ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << "Received Grp update stats item from peer " << grs->PeerId() << " for group " << grs->grpId << ", reporting " << grs->number_of_posts << " posts." << std::endl; -#endif - RS_STACK_MUTEX(mNxsMutex) ; - RsGroupNetworkStatsRecord& rec(mGroupNetworkStats[grs->grpId]) ; - - int32_t old_count = rec.max_visible_count ; - int32_t old_suppliers_count = rec.suppliers.size() ; - - rec.suppliers.insert(grs->PeerId()) ; - rec.max_visible_count = std::max(rec.max_visible_count,grs->number_of_posts) ; - rec.update_TS = time(NULL) ; - - if (old_count != rec.max_visible_count || old_suppliers_count != rec.suppliers.size()) - mObserver->notifyChangedGroupStats(grs->grpId); - } - else - std::cerr << "(EE) RsGxsNetService::handleRecvSyncGrpStatistics(): unknown item type " << grs->request_type << " found. This is a bug." << std::endl; -} - -void RsGxsNetService::subscribeStatusChanged(const RsGxsGroupId& grpId,bool subscribed) -{ - RS_STACK_MUTEX(mNxsMutex) ; - - if(!subscribed) - return ; - - // When we subscribe, we reset the time stamps, so that the entire group list - // gets requested once again, for a proper update. - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG__G(grpId) << "Changing subscribe status for grp " << grpId << " to " << subscribed << ": reseting all server msg time stamps for this group, and server global TS." << std::endl; -#endif - std::map::iterator it = mServerMsgUpdateMap.find(grpId) ; - - if(mServerMsgUpdateMap.end() == it) - { - RsGxsServerMsgUpdateItem *item = new RsGxsServerMsgUpdateItem(mServType) ; - item->grpId = grpId ; - item->msgUpdateTS = 0 ; - } - else - it->second->msgUpdateTS = 0 ; // reset! - - // no need to update mGrpServerUpdateItem since the ::updateServerSyncTS() call will do it. -} - -bool RsGxsNetService::fragmentMsg(RsNxsMsg& msg, MsgFragments& msgFragments) const -{ - // first determine how many fragments - uint32_t msgSize = msg.msg.TlvSize(); - uint32_t dataLeft = msgSize; - uint8_t nFragments = ceil(float(msgSize)/FRAGMENT_SIZE); - char buffer[FRAGMENT_SIZE]; - int currPos = 0; - - - for(uint8_t i=0; i < nFragments; ++i) - { - RsNxsMsg* msgFrag = new RsNxsMsg(mServType); - msgFrag->grpId = msg.grpId; - msgFrag->msgId = msg.msgId; - msgFrag->meta = msg.meta; - msgFrag->transactionNumber = msg.transactionNumber; - msgFrag->pos = i; - msgFrag->PeerId(msg.PeerId()); - msgFrag->count = nFragments; - uint32_t fragSize = std::min(dataLeft, FRAGMENT_SIZE); - - memcpy(buffer, ((char*)msg.msg.bin_data) + currPos, fragSize); - msgFrag->msg.setBinData(buffer, fragSize); - - currPos += fragSize; - dataLeft -= fragSize; - msgFragments.push_back(msgFrag); - } - - return true; -} - -bool RsGxsNetService::fragmentGrp(RsNxsGrp& grp, GrpFragments& grpFragments) const -{ - // first determine how many fragments - uint32_t grpSize = grp.grp.TlvSize(); - uint32_t dataLeft = grpSize; - uint8_t nFragments = ceil(float(grpSize)/FRAGMENT_SIZE); - char buffer[FRAGMENT_SIZE]; - int currPos = 0; - - - for(uint8_t i=0; i < nFragments; ++i) - { - RsNxsGrp* grpFrag = new RsNxsGrp(mServType); - grpFrag->grpId = grp.grpId; - grpFrag->meta = grp.meta; - grpFrag->pos = i; - grpFrag->count = nFragments; - uint32_t fragSize = std::min(dataLeft, FRAGMENT_SIZE); - - memcpy(buffer, ((char*)grp.grp.bin_data) + currPos, fragSize); - grpFrag->grp.setBinData(buffer, fragSize); - - currPos += fragSize; - dataLeft -= fragSize; - grpFragments.push_back(grpFrag); - } - - return true; -} - -RsNxsMsg* RsGxsNetService::deFragmentMsg(MsgFragments& msgFragments) const -{ - if(msgFragments.empty()) return NULL; - - // if there is only one fragment with a count 1 or less then - // the fragment is the msg - if(msgFragments.size() == 1) - { - RsNxsMsg* m = msgFragments.front(); - if(m->count > 1) - return NULL; - else - return m; - } - - // first determine total size for binary data - MsgFragments::iterator mit = msgFragments.begin(); - uint32_t datSize = 0; - - for(; mit != msgFragments.end(); ++mit) - datSize += (*mit)->msg.bin_len; - - char* data = new char[datSize]; - uint32_t currPos = 0; - - for(mit = msgFragments.begin(); mit != msgFragments.end(); ++mit) - { - RsNxsMsg* msg = *mit; - memcpy(data + (currPos), msg->msg.bin_data, msg->msg.bin_len); - currPos += msg->msg.bin_len; - } - - RsNxsMsg* msg = new RsNxsMsg(mServType); - const RsNxsMsg& m = *(*(msgFragments.begin())); - msg->msg.setBinData(data, datSize); - msg->msgId = m.msgId; - msg->grpId = m.grpId; - msg->transactionNumber = m.transactionNumber; - msg->meta = m.meta; - - delete[] data; - return msg; -} - -RsNxsGrp* RsGxsNetService::deFragmentGrp(GrpFragments& grpFragments) const -{ - if(grpFragments.empty()) return NULL; - - // first determine total size for binary data - GrpFragments::iterator mit = grpFragments.begin(); - uint32_t datSize = 0; - - for(; mit != grpFragments.end(); ++mit) - datSize += (*mit)->grp.bin_len; - - char* data = new char[datSize]; - uint32_t currPos = 0; - - for(mit = grpFragments.begin(); mit != grpFragments.end(); ++mit) - { - RsNxsGrp* grp = *mit; - memcpy(data + (currPos), grp->grp.bin_data, grp->grp.bin_len); - currPos += grp->grp.bin_len; - } - - RsNxsGrp* grp = new RsNxsGrp(mServType); - const RsNxsGrp& g = *(*(grpFragments.begin())); - grp->grp.setBinData(data, datSize); - grp->grpId = g.grpId; - grp->transactionNumber = g.transactionNumber; - grp->meta = g.meta; - - delete[] data; - - return grp; -} - -struct GrpFragCollate -{ - RsGxsGroupId mGrpId; - GrpFragCollate(const RsGxsGroupId& grpId) : mGrpId(grpId){ } - bool operator()(RsNxsGrp* grp) { return grp->grpId == mGrpId;} -}; - -void RsGxsNetService::locked_createTransactionFromPending( MsgRespPending* msgPend) -{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(msgPend->mPeerId) << "locked_createTransactionFromPending()" << std::endl; -#endif - MsgAuthorV::const_iterator cit = msgPend->mMsgAuthV.begin(); - std::list reqList; - uint32_t transN = locked_getTransactionId(); - for(; cit != msgPend->mMsgAuthV.end(); ++cit) - { - const MsgAuthEntry& entry = *cit; - - if(entry.mPassedVetting) - { - RsNxsSyncMsgItem* msgItem = new RsNxsSyncMsgItem(mServType); - msgItem->grpId = entry.mGrpId; - msgItem->msgId = entry.mMsgId; - msgItem->authorId = entry.mAuthorId; - msgItem->flag = RsNxsSyncMsgItem::FLAG_REQUEST; - msgItem->transactionNumber = transN; - msgItem->PeerId(msgPend->mPeerId); - reqList.push_back(msgItem); - } -#ifdef NXS_NET_DEBUG_1 - else - GXSNETDEBUG_PG(msgPend->mPeerId,entry.mGrpId) << " entry failed vetting: grpId=" << entry.mGrpId << ", msgId=" << entry.mMsgId << ", peerId=" << msgPend->mPeerId << std::endl; -#endif - } - - if(!reqList.empty()) - locked_pushMsgTransactionFromList(reqList, msgPend->mPeerId, transN); -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(msgPend->mPeerId) << " added " << reqList.size() << " items to transaction." << std::endl; -#endif -} - -void RsGxsNetService::locked_createTransactionFromPending(GrpRespPending* grpPend) -{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(grpPend->mPeerId) << "locked_createTransactionFromPending() from peer " << grpPend->mPeerId << std::endl; -#endif - GrpAuthorV::const_iterator cit = grpPend->mGrpAuthV.begin(); - std::list reqList; - uint32_t transN = locked_getTransactionId(); - for(; cit != grpPend->mGrpAuthV.end(); ++cit) - { - const GrpAuthEntry& entry = *cit; - - if(entry.mPassedVetting) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(grpPend->mPeerId,entry.mGrpId) << " entry Group Id: " << entry.mGrpId << " PASSED" << std::endl; -#endif - RsNxsSyncGrpItem* msgItem = new RsNxsSyncGrpItem(mServType); - msgItem->grpId = entry.mGrpId; - msgItem->authorId = entry.mAuthorId; - msgItem->flag = RsNxsSyncMsgItem::FLAG_REQUEST; - msgItem->transactionNumber = transN; - msgItem->PeerId(grpPend->mPeerId); - reqList.push_back(msgItem); - } -#ifdef NXS_NET_DEBUG_1 - else - GXSNETDEBUG_PG(grpPend->mPeerId,entry.mGrpId) << " entry failed vetting: grpId=" << entry.mGrpId << ", peerId=" << grpPend->mPeerId << std::endl; -#endif - } - - if(!reqList.empty()) - locked_pushGrpTransactionFromList(reqList, grpPend->mPeerId, transN); -} - - -void RsGxsNetService::locked_createTransactionFromPending(GrpCircleIdRequestVetting* grpPend) -{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(grpPend->mPeerId) << "locked_createTransactionFromPending(GrpCircleIdReq)" << std::endl; -#endif - std::vector::iterator cit = grpPend->mGrpCircleV.begin(); - uint32_t transN = locked_getTransactionId(); - std::list itemL; - for(; cit != grpPend->mGrpCircleV.end(); ++cit) - { - const GrpIdCircleVet& entry = *cit; - if(entry.mCleared) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(grpPend->mPeerId,entry.mGroupId) << " Group Id: " << entry.mGroupId << " PASSED" << std::endl; -#endif - RsNxsSyncGrpItem* gItem = new RsNxsSyncGrpItem(mServType); - gItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE; - gItem->grpId = entry.mGroupId; - gItem->publishTs = 0; - gItem->PeerId(grpPend->mPeerId); - gItem->transactionNumber = transN; - gItem->authorId = entry.mAuthorId; - // why it authorId not set here??? - itemL.push_back(gItem); - } -#ifdef NXS_NET_DEBUG_1 - else - GXSNETDEBUG_PG(grpPend->mPeerId,entry.mGroupId) << " Group Id: " << entry.mGroupId << " FAILED" << std::endl; -#endif - } - - if(!itemL.empty()) - locked_pushGrpRespFromList(itemL, grpPend->mPeerId, transN); -} - -void RsGxsNetService::locked_createTransactionFromPending(MsgCircleIdsRequestVetting* msgPend) -{ - std::vector::iterator vit = msgPend->mMsgs.begin(); - std::list itemL; - - uint32_t transN = locked_getTransactionId(); - RsGxsGroupId grp_id ; - - for(; vit != msgPend->mMsgs.end(); ++vit) - { - MsgIdCircleVet& mic = *vit; - RsNxsSyncMsgItem* mItem = new - RsNxsSyncMsgItem(mServType); - mItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE; - mItem->grpId = msgPend->mGrpId; - mItem->msgId = mic.mMsgId; - mItem->authorId = mic.mAuthorId; - mItem->PeerId(msgPend->mPeerId); - mItem->transactionNumber = transN; - itemL.push_back(mItem); - - grp_id = msgPend->mGrpId ; - } - - if(!itemL.empty()) - locked_pushMsgRespFromList(itemL, msgPend->mPeerId,grp_id, transN); -} - -/*bool RsGxsNetService::locked_canReceive(const RsGxsGrpMetaData * const grpMeta - , const RsPeerId& peerId ) -{ - - double timeDelta = 0.2; - - if(grpMeta->mCircleType == GXS_CIRCLE_TYPE_EXTERNAL) { - int i=0; - mCircles->loadCircle(grpMeta->mCircleId); - - // check 5 times at most - // spin for 1 second at most - while(i < 5) { - - if(mCircles->isLoaded(grpMeta->mCircleId)) { - const RsPgpId& pgpId = mPgpUtils->getPGPId(peerId); - return mCircles->canSend(grpMeta->mCircleId, pgpId); - }//if(mCircles->isLoaded(grpMeta->mCircleId)) - - usleep((int) (timeDelta * 1000 * 1000));// timeDelta sec - i++; - }//while(i < 5) - - } else {//if(grpMeta->mCircleType == GXS_CIRCLE_TYPE_EXTERNAL) - return true; - }//else (grpMeta->mCircleType == GXS_CIRCLE_TYPE_EXTERNAL) - - return false; -}*/ - -void RsGxsNetService::collateGrpFragments(GrpFragments fragments, - std::map& partFragments) const -{ - // get all unique grpIds; - GrpFragments::iterator vit = fragments.begin(); - std::set grpIds; - - for(; vit != fragments.end(); ++vit) - grpIds.insert( (*vit)->grpId ); - - std::set::iterator sit = grpIds.begin(); - - for(; sit != grpIds.end(); ++sit) - { - const RsGxsGroupId& grpId = *sit; - GrpFragments::iterator bound = std::partition( - fragments.begin(), fragments.end(), - GrpFragCollate(grpId)); - - // something will always be found for a group id - for(vit = fragments.begin(); vit != bound; ) - { - partFragments[grpId].push_back(*vit); - vit = fragments.erase(vit); - } - - GrpFragments& f = partFragments[grpId]; - RsNxsGrp* grp = *(f.begin()); - - // if counts of fragments is incorrect remove - // from coalescion - if(grp->count != f.size()) - { - GrpFragments::iterator vit2 = f.begin(); - - for(; vit2 != f.end(); ++vit2) - delete *vit2; - - partFragments.erase(grpId); - } - } - - fragments.clear(); -} - -struct MsgFragCollate -{ - RsGxsMessageId mMsgId; - MsgFragCollate(const RsGxsMessageId& msgId) : mMsgId(msgId){ } - bool operator()(RsNxsMsg* msg) { return msg->msgId == mMsgId;} -}; - -void RsGxsNetService::collateMsgFragments(MsgFragments fragments, std::map& partFragments) const -{ - // get all unique message Ids; - MsgFragments::iterator vit = fragments.begin(); - std::set msgIds; - - for(; vit != fragments.end(); ++vit) - msgIds.insert( (*vit)->msgId ); - - - std::set::iterator sit = msgIds.begin(); - - for(; sit != msgIds.end(); ++sit) - { - const RsGxsMessageId& msgId = *sit; - MsgFragments::iterator bound = std::partition( - fragments.begin(), fragments.end(), - MsgFragCollate(msgId)); - - // something will always be found for a group id - for(vit = fragments.begin(); vit != bound; ++vit ) - { - partFragments[msgId].push_back(*vit); - } - - fragments.erase(fragments.begin(), bound); - MsgFragments& f = partFragments[msgId]; - RsNxsMsg* msg = *(f.begin()); - - // if counts of fragments is incorrect remove - // from coalescion - if(msg->count != f.size()) - { - MsgFragments::iterator vit2 = f.begin(); - - for(; vit2 != f.end(); ++vit2) - delete *vit2; - - partFragments.erase(msgId); - } - } - - fragments.clear(); -} - -class StoreHere -{ -public: - - StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap& cmm, RsGxsNetService::ServerMsgMap& smm, RsGxsServerGrpUpdateItem*& sgm) - : mClientGrpMap(cgm), mClientMsgMap(cmm), mServerMsgMap(smm), mServerGrpUpdateItem(sgm) - {} - - void operator() (RsItem* item) - { - RsGxsMsgUpdateItem* mui; - RsGxsGrpUpdateItem* gui; - RsGxsServerGrpUpdateItem* gsui; - RsGxsServerMsgUpdateItem* msui; - - if((mui = dynamic_cast(item)) != NULL) - mClientMsgMap.insert(std::make_pair(mui->peerId, mui)); - else if((gui = dynamic_cast(item)) != NULL) - mClientGrpMap.insert(std::make_pair(gui->peerId, gui)); - else if((msui = dynamic_cast(item)) != NULL) - mServerMsgMap.insert(std::make_pair(msui->grpId, msui)); - else if((gsui = dynamic_cast(item)) != NULL) - { - if(mServerGrpUpdateItem == NULL) - mServerGrpUpdateItem = gsui; - else - { - std::cerr << "Error! More than one server group update item exists!" << std::endl; - delete gsui; - } - } - else - { - std::cerr << "Type not expected!" << std::endl; - delete item ; - } - } - -private: - - RsGxsNetService::ClientGrpMap& mClientGrpMap; - RsGxsNetService::ClientMsgMap& mClientMsgMap; - RsGxsNetService::ServerMsgMap& mServerMsgMap; - RsGxsServerGrpUpdateItem*& mServerGrpUpdateItem; - -}; - -bool RsGxsNetService::loadList(std::list &load) -{ - RS_STACK_MUTEX(mNxsMutex) ; - - // The delete is done in StoreHere, if necessary - - std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mGrpServerUpdateItem)); - time_t now = time(NULL); - - for(ClientMsgMap::iterator it = mClientMsgUpdateMap.begin();it!=mClientMsgUpdateMap.end();++it) - for(std::map::const_iterator it2(it->second->msgUpdateInfos.begin());it2!=it->second->msgUpdateInfos.end();++it2) - { - RsGroupNetworkStatsRecord& gnsr = mGroupNetworkStats[it2->first] ; - - // At each reload, divide the last count by 2. This gradually flushes old information away. - - gnsr.max_visible_count = std::max(it2->second.message_count,gnsr.max_visible_count/2) ; - gnsr.update_TS = now - GROUP_STATS_UPDATE_DELAY + (RSRandom::random_u32()%(GROUP_STATS_UPDATE_DELAY/10)) ; - - // Similarly, we remove some of the suppliers randomly. If they are - // actual suppliers, they will come back automatically. If they are - // not, they will be forgotten. - - if(RSRandom::random_f32() > 0.2) - gnsr.suppliers.insert(it->first) ; - } - return true; -} - -#include - -template -struct get_second : public std::unary_function -{ - RsItem* operator()(const typename UpdateMap::value_type& value) const - { - return value.second; - } -}; - -bool RsGxsNetService::saveList(bool& cleanup, std::list& save) -{ - RS_STACK_MUTEX(mNxsMutex) ; - - // hardcore templates - std::transform(mClientGrpUpdateMap.begin(), mClientGrpUpdateMap.end(), std::back_inserter(save), get_second()); - std::transform(mClientMsgUpdateMap.begin(), mClientMsgUpdateMap.end(), std::back_inserter(save), get_second()); - std::transform(mServerMsgUpdateMap.begin(), mServerMsgUpdateMap.end(), std::back_inserter(save), get_second()); - - save.push_back(mGrpServerUpdateItem); - - cleanup = false; - return true; -} - -RsSerialiser *RsGxsNetService::setupSerialiser() -{ - - RsSerialiser *rss = new RsSerialiser; - rss->addSerialType(new RsGxsUpdateSerialiser(mServType)); - - return rss; -} - -void RsGxsNetService::recvNxsItemQueue() -{ - RsItem *item ; - - while(NULL != (item=recvItem())) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetService Item:" << (void*)item << std::endl ; -#endif - // RsNxsItem needs dynamic_cast, since they have derived siblings. - // - RsNxsItem *ni = dynamic_cast(item) ; - if(ni != NULL) - { - // a live transaction has a non zero value - if(ni->transactionNumber != 0) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(item->PeerId()) << " recvNxsItemQueue() handlingTransaction, transN " << ni->transactionNumber << std::endl; -#endif - - if(!handleTransaction(ni)) - delete ni; - - continue; - } - - - switch(ni->PacketSubType()) - { - case RS_PKT_SUBTYPE_NXS_SYNC_GRP_STATS: handleRecvSyncGrpStatistics (dynamic_cast(ni)) ; break ; - case RS_PKT_SUBTYPE_NXS_SYNC_GRP: handleRecvSyncGroup (dynamic_cast(ni)) ; break ; - case RS_PKT_SUBTYPE_NXS_SYNC_MSG: handleRecvSyncMessage (dynamic_cast(ni)) ; break ; - case RS_PKT_SUBTYPE_NXS_GRP_PUBLISH_KEY: handleRecvPublishKeys (dynamic_cast(ni)) ; break ; - default: - std::cerr << "Unhandled item subtype " << (uint32_t) ni->PacketSubType() << " in RsGxsNetService: " << std::endl; break; - } - delete item ; - } - else - { - std::cerr << "Not a RsNxsItem, deleting!" << std::endl; - delete(item); - } - } -} - - -bool RsGxsNetService::handleTransaction(RsNxsItem* item) -{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(item->PeerId()) << "handleTransaction(RsNxsItem) number=" << item->transactionNumber << std::endl; -#endif - - /*! - * This attempts to handle a transaction - * It first checks if this transaction id already exists - * If it does then check this not a initiating transactions - */ - - RS_STACK_MUTEX(mNxsMutex) ; - - const RsPeerId& peer = item->PeerId(); - - RsNxsTransac* transItem = dynamic_cast(item); - - // if this is a RsNxsTransac item process - if(transItem) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(item->PeerId()) << " this is a RsNxsTransac item. callign process." << std::endl; -#endif - return locked_processTransac(transItem); - } - - - // then this must be transaction content to be consumed - // first check peer exist for transaction - bool peerTransExists = mTransactions.find(peer) != mTransactions.end(); - - // then check transaction exists - - NxsTransaction* tr = NULL; - uint32_t transN = item->transactionNumber; - - if(peerTransExists) - { - TransactionIdMap& transMap = mTransactions[peer]; - - if(transMap.find(transN) != transMap.end()) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(item->PeerId()) << " Consuming Transaction content, transN: " << item->transactionNumber << std::endl; - GXSNETDEBUG_P_(item->PeerId()) << " Consuming Transaction content, from Peer: " << item->PeerId() << std::endl; -#endif - - tr = transMap[transN]; - tr->mItems.push_back(item); - - return true; - } - } - - return false; -} - -bool RsGxsNetService::locked_processTransac(RsNxsTransac* item) -{ - - /*! - * To process the transaction item - * It can either be initiating a transaction - * or ending one that already exists - * - * For initiating an incoming transaction the peer - * and transaction item need not exists - * as the peer will be added and transaction number - * added thereafter - * - * For commencing/starting an outgoing transaction - * the transaction must exist already - * - * For ending a transaction the - */ - - RsPeerId peer; - - // for outgoing transaction use own id - if(item->transactFlag & (RsNxsTransac::FLAG_BEGIN_P2 | RsNxsTransac::FLAG_END_SUCCESS)) - peer = mOwnId; - else - peer = item->PeerId(); - - uint32_t transN = item->transactionNumber; - item->timestamp = time(NULL); // register time received - NxsTransaction* tr = NULL; - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << "locked_processTransac() " << std::endl; - GXSNETDEBUG_P_(peer) << " Received transaction item: " << transN << std::endl; - GXSNETDEBUG_P_(peer) << " With peer: " << item->PeerId() << std::endl; - GXSNETDEBUG_P_(peer) << " trans type: " << item->transactFlag << std::endl; -#endif - - bool peerTrExists = mTransactions.find(peer) != mTransactions.end(); - bool transExists = false; - - if(peerTrExists) - { - TransactionIdMap& transMap = mTransactions[peer]; - // record whether transaction exists already - transExists = transMap.find(transN) != transMap.end(); - } - - // initiating an incoming transaction - if(item->transactFlag & RsNxsTransac::FLAG_BEGIN_P1) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << " initiating Incoming transaction." << std::endl; -#endif - - if(transExists) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << " transaction already exist! ERROR" << std::endl; -#endif - return false; // should not happen! - } - - // create a transaction if the peer does not exist - if(!peerTrExists) - mTransactions[peer] = TransactionIdMap(); - - TransactionIdMap& transMap = mTransactions[peer]; - - - // create new transaction - tr = new NxsTransaction(); - transMap[transN] = tr; - tr->mTransaction = item; - tr->mTimeOut = item->timestamp + mTransactionTimeOut; -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << " Setting timeout of " << mTransactionTimeOut << " secs, which is " << tr->mTimeOut - time(NULL) << " secs from now." << std::endl; -#endif - - // note state as receiving, commencement item - // is sent on next run() loop - tr->mFlag = NxsTransaction::FLAG_STATE_STARTING; - return true; - // commencement item for outgoing transaction - } - else if(item->transactFlag & RsNxsTransac::FLAG_BEGIN_P2) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << " initiating outgoign transaction." << std::endl; -#endif - // transaction must exist - if(!peerTrExists || !transExists) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << " transaction does not exist. Cancelling!" << std::endl; -#endif - - return false; - } - - - // alter state so transaction content is sent on - // next run() loop - TransactionIdMap& transMap = mTransactions[mOwnId]; - NxsTransaction* tr = transMap[transN]; - tr->mFlag = NxsTransaction::FLAG_STATE_SENDING; - delete item; - return true; - // end transac item for outgoing transaction - } - else if(item->transactFlag & RsNxsTransac::FLAG_END_SUCCESS) - { - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << " marking this transaction succeed" << std::endl; -#endif - // transaction does not exist - if(!peerTrExists || !transExists) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << " transaction does not exist. Cancelling!" << std::endl; -#endif - return false; - } - - // alter state so that transaction is removed - // on next run() loop - TransactionIdMap& transMap = mTransactions[mOwnId]; - NxsTransaction* tr = transMap[transN]; - tr->mFlag = NxsTransaction::FLAG_STATE_COMPLETED; - delete item; - return true; - } - else - return false; -} - -void RsGxsNetService::data_tick() -{ - static const double timeDelta = 0.5; - - //Start waiting as nothing to do in runup - usleep((int) (timeDelta * 1000 * 1000)); // timeDelta sec - - if(mUpdateCounter >= 120) // 60 seconds - { - updateServerSyncTS(); - updateClientSyncTS(); - mUpdateCounter = 1; - } - else - mUpdateCounter++; - - if(mUpdateCounter % 20 == 0) // dump the full shit every 20 secs - debugDump() ; - - // process active transactions - processTransactions(); - - // process completed transactions - processCompletedTransactions(); - - // vetting of id and circle info - runVetting(); - - processExplicitGroupRequests(); -} - -void RsGxsNetService::debugDump() -{ -#ifdef NXS_NET_DEBUG_0 - RS_STACK_MUTEX(mNxsMutex) ; - time_t now = time(NULL) ; - - GXSNETDEBUG___<< "RsGxsNetService::debugDump():" << std::endl; - - if(mGrpServerUpdateItem != NULL) - GXSNETDEBUG___<< " mGrpServerUpdateItem time stamp: " << nice_time_stamp(time(NULL) , mGrpServerUpdateItem->grpUpdateTS) << " (is the last local modification time over all groups of this service)" << std::endl; - else - GXSNETDEBUG___<< " mGrpServerUpdateItem time stamp: not inited yet (is the last local modification time over all groups of this service)" << std::endl; - - GXSNETDEBUG___<< " mServerMsgUpdateMap: (is for each subscribed group, the last local modification time)" << std::endl; - - for(std::map::const_iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it) - GXSNETDEBUG__G(it->first) << " Grp:" << it->first << " last local modification (secs ago): " << nice_time_stamp(time(NULL),it->second->msgUpdateTS) << std::endl; - - GXSNETDEBUG___<< " mClientGrpUpdateMap: (is for each friend, last modif time of group meta data at that friend, all groups included, sent by the friend himself)" << std::endl; - - for(std::map::const_iterator it(mClientGrpUpdateMap.begin());it!=mClientGrpUpdateMap.end();++it) - GXSNETDEBUG_P_(it->first) << " From peer: " << it->first << " - last updated at peer (secs ago): " << nice_time_stamp(time(NULL),it->second->grpUpdateTS) << std::endl; - - GXSNETDEBUG___<< " mClientMsgUpdateMap: (is for each friend, the modif time for each group (e.g. last message received), sent by the friend himself)" << std::endl; - - for(std::map::const_iterator it(mClientMsgUpdateMap.begin());it!=mClientMsgUpdateMap.end();++it) - { - GXSNETDEBUG_P_(it->first) << " From peer: " << it->first << std::endl; - - for(std::map::const_iterator it2(it->second->msgUpdateInfos.begin());it2!=it->second->msgUpdateInfos.end();++it2) - GXSNETDEBUG_PG(it->first,it2->first) << " group " << it2->first << " - last updated at peer (secs ago): " << nice_time_stamp(time(NULL),it2->second.time_stamp) << ". Message count=" << it2->second.message_count << std::endl; - } - - GXSNETDEBUG___<< " List of rejected message ids: " << mRejectedMessages.size() << std::endl; -#endif -} - -// This method is normally not needed, but we use it to correct possible inconsistencies in the updte time stamps -// on the client side. - -void RsGxsNetService::updateClientSyncTS() -{ - RS_STACK_MUTEX(mNxsMutex) ; - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG___<< "updateClientSyncTS(): checking last modification time stamps of local data w.r.t. client's modification times" << std::endl; -#endif - - if(mGrpServerUpdateItem == NULL) - mGrpServerUpdateItem = new RsGxsServerGrpUpdateItem(mServType); - - for(ClientGrpMap::iterator it = mClientGrpUpdateMap.begin();it!=mClientGrpUpdateMap.end();++it) - if(it->second->grpUpdateTS > SECURITY_DELAY_TO_FORCE_CLIENT_REUPDATE + mGrpServerUpdateItem->grpUpdateTS) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(it->first) << " last global client GRP modification time known for peer (" << nice_time_stamp(time(NULL),it->second->grpUpdateTS) << " is quite more recent than our own server modification time (" << nice_time_stamp(time(NULL),mGrpServerUpdateItem->grpUpdateTS) << ". Forcing update! " << std::endl; -#endif - it->second->grpUpdateTS = 0 ; - } - - for(ClientMsgMap::iterator it = mClientMsgUpdateMap.begin();it!=mClientMsgUpdateMap.end();++it) - for(std::map::iterator it2 = it->second->msgUpdateInfos.begin();it2!=it->second->msgUpdateInfos.end();++it2) - { - std::map::const_iterator mmit = mServerMsgUpdateMap.find(it2->first) ; - - if(mmit != mServerMsgUpdateMap.end() && it2->second.time_stamp > SECURITY_DELAY_TO_FORCE_CLIENT_REUPDATE + mmit->second->msgUpdateTS) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(it->first,it2->first) << " last group msg modification time known for peer (" << nice_time_stamp(time(NULL),it2->second.time_stamp) << " and group " << it2->first << " is quite more recent than our own server modification time (" << nice_time_stamp(time(NULL),mmit->second->msgUpdateTS) << ". Forcing update! " << std::endl; -#endif - it2->second.time_stamp = 0 ; - } - } -} - -void RsGxsNetService::updateServerSyncTS() -{ - RS_STACK_MUTEX(mNxsMutex) ; - - RsGxsMetaDataTemporaryMap gxsMap; - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG___<< "updateServerSyncTS(): updating last modification time stamp of local data." << std::endl; -#endif - - // retrieve all grps and update TS - mDataStore->retrieveGxsGrpMetaData(gxsMap); - - // as a grp list server also note this is the latest item you have - if(mGrpServerUpdateItem == NULL) - mGrpServerUpdateItem = new RsGxsServerGrpUpdateItem(mServType); - - // First reset it. That's important because it will re-compute correct TS in case - // we have unsubscribed a group. - - mGrpServerUpdateItem->grpUpdateTS = 0 ; - bool change = false; - - // then remove from mServerMsgUpdateMap, all items that are not in the group list! - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG___ << " cleaning server map of groups with no data:" << std::endl; -#endif - - for(std::map::iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();) - if(gxsMap.find(it->first) == gxsMap.end()) - { - // not found! Removing server update info for this group - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG__G(it->first) << " removing server update info for group " << it->first << std::endl; -#endif - std::map::iterator tmp(it) ; - ++tmp ; - mServerMsgUpdateMap.erase(it) ; - it = tmp ; - } - else - ++it; - -#ifdef NXS_NET_DEBUG_0 - if(gxsMap.empty()) - GXSNETDEBUG___<< " database seems to be empty. The modification timestamp will be reset." << std::endl; -#endif - // finally, update timestamps. - - for(std::map::const_iterator mit = gxsMap.begin();mit != gxsMap.end(); ++mit) - { - const RsGxsGroupId& grpId = mit->first; - const RsGxsGrpMetaData* grpMeta = mit->second; - ServerMsgMap::iterator mapIT = mServerMsgUpdateMap.find(grpId); - RsGxsServerMsgUpdateItem* msui = NULL; - - // That accounts for modification of the meta data. - - if(mGrpServerUpdateItem->grpUpdateTS < grpMeta->mPublishTs) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG__G(grpId) << " publish time stamp of group " << grpId << " has changed to " << time(NULL)-grpMeta->mPublishTs << " secs ago. updating!" << std::endl; -#endif - mGrpServerUpdateItem->grpUpdateTS = grpMeta->mPublishTs; - } - - if(mapIT == mServerMsgUpdateMap.end()) - { - msui = new RsGxsServerMsgUpdateItem(mServType); - msui->grpId = grpMeta->mGroupId; - - mServerMsgUpdateMap.insert(std::make_pair(msui->grpId, msui)); -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG__G(grpId) << " created new entry for group " << grpId << std::endl; -#endif - } - else - msui = mapIT->second; - - if(grpMeta->mLastPost > msui->msgUpdateTS ) - { - change = true; - msui->msgUpdateTS = grpMeta->mLastPost; -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG__G(grpId) << " updated msgUpdateTS to last post = " << time(NULL) - grpMeta->mLastPost << " secs ago for group "<< grpId << std::endl; -#endif - } - - // This might be very inefficient with time. This is needed because an old message might have been received, so the last modification time - // needs to account for this so that a friend who hasn't - - if(mGrpServerUpdateItem->grpUpdateTS < grpMeta->mRecvTS) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG__G(grpId) << " updated msgUpdateTS to last RecvTS = " << time(NULL) - grpMeta->mRecvTS << " secs ago for group "<< grpId << std::endl; -#endif - mGrpServerUpdateItem->grpUpdateTS = grpMeta->mRecvTS; - change = true; - } - } - - // actual change in config settings, then save configuration - if(change) - IndicateConfigChanged(); -} -bool RsGxsNetService::locked_checkTransacTimedOut(NxsTransaction* tr) -{ - return tr->mTimeOut < ((uint32_t) time(NULL)); -} - -void RsGxsNetService::processTransactions() -{ -#ifdef NXS_NET_DEBUG_1 - if(!mTransactions.empty()) - GXSNETDEBUG___ << "processTransactions()" << std::endl; -#endif - RS_STACK_MUTEX(mNxsMutex) ; - - TransactionsPeerMap::iterator mit = mTransactions.begin(); - - for(; mit != mTransactions.end(); ++mit) - { - TransactionIdMap& transMap = mit->second; - TransactionIdMap::iterator mmit = transMap.begin(), mmit_end = transMap.end(); - -#ifdef NXS_NET_DEBUG_1 - if(mmit != mmit_end) - GXSNETDEBUG_P_(mit->first) << " peerId=" << mit->first << std::endl; -#endif - // transaction to be removed - std::list toRemove; - - /*! - * Transactions owned by peer - */ - if(mit->first == mOwnId) - { - for(; mmit != mmit_end; ++mmit) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " type: outgoing " << std::endl; - GXSNETDEBUG_P_(mit->first) << " transN = " << mmit->second->mTransaction->transactionNumber << std::endl; -#endif - NxsTransaction* tr = mmit->second; - uint16_t flag = tr->mFlag; - std::list::iterator lit, lit_end; - uint32_t transN = tr->mTransaction->transactionNumber; - - // first check transaction has not expired - if(locked_checkTransacTimedOut(tr)) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " timeout! " << std::endl; - GXSNETDEBUG_P_(mit->first) << std::dec ; - int total_transaction_time = (int)time(NULL) - (tr->mTimeOut - mTransactionTimeOut) ; - GXSNETDEBUG_P_(mit->first) << " Outgoing Transaction has failed, tranN: " << transN << ", Peer: " << mit->first ; - GXSNETDEBUG_P_(mit->first) << ", age: " << total_transaction_time << ", nItems=" << tr->mTransaction->nItems << ". tr->mTimeOut = " << tr->mTimeOut << ", now = " << (uint32_t) time(NULL) << std::endl; -#endif - - tr->mFlag = NxsTransaction::FLAG_STATE_FAILED; - toRemove.push_back(transN); - mComplTransactions.push_back(tr); - continue; - } -#ifdef NXS_NET_DEBUG_1 - else - GXSNETDEBUG_P_(mit->first) << " still on time." << std::endl; -#endif - - // send items requested - if(flag & NxsTransaction::FLAG_STATE_SENDING) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first)<< " Sending Transaction content, transN: " << transN << " with peer: " << tr->mTransaction->PeerId() << std::endl; -#endif - lit = tr->mItems.begin(); - lit_end = tr->mItems.end(); - - for(; lit != lit_end; ++lit){ - sendItem(*lit); - } - - tr->mItems.clear(); // clear so they don't get deleted in trans cleaning - tr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; - - } - else if(flag & NxsTransaction::FLAG_STATE_WAITING_CONFIRM) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first)<< " Waiting confirm! returning." << std::endl; -#endif - continue; - - } - else if(flag & NxsTransaction::FLAG_STATE_COMPLETED) - { - -#ifdef NXS_NET_DEBUG_1 - int total_transaction_time = (int)time(NULL) - (tr->mTimeOut - mTransactionTimeOut) ; - - GXSNETDEBUG_P_(mit->first)<< " Outgoing completed " << tr->mTransaction->nItems << " items transaction in " << total_transaction_time << " seconds." << std::endl; -#endif - // move to completed transactions - toRemove.push_back(transN); - mComplTransactions.push_back(tr); - }else{ - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first)<< " Unknown flag for active transaction, transN: " << transN << ", Peer: " << mit->first<< std::endl; -#endif - - toRemove.push_back(transN); - tr->mFlag = NxsTransaction::FLAG_STATE_FAILED; - mComplTransactions.push_back(tr); - } - } - - }else{ - - /*! - * Essentially these are incoming transactions - * Several states are dealth with - * Receiving: waiting to receive items from peer's transaction - * and checking if all have been received - * Completed: remove transaction from active and tell peer - * involved in transaction - * Starting: this is a new transaction and need to teell peer - * involved in transaction - */ - - for(; mmit != mmit_end; ++mmit){ - - NxsTransaction* tr = mmit->second; - uint16_t flag = tr->mFlag; - uint32_t transN = tr->mTransaction->transactionNumber; - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " type: incoming " << std::endl; - GXSNETDEBUG_P_(mit->first) << " transN = " << mmit->second->mTransaction->transactionNumber << std::endl; -#endif - // first check transaction has not expired - if(locked_checkTransacTimedOut(tr)) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " timeout!" << std::endl; - GXSNETDEBUG_P_(mit->first) << std::dec ; - int total_transaction_time = (int)time(NULL) - (tr->mTimeOut - mTransactionTimeOut) ; - GXSNETDEBUG_P_(mit->first) << " Incoming Transaction has failed, tranN: " << transN << ", Peer: " << mit->first ; - GXSNETDEBUG_P_(mit->first) << ", age: " << total_transaction_time << ", nItems=" << tr->mTransaction->nItems << ". tr->mTimeOut = " << tr->mTimeOut << ", now = " << (uint32_t) time(NULL) << std::endl; -#endif - - tr->mFlag = NxsTransaction::FLAG_STATE_FAILED; - toRemove.push_back(transN); - mComplTransactions.push_back(tr); - continue; - } - - if(flag & NxsTransaction::FLAG_STATE_RECEIVING) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " received " << tr->mItems.size() << " item over a total of " << tr->mTransaction->nItems << std::endl; -#endif - - // if the number it item received equal that indicated - // then transaction is marked as completed - // to be moved to complete transations - // check if done - if(tr->mItems.size() == tr->mTransaction->nItems) - { - tr->mFlag = NxsTransaction::FLAG_STATE_COMPLETED; -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " completed!" << std::endl; -#endif - } - - }else if(flag & NxsTransaction::FLAG_STATE_COMPLETED) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " transaction is completed!" << std::endl; - GXSNETDEBUG_P_(mit->first) << " sending success!" << std::endl; -#endif - - // send completion msg - RsNxsTransac* trans = new RsNxsTransac(mServType); - trans->clear(); - trans->transactFlag = RsNxsTransac::FLAG_END_SUCCESS; - trans->transactionNumber = transN; - trans->PeerId(tr->mTransaction->PeerId()); - sendItem(trans); - - // move to completed transactions - mComplTransactions.push_back(tr); -#ifdef NXS_NET_DEBUG_1 - int total_transaction_time = (int)time(NULL) - (tr->mTimeOut - mTransactionTimeOut) ; - GXSNETDEBUG_P_(mit->first) << " incoming completed " << tr->mTransaction->nItems << " items transaction in " << total_transaction_time << " seconds." << std::endl; -#endif - - // transaction processing done - // for this id, add to removal list - toRemove.push_back(mmit->first); - } - else if(flag & NxsTransaction::FLAG_STATE_STARTING) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " transaction is starting!" << std::endl; - GXSNETDEBUG_P_(mit->first) << " setting state to Receiving" << std::endl; -#endif - // send item to tell peer your are ready to start - RsNxsTransac* trans = new RsNxsTransac(mServType); - trans->clear(); - trans->transactFlag = RsNxsTransac::FLAG_BEGIN_P2 | - (tr->mTransaction->transactFlag & RsNxsTransac::FLAG_TYPE_MASK); - trans->transactionNumber = transN; - trans->PeerId(tr->mTransaction->PeerId()); - sendItem(trans); - tr->mFlag = NxsTransaction::FLAG_STATE_RECEIVING; - - } - else{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(mit->first) << " transaction is in unknown state. ERROR!" << std::endl; - GXSNETDEBUG_P_(mit->first) << " transaction FAILS!" << std::endl; -#endif - - std::cerr << " Unknown flag for active transaction, transN: " << transN << ", Peer: " << mit->first << std::endl; - toRemove.push_back(mmit->first); - mComplTransactions.push_back(tr); - tr->mFlag = NxsTransaction::FLAG_STATE_FAILED; // flag as a failed transaction - } - } - } - - std::list::iterator lit = toRemove.begin(); - - for(; lit != toRemove.end(); ++lit) - { - transMap.erase(*lit); - } - - } -} - -bool RsGxsNetService::getGroupNetworkStats(const RsGxsGroupId& gid,RsGroupNetworkStats& stats) -{ - RS_STACK_MUTEX(mNxsMutex) ; - - std::map::const_iterator it = mGroupNetworkStats.find(gid) ; - - if(it == mGroupNetworkStats.end()) - return false ; - - stats.mSuppliers = it->second.suppliers.size(); - stats.mMaxVisibleCount = it->second.max_visible_count ; - - return true ; -} - -void RsGxsNetService::processCompletedTransactions() -{ - RS_STACK_MUTEX(mNxsMutex) ; - /*! - * Depending on transaction we may have to respond to peer - * responsible for transaction - */ - while(mComplTransactions.size()>0) - { - - NxsTransaction* tr = mComplTransactions.front(); - - bool outgoing = tr->mTransaction->PeerId() == mOwnId; - - if(outgoing){ - locked_processCompletedOutgoingTrans(tr); - }else{ - locked_processCompletedIncomingTrans(tr); - } - - - delete tr; - mComplTransactions.pop_front(); - } -} - -void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) -{ - uint16_t flag = tr->mTransaction->transactFlag; - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "Processing complete Incoming transaction with " << tr->mTransaction->nItems << " items." << std::endl; - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " flags = " << flag << std::endl; - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " peerId= " << tr->mTransaction->PeerId() << std::endl; -#endif - if(tr->mFlag & NxsTransaction::FLAG_STATE_COMPLETED) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " transaction has completed." << std::endl; -#endif - // for a completed list response transaction - // one needs generate requests from this - if(flag & RsNxsTransac::FLAG_TYPE_MSG_LIST_RESP) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = msg list response." << std::endl; - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " => generate msg request based on it." << std::endl; -#endif - // generate request based on a peers response - locked_genReqMsgTransaction(tr); - - }else if(flag & RsNxsTransac::FLAG_TYPE_GRP_LIST_RESP) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = grp list response." << std::endl; - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " => generate group transaction request based on it." << std::endl; -#endif - locked_genReqGrpTransaction(tr); - } - // you've finished receiving request information now gen - else if(flag & RsNxsTransac::FLAG_TYPE_MSG_LIST_REQ) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = msg list request." << std::endl; - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " => generate msg list based on it." << std::endl; -#endif - locked_genSendMsgsTransaction(tr); - } - else if(flag & RsNxsTransac::FLAG_TYPE_GRP_LIST_REQ) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = grp list request." << std::endl; - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " => generate grp list based on it." << std::endl; -#endif - locked_genSendGrpsTransaction(tr); - } - else if(flag & RsNxsTransac::FLAG_TYPE_GRPS) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = groups." << std::endl; -#endif - std::vector grps; - - while(tr->mItems.size() != 0) - { - RsNxsGrp* grp = dynamic_cast(tr->mItems.front()); - - if(grp) - { - tr->mItems.pop_front(); - grps.push_back(grp); -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grp->grpId) << " adding new group " << grp->grpId << " to incoming list!" << std::endl; -#endif - } - else - std::cerr << " /!\\ item did not caste to grp" << std::endl; - } - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " ...and notifying observer " << std::endl; -#endif -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_P_ (tr->mTransaction->PeerId()) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - Received new groups meta data from peer " << tr->mTransaction->PeerId() << std::endl; - for(uint32_t i=0;imTransaction->PeerId(),grps[i]->grpId) ; -#endif - // notify listener of grps - for(uint32_t i=0;imTransaction->PeerId(); - uint32_t updateTS = tr->mTransaction->updateTS; - - ClientGrpMap::iterator it = mClientGrpUpdateMap.find(peerFrom); - - RsGxsGrpUpdateItem* item = NULL; - - if(it != mClientGrpUpdateMap.end()) - { - item = it->second; - }else - { - item = new RsGxsGrpUpdateItem(mServType); - mClientGrpUpdateMap.insert(std::make_pair(peerFrom, item)); - } -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " and updating mClientGrpUpdateMap for peer " << peerFrom << " of new time stamp " << nice_time_stamp(time(NULL),updateTS) << std::endl; -#endif - -#warning should not we conservatively use the most recent one, in case the peer has reset its mServerGrpUpdate time?? What happens if the peer unsubscribed a recent group? - item->grpUpdateTS = updateTS; - item->peerId = peerFrom; - - IndicateConfigChanged(); - - - }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) - { - - std::vector msgs; -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = msgs." << std::endl; -#endif - RsGxsGroupId grpId; - while(tr->mItems.size() > 0) - { - RsNxsMsg* msg = dynamic_cast(tr->mItems.front()); - if(msg) - { - if(grpId.isNull()) - grpId = msg->grpId; - - tr->mItems.pop_front(); - msgs.push_back(msg); -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),msg->grpId) << " pushing grpId="<< msg->grpId << ", msgsId=" << msg->msgId << " to list of incoming messages" << std::endl; -#endif - } - else - std::cerr << "RsGxsNetService::processCompletedTransactions(): item did not caste to msg" << std::endl; - } - -#ifdef NSXS_FRAG - std::map collatedMsgs; - collateMsgFragments(msgs, collatedMsgs); - - msgs.clear(); - - std::map::iterator mit = collatedMsgs.begin(); - for(; mit != collatedMsgs.end(); ++mit) - { - MsgFragments& f = mit->second; - RsNxsMsg* msg = deFragmentMsg(f); - - if(msg) - msgs.push_back(msg); - } -#endif -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << " ...and notifying observer of " << msgs.size() << " new messages." << std::endl; -#endif -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_PG (tr->mTransaction->PeerId(),grpId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - Received new messages from peer " << tr->mTransaction->PeerId() << " for group " << grpId << std::endl; - for(uint32_t i=0;imTransaction->PeerId(),grpId) << " " << msgs[i]->msgId << std::endl ; -#endif - // notify listener of msgs - for(uint32_t i=0;imTransaction, grpId); - - } - } - else if(tr->mFlag == NxsTransaction::FLAG_STATE_FAILED) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " transaction has failed. Wasting it." << std::endl; -#endif - // don't do anything transaction will simply be cleaned - } - return; -} - -void RsGxsNetService::locked_doMsgUpdateWork(const RsNxsTransac *nxsTrans, const RsGxsGroupId &grpId) -{ -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(nxsTrans->PeerId(),grpId) << "updating MsgUpdate time stamps for peerId=" << nxsTrans->PeerId() << ", grpId=" << grpId << std::endl; -#endif - // firts check if peer exists - const RsPeerId& peerFrom = nxsTrans->PeerId(); - - ClientMsgMap::iterator it = mClientMsgUpdateMap.find(peerFrom); - - RsGxsMsgUpdateItem* mui = NULL; - - // now update the peer's entry for this grp id - if(it != mClientMsgUpdateMap.end()) - { - mui = it->second; - } - else - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(nxsTrans->PeerId(),grpId) << " created new entry." << std::endl; -#endif - mui = new RsGxsMsgUpdateItem(mServType); - mClientMsgUpdateMap.insert(std::make_pair(peerFrom, mui)); - } - - mui->peerId = peerFrom; - - if(mPartialMsgUpdates[peerFrom].find(grpId) != mPartialMsgUpdates[peerFrom].end()) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(nxsTrans->PeerId(),grpId) << " this is a partial update. Not using new time stamp." << std::endl; -#endif - } - else - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(nxsTrans->PeerId(),grpId) << " this is a full update. Updating time stamp." << std::endl; -#endif - mui->msgUpdateInfos[grpId].time_stamp = nxsTrans->updateTS; - IndicateConfigChanged(); - } -} - -void RsGxsNetService::locked_processCompletedOutgoingTrans(NxsTransaction* tr) -{ - uint16_t flag = tr->mTransaction->transactFlag; - -#ifdef NXS_NET_DEBUG_0 - RsNxsTransac *nxsTrans = tr->mTransaction; - GXSNETDEBUG_P_(nxsTrans->PeerId()) << "locked_processCompletedOutgoingTrans(): tr->flags = " << flag << std::endl; -#endif - - if(tr->mFlag & NxsTransaction::FLAG_STATE_COMPLETED) - { - // for a completed list response transaction - // one needs generate requests from this - if(flag & RsNxsTransac::FLAG_TYPE_MSG_LIST_RESP) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Msg List Response, transN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - }else if(flag & RsNxsTransac::FLAG_TYPE_GRP_LIST_RESP) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Grp Response, transN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - } - // you've finished sending a request so don't do anything - else if( (flag & RsNxsTransac::FLAG_TYPE_MSG_LIST_REQ) || - (flag & RsNxsTransac::FLAG_TYPE_GRP_LIST_REQ) ) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Msg/Grp Request, transN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - - }else if(flag & RsNxsTransac::FLAG_TYPE_GRPS) - { - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Grp Data, transN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - - }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Msg Data, transN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - } - }else if(tr->mFlag == NxsTransaction::FLAG_STATE_FAILED){ -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(nxsTrans->PeerId())<< " Failed transaction! transN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - }else{ - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(nxsTrans->PeerId())<< " Serious error unrecognised trans Flag! transN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - } -} - - -void RsGxsNetService::locked_pushMsgTransactionFromList(std::list& reqList, const RsPeerId& peerId, const uint32_t& transN) -{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peerId) << "locked_pushMsgTransactionFromList()" << std::endl; - GXSNETDEBUG_P_(peerId) << " nelems = " << reqList.size() << std::endl; - GXSNETDEBUG_P_(peerId) << " peerId = " << peerId << std::endl; - GXSNETDEBUG_P_(peerId) << " transN = " << transN << std::endl; -#endif -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_P_ (peerId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending message request to peer " - << peerId << " for " << reqList.size() << " messages" << std::endl; -#endif - RsNxsTransac* transac = new RsNxsTransac(mServType); - transac->transactFlag = RsNxsTransac::FLAG_TYPE_MSG_LIST_REQ - | RsNxsTransac::FLAG_BEGIN_P1; - transac->timestamp = 0; - transac->nItems = reqList.size(); - transac->PeerId(peerId); - transac->transactionNumber = transN; - NxsTransaction* newTrans = new NxsTransaction(); - newTrans->mItems = reqList; - newTrans->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; - newTrans->mTimeOut = time(NULL) + mTransactionTimeOut; - // create transaction copy with your id to indicate - // its an outgoing transaction - newTrans->mTransaction = new RsNxsTransac(*transac); - newTrans->mTransaction->PeerId(mOwnId); - sendItem(transac); - - if (!locked_addTransaction(newTrans)) - delete newTrans; - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peerId) << " Requested new transaction for " << reqList.size() << " items." << std::endl; -#endif -} - -void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) -{ - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG___ << "RsGxsNetService::genReqMsgTransaction()" << std::endl; -#endif - - // to create a transaction you need to know who you are transacting with - // then what msgs to request - // then add an active Transaction for request - - std::list msgItemL; - std::list::iterator lit = tr->mItems.begin(); - - // first get item list sent from transaction - for(; lit != tr->mItems.end(); ++lit) - { - RsNxsSyncMsgItem* item = dynamic_cast(*lit); - if(item) - { - msgItemL.push_back(item); - }else - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(item->PeerId()) << "RsGxsNetService::genReqMsgTransaction(): item failed cast to RsNxsSyncMsgItem* " << std::endl; -#endif - } - } -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " found " << msgItemL.size()<< " messages in this transaction." << std::endl; -#endif - - if(msgItemL.empty()) - return; - - // get grp id for this transaction - RsNxsSyncMsgItem* item = msgItemL.front(); - const RsGxsGroupId& grpId = item->grpId; - - // store the count for the peer who sent the message list - uint32_t mcount = msgItemL.size() ; - RsPeerId pid = msgItemL.front()->PeerId() ; - - RsGroupNetworkStatsRecord& gnsr = mGroupNetworkStats[grpId]; - - std::set::size_type oldSuppliersCount = gnsr.suppliers.size(); - uint32_t oldVisibleCount = gnsr.max_visible_count; - - gnsr.suppliers.insert(pid) ; - gnsr.max_visible_count = std::max(gnsr.max_visible_count, mcount) ; - - if (oldVisibleCount != gnsr.max_visible_count || oldSuppliersCount != gnsr.suppliers.size()) - mObserver->notifyChangedGroupStats(grpId); - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " grpId = " << grpId << std::endl; - GXSNETDEBUG_PG(item->PeerId(),grpId) << " retrieving grp mesta data..." << std::endl; -#endif - RsGxsMetaDataTemporaryMap grpMetaMap; - grpMetaMap[grpId] = NULL; - - mDataStore->retrieveGxsGrpMetaData(grpMetaMap); - RsGxsGrpMetaData* grpMeta = grpMetaMap[grpId]; - -#warning TODO: what if grpMeta is NULL? - if(! (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )) - { - // For unsubscribed groups, we update the timestamp something more recent, so that the group content will not be asked to the same - // peer again, unless the peer has new info about it. It's important to use the same clock (this is peer's clock) so that - // we never compare times from different (and potentially badly sync-ed clocks) - - std::cerr << "(EE) stepping in part of the code (" << __PRETTY_FUNCTION__ << ") where we shouldn't. This is a bug." << std::endl; -#ifdef TO_REMOVE - locked_stampPeerGroupUpdateTime(pid,grpId,tr->mTransaction->updateTS,msgItemL.size()) ; -#endif - - return ; - } - - int cutoff = 0; - if(grpMeta != NULL) - cutoff = grpMeta->mReputationCutOff; - - GxsMsgReq reqIds; - reqIds[grpId] = std::vector(); - GxsMsgMetaResult result; - mDataStore->retrieveGxsMsgMetaData(reqIds, result); - std::vector &msgMetaV = result[grpId]; - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " retrieving grp message list..." << std::endl; - GXSNETDEBUG_PG(item->PeerId(),grpId) << " grp locally contains " << msgMetaV.size() << " messsages." << std::endl; -#endif - std::vector::const_iterator vit = msgMetaV.begin(); - std::set msgIdSet; - - // put ids in set for each searching - for(; vit != msgMetaV.end(); ++vit) - { - msgIdSet.insert((*vit)->mMsgId); - delete(*vit); - } - msgMetaV.clear(); - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " grp locally contains " << msgIdSet.size() << " unique messsages." << std::endl; -#endif - // get unique id for this transaction - uint32_t transN = locked_getTransactionId(); - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " new transaction ID: " << transN << std::endl; -#endif - // add msgs that you don't have to request list - std::list::iterator llit = msgItemL.begin(); - std::list reqList; - int reqListSize = 0 ; - - const RsPeerId peerFrom = tr->mTransaction->PeerId(); - - MsgAuthorV toVet; - - std::list peers; - peers.push_back(tr->mTransaction->PeerId()); - bool reqListSizeExceeded = false ; - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " sorting items..." << std::endl; -#endif - for(; llit != msgItemL.end(); ++llit) - { - RsNxsSyncMsgItem*& syncItem = *llit; - const RsGxsMessageId& msgId = syncItem->msgId; - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " msg ID = " << msgId ; -#endif - if(reqListSize >= MAX_REQLIST_SIZE) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << ". reqlist too big. Pruning out this item for now." << std::endl; -#endif - reqListSizeExceeded = true ; - continue ; // we should actually break, but we need to print some debug info. - } - - if(reqListSize < MAX_REQLIST_SIZE && msgIdSet.find(msgId) == msgIdSet.end()) - { - - // if reputation is in reputations cache then proceed - // or if there isn't an author (note as author requirement is - // enforced at service level, if no author is needed then reputation - // filtering is optional) - bool noAuthor = syncItem->authorId.isNull(); - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << ", reqlist size=" << reqListSize << ", message not present." ; -#endif - // grp meta must be present if author present - if(!noAuthor && grpMeta == NULL) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << ", no group meta found. Givign up." << std::endl; -#endif - continue; - } - - if(rsReputations->isIdentityBanned(syncItem->authorId)) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << ", Identity " << syncItem->authorId << " is banned. Not requesting message!" << std::endl; -#endif - continue ; - } - - if(mRejectedMessages.find(msgId) != mRejectedMessages.end()) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << ", message has been recently rejected. Not requesting message!" << std::endl; -#endif - continue ; - } - - - if(mReputations->haveReputation(syncItem->authorId) || noAuthor) - { - GixsReputation rep; - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << ", author Id=" << syncItem->authorId << ". Reputation: " ; -#endif - if(!noAuthor) - mReputations->getReputation(syncItem->authorId, rep); - - // if author is required for this message, it will simply get dropped - // at genexchange side of things - if(rep.score >= (int)grpMeta->mReputationCutOff || noAuthor) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << ", passed! Adding message to req list." << std::endl; -#endif - RsNxsSyncMsgItem* msgItem = new RsNxsSyncMsgItem(mServType); - msgItem->grpId = grpId; - msgItem->msgId = msgId; - msgItem->flag = RsNxsSyncMsgItem::FLAG_REQUEST; - msgItem->transactionNumber = transN; - msgItem->PeerId(peerFrom); - reqList.push_back(msgItem); - ++reqListSize ; - } -#ifdef NXS_NET_DEBUG_1 - else - GXSNETDEBUG_PG(item->PeerId(),grpId) << ", failed!" << std::endl; -#endif - } - else - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << ", no author/no reputation. Pushed to Vetting list." << std::endl; -#endif - // preload for speed - mReputations->loadReputation(syncItem->authorId, peers); - MsgAuthEntry entry; - entry.mAuthorId = syncItem->authorId; - entry.mGrpId = syncItem->grpId; - entry.mMsgId = syncItem->msgId; - toVet.push_back(entry); - } - } -#ifdef NXS_NET_DEBUG_1 - else - GXSNETDEBUG_PG(item->PeerId(),grpId) << ". already here." << std::endl; -#endif - } - - if(!toVet.empty()) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " Vetting list: " << toVet.size() << " elements." << std::endl; -#endif - MsgRespPending* mrp = new MsgRespPending(mReputations, tr->mTransaction->PeerId(), toVet, cutoff); - mPendingResp.push_back(mrp); - } - - if(!reqList.empty()) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " Request list: " << reqList.size() << " elements." << std::endl; -#endif - locked_pushMsgTransactionFromList(reqList, tr->mTransaction->PeerId(), transN); - - if(reqListSizeExceeded) - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " Marking update operation as unfinished." << std::endl; -#endif - mPartialMsgUpdates[tr->mTransaction->PeerId()].insert(item->grpId) ; - } - else - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " Marking update operation as terminal." << std::endl; -#endif - mPartialMsgUpdates[tr->mTransaction->PeerId()].erase(item->grpId) ; - } - } - else - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(item->PeerId(),grpId) << " Request list is empty. Not doing anything. " << std::endl; -#endif - // The list to req is empty. That means we already have all messages that this peer can - // provide. So we can stamp the group from this peer to be up to date. - - // Part of this is already achieved in two other places: - // - the GroupStats exchange system, which counts the messages at each peer. It could also supply TS for the messages, but it does not for the time being - // - client TS are updated when receiving messages - - locked_stampPeerGroupUpdateTime(pid,grpId,tr->mTransaction->updateTS,msgItemL.size()) ; - } -} - -void RsGxsNetService::locked_stampPeerGroupUpdateTime(const RsPeerId& pid,const RsGxsGroupId& grpId,time_t tm,uint32_t n_messages) -{ - std::map::iterator it = mClientMsgUpdateMap.find(pid) ; - - RsGxsMsgUpdateItem *pitem; - - if(it == mClientMsgUpdateMap.end()) - { - pitem = new RsGxsMsgUpdateItem(mServType) ; - pitem->peerId = pid ; - - mClientMsgUpdateMap[pid] = pitem ; - } - else - pitem = it->second ; - - pitem->msgUpdateInfos[grpId].time_stamp = tm; - pitem->msgUpdateInfos[grpId].message_count = std::max(n_messages, pitem->msgUpdateInfos[grpId].message_count) ; - - IndicateConfigChanged(); -} - -void RsGxsNetService::locked_pushGrpTransactionFromList( std::list& reqList, const RsPeerId& peerId, const uint32_t& transN) -{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peerId) << "locked_pushGrpTransactionFromList()" << std::endl; - GXSNETDEBUG_P_(peerId) << " nelems = " << reqList.size() << std::endl; - GXSNETDEBUG_P_(peerId) << " peerId = " << peerId << std::endl; - GXSNETDEBUG_P_(peerId) << " transN = " << transN << std::endl; -#endif -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_P_ (peerId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending group request to peer " - << peerId << " for " << reqList.size() << " groups" << std::endl; -#endif - RsNxsTransac* transac = new RsNxsTransac(mServType); - transac->transactFlag = RsNxsTransac::FLAG_TYPE_GRP_LIST_REQ - | RsNxsTransac::FLAG_BEGIN_P1; - transac->timestamp = 0; - transac->nItems = reqList.size(); - transac->PeerId(peerId); - transac->transactionNumber = transN; - NxsTransaction* newTrans = new NxsTransaction(); - newTrans->mItems = reqList; - newTrans->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; - newTrans->mTimeOut = time(NULL) + mTransactionTimeOut; - newTrans->mTransaction = new RsNxsTransac(*transac); - newTrans->mTransaction->PeerId(mOwnId); - sendItem(transac); - if (!locked_addTransaction(newTrans)) - delete newTrans; -} -void RsGxsNetService::addGroupItemToList(NxsTransaction*& tr, const RsGxsGroupId& grpId, uint32_t& transN, std::list& reqList) -{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << "RsGxsNetService::addGroupItemToList() Added GroupID: << grpId" << std::endl; -#endif - - RsNxsSyncGrpItem* grpItem = new RsNxsSyncGrpItem(mServType); - grpItem->PeerId(tr->mTransaction->PeerId()); - grpItem->grpId = grpId; - grpItem->flag = RsNxsSyncMsgItem::FLAG_REQUEST; - grpItem->transactionNumber = transN; - reqList.push_back(grpItem); -} - -void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) -{ - // to create a transaction you need to know who you are transacting with - // then what grps to request - // then add an active Transaction for request - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genReqGrpTransaction(): " << std::endl; -#endif - - RsGxsMetaDataTemporaryMap grpMetaMap; - - std::list grpItemL; - - for(std::list::iterator lit = tr->mItems.begin(); lit != tr->mItems.end(); ++lit) - { - RsNxsSyncGrpItem* item = dynamic_cast(*lit); - if(item) - { - grpItemL.push_back(item); - grpMetaMap[item->grpId] = NULL; - }else - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),item->grpId) << "RsGxsNetService::genReqGrpTransaction(): item failed to caste to RsNxsSyncMsgItem* " << std::endl; -#endif - } - } - - if (grpItemL.empty()) - return; - - mDataStore->retrieveGxsGrpMetaData(grpMetaMap); - - // now do compare and add loop - std::list::iterator llit = grpItemL.begin(); - std::list reqList; - - uint32_t transN = locked_getTransactionId(); - - GrpAuthorV toVet; - std::list peers; - peers.push_back(tr->mTransaction->PeerId()); - - for(; llit != grpItemL.end(); ++llit) - { - RsNxsSyncGrpItem*& grpSyncItem = *llit; - const RsGxsGroupId& grpId = grpSyncItem->grpId; - - std::map::const_iterator metaIter = grpMetaMap.find(grpId); - bool haveItem = false; - bool latestVersion = false; - - if (metaIter != grpMetaMap.end() && metaIter->second) - { - haveItem = true; - latestVersion = grpSyncItem->publishTs > metaIter->second->mPublishTs; - } - - if(!grpSyncItem->authorId.isNull() && rsReputations->isIdentityBanned(grpSyncItem->authorId)) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << " Identity " << grpSyncItem->authorId << " is banned. Not syncing group." << std::endl; -#endif - continue ; - } - - if( (mGrpAutoSync && !haveItem) || latestVersion) - { - // determine if you need to check reputation - bool checkRep = !grpSyncItem->authorId.isNull(); - - // check if you have reputation, if you don't then - // place in holding pen - if(checkRep) - { - if(mReputations->haveReputation(grpSyncItem->authorId)) - { - GixsReputation rep; - mReputations->getReputation(grpSyncItem->authorId, rep); - - if(rep.score >= GIXS_CUT_OFF) - { - addGroupItemToList(tr, grpId, transN, reqList); -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId)<< " reputation cut off: limit=" << GIXS_CUT_OFF << " value=" << rep.score << ": allowed." << std::endl; -#endif - } -#ifdef NXS_NET_DEBUG_0 - else - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId)<< " reputation cut off: limit=" << GIXS_CUT_OFF << " value=" << rep.score << ": you shall not pass." << std::endl; -#endif - } - else - { - // preload reputation for later - mReputations->loadReputation(grpSyncItem->authorId, peers); - GrpAuthEntry entry; - entry.mAuthorId = grpSyncItem->authorId; - entry.mGrpId = grpSyncItem->grpId; - toVet.push_back(entry); - } - } - else - { - addGroupItemToList(tr, grpId, transN, reqList); - } - } - } - - if(!toVet.empty()) - { - RsPeerId peerId = tr->mTransaction->PeerId(); - GrpRespPending* grp = new GrpRespPending(mReputations, peerId, toVet); - mPendingResp.push_back(grp); - } - - - if(!reqList.empty()) - locked_pushGrpTransactionFromList(reqList, tr->mTransaction->PeerId(), transN); -} - -void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) -{ - -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendGrpsTransaction() Generating Grp data send fron TransN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - - // go groups requested in transaction tr - - std::list::iterator lit = tr->mItems.begin(); - - RsGxsMetaDataTemporaryMap grps ; - - for(;lit != tr->mItems.end(); ++lit) - { - RsNxsSyncGrpItem* item = dynamic_cast(*lit); - if (item) - grps[item->grpId] = NULL; - else - { -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),item->grpId) << "RsGxsNetService::locked_genSendGrpsTransaction(): item failed to caste to RsNxsSyncGrpItem* " << std::endl; -#endif - } - } - - if(!grps.empty()) - mDataStore->retrieveNxsGrps(grps, false, false); - else - return; - - NxsTransaction* newTr = new NxsTransaction(); - newTr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; - - uint32_t transN = locked_getTransactionId(); - - // store grp items to send in transaction - std::map::iterator mit = grps.begin(); - RsPeerId peerId = tr->mTransaction->PeerId(); - for(;mit != grps.end(); ++mit) - { - mit->second->PeerId(peerId); // set so it gets sent to right peer - mit->second->transactionNumber = transN; - newTr->mItems.push_back(mit->second); - mit->second = NULL ; // avoids deletion - } - - if(newTr->mItems.empty()){ - delete newTr; - return; - } - - uint32_t updateTS = 0; - if(mGrpServerUpdateItem) - updateTS = mGrpServerUpdateItem->grpUpdateTS; - -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_P_ (tr->mTransaction->PeerId()) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending global group TS " - << updateTS << " to peer " << tr->mTransaction->PeerId() << std::endl; -#endif - RsNxsTransac* ntr = new RsNxsTransac(mServType); - ntr->transactionNumber = transN; - ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_GRPS; - ntr->updateTS = updateTS; - ntr->nItems = grps.size(); - ntr->PeerId(tr->mTransaction->PeerId()); - - newTr->mTransaction = new RsNxsTransac(*ntr); - newTr->mTransaction->PeerId(mOwnId); - newTr->mTimeOut = time(NULL) + mTransactionTimeOut; - - ntr->PeerId(tr->mTransaction->PeerId()); - sendItem(ntr); - - locked_addTransaction(newTr); - - return; -} - -void RsGxsNetService::runVetting() -{ - RS_STACK_MUTEX(mNxsMutex) ; - - std::vector::iterator vit = mPendingResp.begin(); - - for(; vit != mPendingResp.end(); ) - { - AuthorPending* ap = *vit; - - if(ap->accepted() || ap->expired()) - { - // add to transactions - if(AuthorPending::MSG_PEND == ap->getType()) - { - MsgRespPending* mrp = static_cast(ap); - locked_createTransactionFromPending(mrp); - } - else if(AuthorPending::GRP_PEND == ap->getType()) - { - GrpRespPending* grp = static_cast(ap); - locked_createTransactionFromPending(grp); - }else - std::cerr << "RsGxsNetService::runVetting(): Unknown pending type! Type: " << ap->getType() << std::endl; - - delete ap; - vit = mPendingResp.erase(vit); - } - else - { - ++vit; - } - - } - - - // now lets do circle vetting - std::vector::iterator vit2 = mPendingCircleVets.begin(); - for(; vit2 != mPendingCircleVets.end(); ) - { - GrpCircleVetting*& gcv = *vit2; - if(gcv->cleared() || gcv->expired()) - { - if(gcv->getType() == GrpCircleVetting::GRP_ID_PEND) - { - GrpCircleIdRequestVetting* gcirv = - static_cast(gcv); - - locked_createTransactionFromPending(gcirv); - } - else if(gcv->getType() == GrpCircleVetting::MSG_ID_SEND_PEND) - { - MsgCircleIdsRequestVetting* mcirv = - static_cast(gcv); - - if(mcirv->cleared()) - locked_createTransactionFromPending(mcirv); - } - else - { -#ifdef NXS_NET_DEBUG_4 - std::cerr << "RsGxsNetService::runVetting(): Unknown Circle pending type! Type: " << gcv->getType() << std::endl; -#endif - } - - delete gcv; - vit2 = mPendingCircleVets.erase(vit2); - } - else - { - ++vit2; - } - } -} - -void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) -{ - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendMsgsTransaction() Generating Msg data send fron TransN: " << tr->mTransaction->transactionNumber << std::endl; -#endif - - // go groups requested in transaction tr - - std::list::iterator lit = tr->mItems.begin(); - - GxsMsgReq msgIds; - GxsMsgResult msgs; - - if(tr->mItems.empty()){ - return; - } - - // hacky assumes a transaction only consist of a single grpId - RsGxsGroupId grpId; - - for(;lit != tr->mItems.end(); ++lit) - { - RsNxsSyncMsgItem* item = dynamic_cast(*lit); - if (item) - { - msgIds[item->grpId].push_back(item->msgId); - - if(grpId.isNull()) - grpId = item->grpId; - } - else - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << "RsGxsNetService::locked_genSendMsgsTransaction(): item failed to caste to RsNxsSyncMsgItem* " << std::endl; -#endif - } - } - - mDataStore->retrieveNxsMsgs(msgIds, msgs, false, false); - - NxsTransaction* newTr = new NxsTransaction(); - newTr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; - - uint32_t transN = locked_getTransactionId(); - - // store msg items to send in transaction - GxsMsgResult::iterator mit = msgs.begin(); - RsPeerId peerId = tr->mTransaction->PeerId(); - uint32_t msgSize = 0; - - for(;mit != msgs.end(); ++mit) - { - std::vector& msgV = mit->second; - std::vector::iterator vit = msgV.begin(); - - for(; vit != msgV.end(); ++vit) - { - RsNxsMsg* msg = *vit; - msg->PeerId(peerId); - msg->transactionNumber = transN; - -#ifndef NXS_FRAG - newTr->mItems.push_back(msg); - msgSize++; -#else - MsgFragments fragments; - fragmentMsg(*msg, fragments); - - MsgFragments::iterator mit = fragments.begin(); - - for(; mit != fragments.end(); ++mit) - { - newTr->mItems.push_back(*mit); - msgSize++; - } -#endif - } - } - - if(newTr->mItems.empty()){ - delete newTr; - return; - } - - uint32_t updateTS = 0; - - ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(grpId); - - if(cit != mServerMsgUpdateMap.end()) - updateTS = cit->second->msgUpdateTS; - - RsNxsTransac* ntr = new RsNxsTransac(mServType); - ntr->transactionNumber = transN; - ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | - RsNxsTransac::FLAG_TYPE_MSGS; - ntr->updateTS = updateTS; - ntr->nItems = msgSize; - ntr->PeerId(peerId); - - newTr->mTransaction = new RsNxsTransac(*ntr); - newTr->mTransaction->PeerId(mOwnId); - newTr->mTimeOut = time(NULL) + mTransactionTimeOut; - -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_PG (peerId,grpId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending message update to peer " - << peerId << " for group " << grpId << " with TS=" << nice_time_stamp(time(NULL),updateTS) <<" (secs ago)" << std::endl; -#endif - ntr->PeerId(tr->mTransaction->PeerId()); - sendItem(ntr); - - locked_addTransaction(newTr); - - return; -} -uint32_t RsGxsNetService::locked_getTransactionId() -{ - return ++mTransactionN; -} -bool RsGxsNetService::locked_addTransaction(NxsTransaction* tr) -{ - const RsPeerId& peer = tr->mTransaction->PeerId(); - uint32_t transN = tr->mTransaction->transactionNumber; - TransactionIdMap& transMap = mTransactions[peer]; - bool transNumExist = transMap.find(transN) - != transMap.end(); - - - if(transNumExist){ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << "locked_addTransaction() " << std::endl; - GXSNETDEBUG_P_(peer) << "Transaction number exist already, transN: " << transN << std::endl; -#endif - return false; - }else{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(peer) << "locked_addTransaction() " << std::endl; - GXSNETDEBUG_P_(peer) << "Added transaction number " << transN << std::endl; -#endif - transMap[transN] = tr; - return true; - } -} - -void RsGxsNetService::cleanTransactionItems(NxsTransaction* tr) const -{ - std::list::iterator lit = tr->mItems.begin(); - - for(; lit != tr->mItems.end(); ++lit) - { - delete *lit; - } - - tr->mItems.clear(); -} - -void RsGxsNetService::locked_pushGrpRespFromList(std::list& respList, const RsPeerId& peer, const uint32_t& transN) -{ -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(peer) << "locked_pushGrpResponseFromList()" << std::endl; - GXSNETDEBUG_P_(peer) << " nelems = " << respList.size() << std::endl; - GXSNETDEBUG_P_(peer) << " peerId = " << peer << std::endl; - GXSNETDEBUG_P_(peer) << " transN = " << transN << std::endl; -#endif - NxsTransaction* tr = new NxsTransaction(); - tr->mItems = respList; - - tr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; - RsNxsTransac* trItem = new RsNxsTransac(mServType); - trItem->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 - | RsNxsTransac::FLAG_TYPE_GRP_LIST_RESP; - trItem->nItems = respList.size(); - trItem->timestamp = 0; - trItem->PeerId(peer); - trItem->transactionNumber = transN; - // also make a copy for the resident transaction - tr->mTransaction = new RsNxsTransac(*trItem); - tr->mTransaction->PeerId(mOwnId); - tr->mTimeOut = time(NULL) + mTransactionTimeOut; - // signal peer to prepare for transaction -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_P_ (peer) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending group response to peer " - << peer << " with " << respList.size() << " groups " << std::endl; -#endif - sendItem(trItem); - locked_addTransaction(tr); -} - -bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncGrp *item) -{ - // Do we have new updates for this peer? - - // This is one of the few places where we compare a local time stamp (mGrpServerUpdateItem->grpUpdateTS) to a peer's time stamp. - // Because this is the global modification time for groups, async-ed computers will eventually figure out that their data needs - // to be synced. - - if(mGrpServerUpdateItem) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(item->PeerId()) << " local modification time stamp: " << std::dec<< time(NULL) - mGrpServerUpdateItem->grpUpdateTS << " secs ago. Update sent: " << - ((item->updateTS < mGrpServerUpdateItem->grpUpdateTS)?"YES":"NO") << std::endl; -#endif - return item->updateTS < mGrpServerUpdateItem->grpUpdateTS; - } -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(item->PeerId()) << " no local time stamp. This will be fixed after updateServerSyncTS(). Not sending for now. " << std::endl; -#endif - - return false; -} - -void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) -{ - if (!item) - return; - - RS_STACK_MUTEX(mNxsMutex) ; - - RsPeerId peer = item->PeerId(); -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(peer) << "HandleRecvSyncGroup(): Service: " << mServType << " from " << peer << ", Last update TS (from myself) sent from peer is T = " << std::dec<< time(NULL) - item->updateTS << " secs ago" << std::endl; -#endif - - if(!locked_CanReceiveUpdate(item)) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(peer) << " RsGxsNetService::handleRecvSyncGroup() update will not be sent." << std::endl; -#endif - return; - } - - RsGxsMetaDataTemporaryMap grp; - mDataStore->retrieveGxsGrpMetaData(grp); - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(peer) << " RsGxsNetService::handleRecvSyncGroup() retrieving local list of groups..." << std::endl; -#endif - if(grp.empty()) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(peer) << " RsGxsNetService::handleRecvSyncGroup() Grp Empty" << std::endl; -#endif - return; - } - - std::list itemL; - - uint32_t transN = locked_getTransactionId(); - - std::vector toVet; -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(peer) << " Group list beings being sent: " << std::endl; -#endif - - for(std::map::iterator mit = grp.begin(); mit != grp.end(); ++mit) - { - RsGxsGrpMetaData* grpMeta = mit->second; - - // Only send info about subscribed groups. - - if(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) - { - - // check if you can send this id to peer - // or if you need to add to the holding - // pen for peer to be vetted - if(canSendGrpId(peer, *grpMeta, toVet)) - { - RsNxsSyncGrpItem* gItem = new RsNxsSyncGrpItem(mServType); - gItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE; - gItem->grpId = mit->first; - gItem->publishTs = mit->second->mPublishTs; - gItem->authorId = grpMeta->mAuthorId; - gItem->PeerId(peer); - gItem->transactionNumber = transN; - itemL.push_back(gItem); -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(peer,mit->first) << " sending item for Grp " << mit->first << " name=" << grpMeta->mGroupName << ", publishTS=" << std::dec<< time(NULL) - mit->second->mPublishTs << " secs ago to peer ID " << peer << std::endl; -#endif - } - } - } - - if(!toVet.empty()) - { - mPendingCircleVets.push_back(new GrpCircleIdRequestVetting(mCircles, mPgpUtils, toVet, peer)); - } - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_P_(peer) << " final list sent (after vetting): " << itemL.size() << " elements." << std::endl; -#endif - locked_pushGrpRespFromList(itemL, peer, transN); - - return; -} - - - -bool RsGxsNetService::canSendGrpId(const RsPeerId& sslId, RsGxsGrpMetaData& grpMeta, std::vector& toVet) -{ -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId()"<< std::endl; -#endif - // first do the simple checks - uint8_t circleType = grpMeta.mCircleType; - - if(circleType == GXS_CIRCLE_TYPE_LOCAL) - { -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId)<< "RsGxsNetService::canSendGrpId() LOCAL_CIRCLE, cannot send"<< std::endl; -#endif - return false; - } - - if(circleType == GXS_CIRCLE_TYPE_PUBLIC) - { -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId)<< "RsGxsNetService::canSendGrpId() PUBLIC_CIRCLE, can send"<< std::endl; -#endif - return true; - } - - if(circleType == GXS_CIRCLE_TYPE_EXTERNAL) - { - const RsGxsCircleId& circleId = grpMeta.mCircleId; - if(circleId.isNull()) - { - std::cerr << "RsGxsNetService::canSendGrpId() ERROR; EXTERNAL_CIRCLE missing NULL CircleId: " << grpMeta.mGroupId<< std::endl; - - // ERROR, will never be shared. - return false; - } - - if(mCircles->isLoaded(circleId)) - { -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId)<< "RsGxsNetService::canSendGrpId() EXTERNAL_CIRCLE, checking mCircles->canSend"<< std::endl; -#endif - const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); - return mCircles->canSend(circleId, pgpId); - } - - toVet.push_back(GrpIdCircleVet(grpMeta.mGroupId, circleId, grpMeta.mAuthorId)); - return false; - } - - if(circleType == GXS_CIRCLE_TYPE_YOUREYESONLY) - { -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId)<< "RsGxsNetService::canSendGrpId() YOUREYESONLY, checking further"<< std::endl; -#endif - // a non empty internal circle id means this - // is the personal circle owner - if(!grpMeta.mInternalCircle.isNull()) - { - const RsGxsCircleId& internalCircleId = grpMeta.mInternalCircle; -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() have mInternalCircle - we are Group creator" << std::endl; - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() mCircleId: " << grpMeta.mCircleId << std::endl; - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() mInternalCircle: " << grpMeta.mInternalCircle << std::endl; -#endif - - if(mCircles->isLoaded(internalCircleId)) - { -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() circle Loaded - checking mCircles->canSend" << std::endl; -#endif - const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); - return mCircles->canSend(internalCircleId, pgpId); - } - -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() Circle Not Loaded - add to vetting"<< std::endl; -#endif - toVet.push_back(GrpIdCircleVet(grpMeta.mGroupId, internalCircleId, grpMeta.mAuthorId)); - return false; - } - else - { - // an empty internal circle id means this peer can only - // send circle related info from peer he received it -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() mInternalCircle not set, someone else's personal circle"<< std::endl; -#endif - if(grpMeta.mOriginator == sslId) - { -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() Originator matches -> can send"<< std::endl; -#endif - return true; - } - else - { -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() Originator doesn't match -> cannot send"<< std::endl; -#endif - return false; - } - } - } - - return true; -} - -bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxsGrpMetaData& grpMeta) -{ - - #ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::checkCanRecvMsgFromPeer()"; - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " peer Id = " << sslId << ", grpId=" << grpMeta.mGroupId <isLoaded(circleId)) - { - #ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " EXTERNAL_CIRCLE, checking mCircles->canSend" << std::endl; - #endif - const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); - return mCircles->canSend(circleId, pgpId); - } - else - mCircles->loadCircle(circleId); // simply request for next pass - - return false; - } - - if(circleType == GXS_CIRCLE_TYPE_YOUREYESONLY) // do not attempt to sync msg unless to originator or those permitted - { - #ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " YOUREYESONLY, checking further" << std::endl; - #endif - // a non empty internal circle id means this - // is the personal circle owner - if(!grpMeta.mInternalCircle.isNull()) - { - const RsGxsCircleId& internalCircleId = grpMeta.mInternalCircle; - #ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " have mInternalCircle - we are Group creator" << std::endl; - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " mCircleId: " << grpMeta.mCircleId << std::endl; - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " mInternalCircle: " << grpMeta.mInternalCircle << std::endl; - #endif - - if(mCircles->isLoaded(internalCircleId)) - { - #ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " circle Loaded - checking mCircles->canSend" << std::endl; - #endif - const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); - return mCircles->canSend(internalCircleId, pgpId); - } - else - mCircles->loadCircle(internalCircleId); // request for next pass - - return false; - } - else - { - // an empty internal circle id means this peer can only - // send circle related info from peer he received it - #ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " mInternalCircle not set, someone else's personal circle" << std::endl; - #endif - if(grpMeta.mOriginator == sslId) - { - #ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Originator matches -> can send" << std::endl; - #endif - return true; - } - else - { - #ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Originator doesn't match -> cannot send"<< std::endl; - #endif - return false; - } - } - } - - return true; -} - -bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncMsg *item) -{ - // Do we have new updates for this peer? - // Here we compare times in the same clock: the friend's clock, so it should be fine. - - ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId); - - if(cit != mServerMsgUpdateMap.end()) - { - const RsGxsServerMsgUpdateItem *msui = cit->second; - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " local time stamp: " << std::dec<< time(NULL) - msui->msgUpdateTS << " secs ago. Update sent: " << (item->updateTS < msui->msgUpdateTS) ; -#endif - return item->updateTS < msui->msgUpdateTS ; - } -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " no local time stamp for this grp. " ; -#endif - - return false; -} -void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsg* item) -{ - if (!item) - return; - - RS_STACK_MUTEX(mNxsMutex) ; - - const RsPeerId& peer = item->PeerId(); - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << "handleRecvSyncMsg(): Received last update TS of group " << item->grpId << ", for peer " << peer << ", TS = " << time(NULL) - item->updateTS << " secs ago." ; -#endif - - if(!locked_CanReceiveUpdate(item)) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " no update will be sent." << std::endl; -#endif - return; - } - - GxsMsgMetaResult metaResult; - GxsMsgReq req; - - RsGxsMetaDataTemporaryMap grpMetas; - grpMetas[item->grpId] = NULL; - - mDataStore->retrieveGxsGrpMetaData(grpMetas); - RsGxsGrpMetaData* grpMeta = grpMetas[item->grpId]; - - if(grpMeta == NULL) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " Grp is unknown." << std::endl; -#endif - return; - } - if(!(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " Grp is not subscribed." << std::endl; -#endif - return ; - } - - req[item->grpId] = std::vector(); - mDataStore->retrieveGxsMsgMetaData(req, metaResult); - std::vector& msgMetas = metaResult[item->grpId]; - -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " retrieving message meta data." << std::endl; -#endif - if(req.empty()) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " No msg meta data.." << std::endl; -#endif - } -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " Sending MSG meta data!" << std::endl; -#endif - - std::list itemL; - - uint32_t transN = locked_getTransactionId(); - - if(canSendMsgIds(msgMetas, *grpMeta, peer)) - { - std::vector::iterator vit = msgMetas.begin(); - - for(; vit != msgMetas.end(); ++vit) - { - RsGxsMsgMetaData* m = *vit; - - RsNxsSyncMsgItem* mItem = new RsNxsSyncMsgItem(mServType); - mItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE; - mItem->grpId = m->mGroupId; - mItem->msgId = m->mMsgId; - mItem->authorId = m->mAuthorId; - mItem->PeerId(peer); - mItem->transactionNumber = transN; - itemL.push_back(mItem); -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " sending info item for msg id " << mItem->msgId << std::endl; -#endif - } - - if(!itemL.empty()) - { -#ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " sending final msg info list of " << itemL.size() << " items." << std::endl; -#endif - locked_pushMsgRespFromList(itemL, peer, item->grpId,transN); - } - } -#ifdef NXS_NET_DEBUG_0 - else - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " vetting forbids sending. Nothing will be sent." << itemL.size() << " items." << std::endl; -#endif - - std::vector::iterator vit = msgMetas.begin(); - // release meta resource - for(vit = msgMetas.begin(); vit != msgMetas.end(); ++vit) - delete *vit; -} - -void RsGxsNetService::locked_pushMsgRespFromList(std::list& itemL, const RsPeerId& sslId, const RsGxsGroupId& grp_id,const uint32_t& transN) -{ -#ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_PG(sslId,grp_id) << "locked_pushMsgResponseFromList()" << std::endl; - GXSNETDEBUG_PG(sslId,grp_id) << " nelems = " << itemL.size() << std::endl; - GXSNETDEBUG_PG(sslId,grp_id) << " peerId = " << sslId << std::endl; - GXSNETDEBUG_PG(sslId,grp_id) << " transN = " << transN << std::endl; -#endif - NxsTransaction* tr = new NxsTransaction(); - tr->mItems = itemL; - tr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; - RsNxsTransac* trItem = new RsNxsTransac(mServType); - trItem->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_MSG_LIST_RESP; - trItem->nItems = itemL.size(); - trItem->timestamp = 0 ; - trItem->PeerId(sslId); - trItem->transactionNumber = transN; - - ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(grp_id); - - // This time stamp is not supposed to be used on the other side. We just set it to avoid sending an uninitialiszed value. - - if(cit != mServerMsgUpdateMap.end()) - trItem->updateTS = cit->second->msgUpdateTS; - else - { - std::cerr << "(EE) cannot find a server TS for message of group " << grp_id << " in locked_pushMsgRespFromList. This is weird." << std::endl; - trItem->updateTS = 0 ; - } - - // also make a copy for the resident transaction - tr->mTransaction = new RsNxsTransac(*trItem); - tr->mTransaction->PeerId(mOwnId); - tr->mTimeOut = time(NULL) + mTransactionTimeOut; - -#ifdef NXS_NET_DEBUG_5 - GXSNETDEBUG_P_ (sslId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending messages response to peer " - << sslId << " with " << itemL.size() << " messages " << std::endl; -#endif - // signal peer to prepare for transaction - sendItem(trItem); - - locked_addTransaction(tr); -} - -bool RsGxsNetService::canSendMsgIds(const std::vector& msgMetas, - const RsGxsGrpMetaData& grpMeta, const RsPeerId& sslId) -{ -#ifdef NXS_NET_DEBUG_4 - GXSNETDEBUG__G(grpMeta.mGroupId) << "RsGxsNetService::canSendMsgIds() CIRCLE VETTING" << std::endl; -#endif - - // first do the simple checks - uint8_t circleType = grpMeta.mCircleType; - - if(circleType == GXS_CIRCLE_TYPE_LOCAL) - return false; - - if(circleType == GXS_CIRCLE_TYPE_PUBLIC) - return true; - - const RsGxsCircleId& circleId = grpMeta.mCircleId; - - if(circleType == GXS_CIRCLE_TYPE_EXTERNAL) - { - if(mCircles->isLoaded(circleId)) - { - const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); - return mCircles->canSend(circleId, pgpId); - } - - std::vector toVet; - std::vector::const_iterator vit = msgMetas.begin(); - - for(; vit != msgMetas.end(); ++vit) - { - const RsGxsMsgMetaData* const& meta = *vit; - - MsgIdCircleVet mic(meta->mMsgId, meta->mAuthorId); - toVet.push_back(mic); - } - - if(!toVet.empty()) - mPendingCircleVets.push_back(new MsgCircleIdsRequestVetting(mCircles, mPgpUtils, toVet, grpMeta.mGroupId, - sslId, grpMeta.mCircleId)); - - return false; - } - - if(circleType == GXS_CIRCLE_TYPE_YOUREYESONLY) - { - // a non empty internal circle id means this - // is the personal circle owner - if(!grpMeta.mInternalCircle.isNull()) - { - const RsGxsCircleId& internalCircleId = grpMeta.mInternalCircle; - if(mCircles->isLoaded(internalCircleId)) - { - const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); - return mCircles->canSend(internalCircleId, pgpId); - } - - std::vector toVet; - std::vector::const_iterator vit = msgMetas.begin(); - - for(; vit != msgMetas.end(); ++vit) - { - const RsGxsMsgMetaData* const& meta = *vit; - - MsgIdCircleVet mic(meta->mMsgId, meta->mAuthorId); - toVet.push_back(mic); - } - - if(!toVet.empty()) - mPendingCircleVets.push_back(new MsgCircleIdsRequestVetting(mCircles, mPgpUtils, - toVet, grpMeta.mGroupId, - sslId, grpMeta.mCircleId)); - - return false; - } - else - { - // an empty internal circle id means this peer can only - // send circle related info from peer he received it - if(grpMeta.mOriginator == sslId) - return true; - else - return false; - } - } - - return true; -} - -/** inherited methods **/ - -void RsGxsNetService::pauseSynchronisation(bool /* enabled */) -{ - -} - -void RsGxsNetService::setSyncAge(uint32_t /* age */) -{ - -} - -int RsGxsNetService::requestGrp(const std::list& grpId, const RsPeerId& peerId) -{ - RS_STACK_MUTEX(mNxsMutex) ; - mExplicitRequest[peerId].assign(grpId.begin(), grpId.end()); - return 1; -} - -void RsGxsNetService::processExplicitGroupRequests() -{ - RS_STACK_MUTEX(mNxsMutex) ; - - std::map >::const_iterator cit = mExplicitRequest.begin(); - - for(; cit != mExplicitRequest.end(); ++cit) - { - const RsPeerId& peerId = cit->first; - const std::list& groupIdList = cit->second; - - std::list grpSyncItems; - std::list::const_iterator git = groupIdList.begin(); - uint32_t transN = locked_getTransactionId(); - for(; git != groupIdList.end(); ++git) - { - RsNxsSyncGrpItem* item = new RsNxsSyncGrpItem(mServType); - item->grpId = *git; - item->PeerId(peerId); - item->flag = RsNxsSyncGrpItem::FLAG_REQUEST; - item->transactionNumber = transN; - grpSyncItems.push_back(item); - } - - if(!grpSyncItems.empty()) - locked_pushGrpTransactionFromList(grpSyncItems, peerId, transN); - } - - mExplicitRequest.clear(); -} - -int RsGxsNetService::sharePublishKey(const RsGxsGroupId& grpId,const std::set& peers) -{ - RS_STACK_MUTEX(mNxsMutex) ; - - mPendingPublishKeyRecipients[grpId] = peers ; -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG__G(grpId) << "RsGxsNetService::sharePublishKeys() " << (void*)this << " adding publish keys for grp " << grpId << " to sending list" << std::endl; -#endif - - return true ; -} - -void RsGxsNetService::sharePublishKeysPending() -{ - RS_STACK_MUTEX(mNxsMutex) ; - - if(mPendingPublishKeyRecipients.empty()) - return ; - -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG___ << "RsGxsNetService::sharePublishKeys() " << (void*)this << std::endl; -#endif - // get list of peers that are online - - std::set peersOnline; - std::list toDelete; - std::map >::iterator mit ; - - mNetMgr->getOnlineList(mServiceInfo.mServiceType, peersOnline); - -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG___ << " " << peersOnline.size() << " peers online." << std::endl; -#endif - /* send public key to peers online */ - - for(mit = mPendingPublishKeyRecipients.begin(); mit != mPendingPublishKeyRecipients.end(); ++mit) - { - // Compute the set of peers to send to. We start with this, to avoid retrieving the data for nothing. - - std::list recipients ; - std::set offline_recipients ; - - for(std::set::const_iterator it(mit->second.begin());it!=mit->second.end();++it) - if(peersOnline.find(*it) != peersOnline.end()) - { -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG_P_(*it) << " " << *it << ": online. Adding." << std::endl; -#endif - recipients.push_back(*it) ; - } - else - { -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG_P_(*it) << " " << *it << ": offline. Keeping for next try." << std::endl; -#endif - offline_recipients.insert(*it) ; - } - - // If empty, skip - - if(recipients.empty()) - { -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG___ << " No recipients online. Skipping." << std::endl; -#endif - continue ; - } - - // Get the meta data for this group Id - // - RsGxsMetaDataTemporaryMap grpMetaMap; - grpMetaMap[mit->first] = NULL; - mDataStore->retrieveGxsGrpMetaData(grpMetaMap); - - // Find the publish keys in the retrieved info - - RsGxsGrpMetaData *grpMeta = grpMetaMap[mit->first] ; - - if(grpMeta == NULL) - { - std::cerr << "(EE) RsGxsNetService::sharePublishKeys() Publish keys cannot be found for group " << mit->first << std::endl; - continue ; - } - - const RsTlvSecurityKeySet& keys = grpMeta->keys; - - std::map::const_iterator kit = keys.keys.begin(), kit_end = keys.keys.end(); - bool publish_key_found = false; - RsTlvSecurityKey publishKey ; - - for(; kit != kit_end && !publish_key_found; ++kit) - { - publish_key_found = (kit->second.keyFlags == (RSTLV_KEY_DISTRIB_PUBLISH | RSTLV_KEY_TYPE_FULL)); - publishKey = kit->second ; - } - - if(!publish_key_found) - { - std::cerr << "(EE) no publish key in group " << mit->first << ". Cannot share!" << std::endl; - continue ; - } - - -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG__G(grpMeta->mGroupId) << " using publish key ID=" << publishKey.keyId << ", flags=" << publishKey.keyFlags << std::endl; -#endif - for(std::list::const_iterator it(recipients.begin());it!=recipients.end();++it) - { - /* Create publish key sharing item */ - RsNxsGroupPublishKeyItem *publishKeyItem = new RsNxsGroupPublishKeyItem(mServType); - - publishKeyItem->clear(); - publishKeyItem->grpId = mit->first; - - publishKeyItem->key = publishKey ; - publishKeyItem->PeerId(*it); - - sendItem(publishKeyItem); -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG_PG(*it,grpMeta->mGroupId) << " sent key item to " << *it << std::endl; -#endif - } - - mit->second = offline_recipients ; - - // If given peers have all received key(s) then stop sending for group - if(offline_recipients.empty()) - toDelete.push_back(mit->first); - } - - // delete pending peer list which are done with - for(std::list::const_iterator lit = toDelete.begin(); lit != toDelete.end(); ++lit) - mPendingPublishKeyRecipients.erase(*lit); -} - -void RsGxsNetService::handleRecvPublishKeys(RsNxsGroupPublishKeyItem *item) -{ -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << "RsGxsNetService::sharePublishKeys() " << std::endl; -#endif - - if (!item) - return; - - RS_STACK_MUTEX(mNxsMutex) ; - -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " PeerId : " << item->PeerId() << std::endl; - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " GrpId: " << item->grpId << std::endl; - GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " Got key Item: " << item->key.keyId << std::endl; -#endif - - // Get the meta data for this group Id - // - RsGxsMetaDataTemporaryMap grpMetaMap; - grpMetaMap[item->grpId] = NULL; - - mDataStore->retrieveGxsGrpMetaData(grpMetaMap); - - // update the publish keys in this group meta info - - RsGxsGrpMetaData *grpMeta = grpMetaMap[item->grpId] ; - - // Check that the keys correspond, and that FULL keys are supplied, etc. - -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " Key received: " << std::endl; -#endif - - bool admin = (item->key.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) && (item->key.keyFlags & RSTLV_KEY_TYPE_FULL) ; - bool publi = (item->key.keyFlags & RSTLV_KEY_DISTRIB_PUBLISH) && (item->key.keyFlags & RSTLV_KEY_TYPE_FULL) ; - -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " Key id = " << item->key.keyId << " admin=" << admin << ", publish=" << publi << " ts=" << item->key.endTS << std::endl; -#endif - - if(!(!admin && publi)) - { - std::cerr << " Key is not a publish private key. Discarding!" << std::endl; - return ; - } - // Also check that we don't already have full keys for that group. - - std::map::iterator it = grpMeta->keys.keys.find(item->key.keyId) ; - - if(it == grpMeta->keys.keys.end()) - { - std::cerr << " (EE) Key not found in known group keys. This is an inconsistency." << std::endl; - return ; - } - - if((it->second.keyFlags & RSTLV_KEY_DISTRIB_PUBLISH) && (it->second.keyFlags & RSTLV_KEY_TYPE_FULL)) - { -#ifdef NXS_NET_DEBUG_3 - GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " (EE) Publish key already present in database. Discarding message." << std::endl; -#endif - return ; - } - - // Store/update the info. - - it->second = item->key ; - bool ret = mDataStore->updateGroupKeys(item->grpId,grpMeta->keys, grpMeta->mSubscribeFlags | GXS_SERV::GROUP_SUBSCRIBE_PUBLISH) ; - - if(ret) - { -#ifdef NXS_NET_DEBUG - GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " updated database with new publish keys." << std::endl; -#endif - mObserver->notifyReceivePublishKey(item->grpId); - } - else - { - std::cerr << "(EE) could not update database. Something went wrong." << std::endl; - } -} + +/* + * libretroshare/src/gxs: rsgxnetservice.cc + * + * Access to rs network and synchronisation service implementation + * + * Copyright 2012-2012 by Christopher Evi-Parker + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +// +// RsNxsItem +// | +// +-- RsNxsSyncGrp send req for group list, with time stamp of what we have +// +-- RsNxsSyncMsg +// +-- RsNxsGroupPublishKeyItem +// +-- RsNxsSyncGrpItem send individual grp info with time stamps, authors, etc. +// +// +// tick() +// | +// +----------- sharePublishKeys() +// | +// +----------- syncWithPeers() +// | | +// | +--if AutoSync--- send global UpdateTS of each peer to itself => the peer knows the last +// | | time current peer has received an updated from himself +// | | type=RsNxsSyncGrp +// | | role: advise to request grp list for mServType +// | | +// | +--Retrive all grp Id + meta +// | +// | For each peer +// | For each grp to request +// | send RsNxsSyncMsg(ServiceType, grpId, updateTS) +// | | +// | (Only send if rand() < sendingProb()) +---comes from mClientMsgUpdateMap +// | +// +----------- recvNxsItemQueue() +// | +// +------ handleRecvPublishKeys(auto*) +// | +// | +// | +// +------ handleRecvSyncGroup( RsNxsSyncGrp*) +// | - parse all subscribed groups. For each, send a RsNxsSyncGrpItem with publish TS +// | - pack into a single RsNxsTransac item +// | | +// | +---- canSendGrpId(peer, grpMeta, toVet) // determines if put in vetting list +// | | | // or sent right away +// | | +--CIRCLES_TYPE_LOCAL------- false +// | | +--CIRCLES_TYPE_PUBLIC------ true +// | | +--CIRCLES_TYPE_EXTERNAL---- mCircles->canSend(circleId, getPgpId(peerId)) +// | | +--CIRCLES_TYPE_YOUR_EYES--- internal circle stuff +// | | +// | +---- store in mPendingCircleVet ou directement locked_pushGrpRespFromList() +// | +// +------ handleRecvSyncMessage( RsNxsSyncMsg*) +// - parse msgs from group +// - send all msg IDs for this group +// data_tick() +// | +// +----------- updateServerSyncTS() +// | - retrieve all group meta data +// | - updates mServerMsgUpdateMap[grpId]=grp->mLastPostTS for all grps +// | - updates mGrpServerUpdateItem to max of all received TS +// | +// +----------- processTransactions() +// | +// +----------- processCompletedTransactions() +// | | +// | +------ locked_processCompletedIncomingTrans() +// | | | +// | | +-------- locked_genReqMsgTransaction() // request messages based on list +// | | | +// | | +-------- locked_genReqGrpTransaction() // request groups based on list +// | | | +// | | +-------- locked_genSendMsgsTransaction() // send msg list +// | | | +// | | +-------- locked_genSendGrpsTransaction() // send group list +// | | +// | +------ locked_processCompletedOutgoingTrans() +// | +// +----------- processExplicitGroupRequests() +// | - parse mExplicitRequest and for each element (containing a grpId list), +// | send the group ID (?!?!) +// | +// +----------- runVetting() +// | +// +--------- sort items from mPendingResp +// | | +// | +------ locked_createTransactionFromPending(GrpRespPending / MsgRespPending) +// | | // takes accepted transaction and adds them to the list of active trans +// | +// +--------- sort items from mPendingCircleVetting +// | +// +------ locked_createTransactionFromPending(GrpCircleIdsRequestVetting / MsgCircleIdsRequestVetting) +// // takes accepted transaction and adds them to the list of active trans +// +// Objects for time stamps +// ======================= +// +// mClientGrpUpdateMap: map< RsPeerId, TimeStamp > Time stamp of last modification of group data for that peer (in peer's clock time!) +// (Set at server side to be mGrpServerUpdateItem->grpUpdateTS) +// +// Only updated in processCompletedIncomingTransaction() from Grp list transaction. +// Used in syncWithPeers() sending in RsNxsSyncGrp once to all peers: peer will send data if +// has something new. All time comparisons are in the friends' clock time. +// +// mClientMsgUpdateMap: map< RsPeerId, map > +// +// Last msg list modification time sent by that peer Id +// Updated in processCompletedIncomingTransaction() from Grp list trans. +// Used in syncWithPeers() sending in RsNxsSyncGrp once to all peers. +// Set at server to be mServerMsgUpdateMap[grpId]->msgUpdateTS +// +// mGrpServerUpdateItem: TimeStamp Last group local modification timestamp over all groups +// +// mServerMsgUpdateMap: map< GrpId, TimeStamp > Timestamp local modification for each group (i.e. time of most recent msg / metadata update) +// +// +// Group update algorithm +// ====================== +// +// CLient Server +// ====== ====== +// +// tick() tick() +// | | +// +---- SyncWithPeers +-- recvNxsItemQueue() +// | | +// +---------------- Send global UpdateTS of each peer to itself => the peer knows +---------> +------ handleRecvSyncGroup( RsNxsSyncGrp*) +// | the last msg sent (stored in mClientGrpUpdateMap[peer_id]), | | - parse all subscribed groups. For each, send a RsNxsSyncGrpItem with publish TS +// | type=RsNxsSyncGrp | | - pack into a single RsNxsTransac item +// | role: advise to request grp list for mServType -------------------+ | +// | +--> +------ handleRecvSyncMessage( RsNxsSyncMsg*) +// +---------------- Retrieve all grp Id + meta | - parse msgs from group +// | | - send all msg IDs for this group +// +-- For each peer | +// For each grp to request | +// send RsNxsSyncMsg(ServiceType, grpId, updateTS) | +// | | +// (Only send if rand() < sendingProb()) +---comes from mClientMsgUpdateMap -----+ +// +// Suggestions +// =========== +// * handleRecvSyncGroup should use mit->second.mLastPost to limit the sending of already known data +// X * apparently mServerMsgUpdateMap is initially empty -> by default clients will always want to receive the data. +// => new peers will always send data for each group until they get an update for that group. +// X * check that there is a timestamp for unsubscribed items, otherwise we always send TS=0 and we always get them!! (in 346) +// +// -> there is not. mClientMsgUpdateMap is updated when msgs are received. +// -> 1842: leaves before asking for msg content. +// +// Proposed changes: +// - for unsubsribed groups, mClientMsgUpdateMap[peerid][grpId]=now when the group list is received => wont be asked again +// - when we subscribe, we reset the time stamp. +// +// Better change: +// - each peer sends last +// +// * the last TS method is not perfect: do new peers always receive old messages? +// +// * there's double information between mServerMsgUpdateMap first element (groupId) and second->grpId +// * processExplicitGroupRequests() seems to send the group list that it was asked for without further information. How is that useful??? +// +// * grps without messages will never be stamped because stamp happens in genReqMsgTransaction, after checking msgListL.empty() +// Problem: without msg, we cannot know the grpId!! +// +// * mClientMsgUpdateMap[peerid][grpId] is only updated when new msgs are received. Up to date groups will keep asking for lists! + + +#include +#include +#include +#include + +#include "rsgxsnetservice.h" +#include "retroshare/rsconfig.h" +#include "retroshare/rsreputations.h" +#include "retroshare/rsgxsflags.h" +#include "retroshare/rsgxscircles.h" +#include "pgp/pgpauxutils.h" + +/*** + * Use the following defines to debug: + NXS_NET_DEBUG_0 shows group update high level information + NXS_NET_DEBUG_1 shows group update low level info (including transaction details) + NXS_NET_DEBUG_2 bandwidth information + NXS_NET_DEBUG_3 publish key exchange + NXS_NET_DEBUG_4 vetting + NXS_NET_DEBUG_5 summary of transactions (useful to just know what comes in/out) + ***/ +//#define NXS_NET_DEBUG_0 1 +//#define NXS_NET_DEBUG_1 1 +//#define NXS_NET_DEBUG_2 1 +//#define NXS_NET_DEBUG_3 1 +//#define NXS_NET_DEBUG_4 1 +//#define NXS_NET_DEBUG_5 1 +//#define NXS_NET_DEBUG_6 1 + +#define GIXS_CUT_OFF 0 + +// The constant below have a direct influence on how fast forums/channels/posted/identity groups propagate and on the overloading of queues: +// +// Channels/forums will update at a rate of SYNC_PERIOD*MAX_REQLIST_SIZE/60 messages per minute. +// A large TRANSAC_TIMEOUT helps large transactions to finish before anything happens (e.g. disconnexion) or when the server has low upload bandwidth, +// but also uses more memory. +// A small value for MAX_REQLIST_SIZE is likely to help messages to propagate in a chaotic network, but will also slow them down. +// A small SYNC_PERIOD fasten message propagation, but is likely to overload the server side of transactions (e.g. overload outqueues). +// +#define SYNC_PERIOD 60 +#define MAX_REQLIST_SIZE 20 // No more than 20 items per msg request list => creates smaller transactions that are less likely to be cancelled. +#define TRANSAC_TIMEOUT 2000 // In seconds. Has been increased to avoid epidemic transaction cancelling due to overloaded outqueues. +#define SECURITY_DELAY_TO_FORCE_CLIENT_REUPDATE 3600 // force re-update if there happens to be a large delay between our server side TS and the client side TS of friends +#define REJECTED_MESSAGE_RETRY_DELAY 24*3600 // re-try rejected messages every 24hrs. Most of the time this is because the peer's reputation has changed. +#define GROUP_STATS_UPDATE_DELAY 1800 // update unsubscribed group statistics every 30 mins +#define GROUP_STATS_UPDATE_NB_PEERS 2 // update unsubscribed group statistics every 30 mins + +// Debug system to allow to print only for some IDs (group, Peer, etc) + +#if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2) || defined(NXS_NET_DEBUG_3) || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6) + +static const RsPeerId peer_to_print = RsPeerId(std::string("")) ; +static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("" )) ; // use this to allow to this group id only, or "" for all IDs +static const uint32_t service_to_print = 0 ; // use this to allow to this service id only, or 0 for all services + // warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums) + +class nullstream: public std::ostream {}; + +#if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2) || defined(NXS_NET_DEBUG_3) || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5)|| defined(NXS_NET_DEBUG_6) +static std::string nice_time_stamp(time_t now,time_t TS) +{ + if(TS == 0) + return "Never" ; + else + { + std::ostringstream s; + s << now - TS << " secs ago" ; + return s.str() ; + } +} +#endif + +static std::ostream& gxsnetdebug(const RsPeerId& peer_id,const RsGxsGroupId& grp_id,uint32_t service_type) +{ + static nullstream null ; + + if((peer_to_print.isNull() || peer_id.isNull() || peer_id == peer_to_print) + && (group_id_to_print.isNull() || grp_id.isNull() || grp_id == group_id_to_print) + && (service_to_print==0 || service_type == 0 || ((service_type >> 8)&0xffff) == service_to_print)) + return std::cerr << time(NULL) << ": " ; + else + return null ; +} + +#define GXSNETDEBUG___ gxsnetdebug(RsPeerId(),RsGxsGroupId(),mServiceInfo.mServiceType) +#define GXSNETDEBUG_P_(peer_id ) gxsnetdebug(peer_id ,RsGxsGroupId(),mServiceInfo.mServiceType) +#define GXSNETDEBUG__G( group_id) gxsnetdebug(RsPeerId(),group_id ,mServiceInfo.mServiceType) +#define GXSNETDEBUG_PG(peer_id,group_id) gxsnetdebug(peer_id ,group_id ,mServiceInfo.mServiceType) + +#endif + +const uint32_t RsGxsNetService::FRAGMENT_SIZE = 150000; + +RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, + RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, + const RsServiceInfo serviceInfo, + RsGixsReputation* reputations, RsGcxs* circles, + PgpAuxUtils *pgpUtils, bool grpAutoSync,bool msgAutoSync) + : p3ThreadedService(), p3Config(), mTransactionN(0), + mObserver(nxsObs), mDataStore(gds), mServType(servType), + mTransactionTimeOut(TRANSAC_TIMEOUT), mNetMgr(netMgr), mNxsMutex("RsGxsNetService"), + mSyncTs(0), mLastKeyPublishTs(0),mLastCleanRejectedMessages(0), mSYNC_PERIOD(SYNC_PERIOD), mCircles(circles), mReputations(reputations), + mPgpUtils(pgpUtils), + mGrpAutoSync(grpAutoSync),mAllowMsgSync(msgAutoSync), mGrpServerUpdateItem(NULL), + mServiceInfo(serviceInfo) + +{ + addSerialType(new RsNxsSerialiser(mServType)); + mOwnId = mNetMgr->getOwnId(); + mUpdateCounter = 0; +} + +RsGxsNetService::~RsGxsNetService() +{ + RS_STACK_MUTEX(mNxsMutex) ; + + for(TransactionsPeerMap::iterator it = mTransactions.begin();it!=mTransactions.end();++it) + { + for(TransactionIdMap::iterator it2 = it->second.begin();it2!=it->second.end();++it2) + delete it2->second ; + + it->second.clear() ; + } + mTransactions.clear() ; + + delete mGrpServerUpdateItem ; + + for(ClientGrpMap::iterator it = mClientGrpUpdateMap.begin();it!=mClientGrpUpdateMap.end();++it) + delete it->second ; + + mClientGrpUpdateMap.clear() ; + + for(std::map::iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();) + delete it->second ; + + mServerMsgUpdateMap.clear() ; +} + + +int RsGxsNetService::tick() +{ + // always check for new items arriving + // from peers + if(receivedItems()) + recvNxsItemQueue(); + + bool should_notify = false; + + { + RS_STACK_MUTEX(mNxsMutex) ; + + should_notify = should_notify || !mNewGroupsToNotify.empty() ; + should_notify = should_notify || !mNewMessagesToNotify.empty() ; + } + + if(should_notify) + processObserverNotifications() ; + + time_t now = time(NULL); + time_t elapsed = mSYNC_PERIOD + mSyncTs; + + if((elapsed) < now) + { + syncWithPeers(); + syncGrpStatistics(); + + mSyncTs = now; + } + + if(now > 10 + mLastKeyPublishTs) + { + sharePublishKeysPending() ; + + mLastKeyPublishTs = now ; + } + + if(now > 3600 + mLastCleanRejectedMessages) + { + mLastCleanRejectedMessages = now ; + cleanRejectedMessages() ; + } + return 1; +} + +void RsGxsNetService::processObserverNotifications() +{ +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG___ << "Processing observer notification." << std::endl; +#endif + + std::vector grps_copy ; + std::vector msgs_copy ; + + { + RS_STACK_MUTEX(mNxsMutex) ; + + grps_copy = mNewGroupsToNotify ; + msgs_copy = mNewMessagesToNotify ; + + mNewGroupsToNotify.clear() ; + mNewMessagesToNotify.clear() ; + } + + mObserver->notifyNewGroups(grps_copy); + mObserver->notifyNewMessages(msgs_copy); +} + +void RsGxsNetService::rejectMessage(const RsGxsMessageId& msg_id) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + mRejectedMessages[msg_id] = time(NULL) ; +} +void RsGxsNetService::cleanRejectedMessages() +{ + RS_STACK_MUTEX(mNxsMutex) ; + time_t now = time(NULL) ; + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG___ << "Cleaning rejected messages." << std::endl; +#endif + + for(std::map::iterator it(mRejectedMessages.begin());it!=mRejectedMessages.end();) + if(it->second + REJECTED_MESSAGE_RETRY_DELAY < now) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG___ << " message id " << it->first << " should be re-tried. removing from list..." << std::endl; +#endif + + std::map::iterator tmp = it ; + ++tmp ; + mRejectedMessages.erase(it) ; + it=tmp ; + } + else + ++it ; +} + +// This class collects outgoing items due to the broadcast of Nxs messages. It computes +// a probability that can be used to temper the broadcast of items so as to match the +// residual bandwidth (difference between max allowed bandwidth and current outgoing rate. + +class NxsBandwidthRecorder +{ +public: + static const int OUTQUEUE_CUTOFF_VALUE = 500 ; + static const int BANDWIDTH_ESTIMATE_DELAY = 20 ; + + static void recordEvent(uint16_t service_type, RsItem *item) + { + RS_STACK_MUTEX(mtx) ; + + uint32_t bw = RsNxsSerialiser(service_type).size(item) ; // this is used to estimate bandwidth. + timeval tv ; + gettimeofday(&tv,NULL) ; + + // compute time(NULL) in msecs, for a more accurate bw estimate. + + uint64_t now = (uint64_t) tv.tv_sec * 1000 + tv.tv_usec/1000 ; + + total_record += bw ; + ++total_events ; + +#ifdef NXS_NET_DEBUG_2 + std::cerr << "bandwidthRecorder::recordEvent() Recording event time=" << now << ". bw=" << bw << std::endl; +#endif + + // Every 20 seconds at min, compute a new estimate of the required bandwidth. + + if(now > last_event_record + BANDWIDTH_ESTIMATE_DELAY*1000) + { + // Compute the bandwidth using recorded times, in msecs + float speed = total_record/1024.0f/(now - last_event_record)*1000.0f ; + + // Apply a small temporal convolution. + estimated_required_bandwidth = 0.75*estimated_required_bandwidth + 0.25 * speed ; + +#ifdef NXS_NET_DEBUG_2 + std::cerr << std::dec << " " << total_record << " Bytes (" << total_events << " items)" + << " received in " << now - last_event_record << " seconds. Speed: " << speed << " KBytes/sec" << std::endl; + std::cerr << " instantaneous speed = " << speed << " KB/s" << std::endl; + std::cerr << " cumulated estimated = " << estimated_required_bandwidth << " KB/s" << std::endl; +#endif + + last_event_record = now ; + total_record = 0 ; + total_events = 0 ; + } + } + + // Estimate the probability of sending an item so that the expected bandwidth matches the residual bandwidth + + static float computeCurrentSendingProbability() + { + int maxIn=50,maxOut=50; + float currIn=0,currOut=0 ; + + rsConfig->GetMaxDataRates(maxIn,maxOut) ; + rsConfig->GetCurrentDataRates(currIn,currOut) ; + + RsConfigDataRates rates ; + rsConfig->getTotalBandwidthRates(rates) ; + +#ifdef NXS_NET_DEBUG_2 + std::cerr << std::dec << std::endl; +#endif + + float outqueue_factor = 1.0f/pow( std::max(0.02f,rates.mQueueOut / (float)OUTQUEUE_CUTOFF_VALUE),5.0f) ; + float accepted_bandwidth = std::max( 0.0f, maxOut - currOut) ; + float max_bandwidth_factor = std::min( accepted_bandwidth / estimated_required_bandwidth,1.0f ) ; + + // We account for two things here: + // 1 - the required max bandwidth + // 2 - the current network overload, measured from the size of the outqueues. + // + // Only the later can limit the traffic if the internet connexion speed is responsible for outqueue overloading. + + float sending_probability = std::min(outqueue_factor,max_bandwidth_factor) ; + +#ifdef NXS_NET_DEBUG_2 + std::cerr << "bandwidthRecorder::computeCurrentSendingProbability()" << std::endl; + std::cerr << " current required bandwidth : " << estimated_required_bandwidth << " KB/s" << std::endl; + std::cerr << " max_bandwidth_factor : " << max_bandwidth_factor << std::endl; + std::cerr << " outqueue size : " << rates.mQueueOut << ", factor=" << outqueue_factor << std::endl; + std::cerr << " max out : " << maxOut << ", currOut=" << currOut << std::endl; + std::cerr << " computed probability : " << sending_probability << std::endl; +#endif + + return sending_probability ; + } + +private: + static RsMutex mtx; + static uint64_t last_event_record ; + static float estimated_required_bandwidth ; + static uint32_t total_events ; + static uint64_t total_record ; +}; + +uint32_t NxsBandwidthRecorder::total_events =0 ; // total number of events. Not used. +uint64_t NxsBandwidthRecorder::last_event_record = time(NULL) * 1000;// starting time of bw estimate period (in msec) +uint64_t NxsBandwidthRecorder::total_record =0 ; // total bytes recorded in the current time frame +float NxsBandwidthRecorder::estimated_required_bandwidth = 10.0f ;// Estimated BW for sending sync data. Set to 10KB/s, to avoid 0. +RsMutex NxsBandwidthRecorder::mtx("Bandwidth recorder") ; // Protects the recorder since bw events are collected from multiple GXS Net services + +// temporary holds a map of pointers to class T, and destroys all pointers on delete. + +template +class RsGxsMetaDataTemporaryMap: public std::map +{ +public: + virtual ~RsGxsMetaDataTemporaryMap() + { + clear() ; + } + + virtual void clear() + { + for(typename RsGxsMetaDataTemporaryMap::iterator it = this->begin();it!=this->end();++it) + if(it->second != NULL) + delete it->second ; + + std::map::clear() ; + } +}; + +void RsGxsNetService::syncWithPeers() +{ +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG___ << "RsGxsNetService::syncWithPeers() this=" << (void*)this << ". serviceInfo=" << mServiceInfo << std::endl; +#endif + + static RsNxsSerialiser ser(mServType) ; // this is used to estimate bandwidth. + + RS_STACK_MUTEX(mNxsMutex) ; + + std::set peers; + mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers); + if (peers.empty()) { + // nothing to do + return; + } + + std::set::iterator sit = peers.begin(); + + // for now just grps + for(; sit != peers.end(); ++sit) + { + + const RsPeerId peerId = *sit; + + ClientGrpMap::const_iterator cit = mClientGrpUpdateMap.find(peerId); + uint32_t updateTS = 0; + if(cit != mClientGrpUpdateMap.end()) + { + const RsGxsGrpUpdateItem *gui = cit->second; + updateTS = gui->grpUpdateTS; + } + RsNxsSyncGrp *grp = new RsNxsSyncGrp(mServType); + grp->clear(); + grp->PeerId(*sit); + grp->updateTS = updateTS; + + NxsBandwidthRecorder::recordEvent(mServType,grp) ; + +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_P_(*sit) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global group TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) to himself" << std::endl; +#endif + sendItem(grp); + } + + if(!mAllowMsgSync) + return ; + +#ifndef GXS_DISABLE_SYNC_MSGS + + typedef RsGxsMetaDataTemporaryMap GrpMetaMap; + GrpMetaMap grpMeta; + + mDataStore->retrieveGxsGrpMetaData(grpMeta); + + GrpMetaMap toRequest; + + for(GrpMetaMap::iterator mit = grpMeta.begin(); mit != grpMeta.end(); ++mit) + { + RsGxsGrpMetaData* meta = mit->second; + + // This was commented out because we want to know how many messages are available for unsubscribed groups. + + if(meta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) + { + toRequest.insert(std::make_pair(mit->first, meta)); + mit->second = NULL ; // avoids destruction ;-) + } + } + + sit = peers.begin(); + + float sending_probability = NxsBandwidthRecorder::computeCurrentSendingProbability() ; +#ifdef NXS_NET_DEBUG_2 + std::cerr << " syncWithPeers(): Sending probability = " << sending_probability << std::endl; +#endif + + // Synchronise group msg for groups which we're subscribed to + // For each peer and each group, we send to the peer the time stamp of the most + // recent modification the peer has sent. If the peer has more recent messages he will send them, because its latest + // modifications will be more recent. This ensures that we always compare timestamps all taken in the same + // computer (the peer's computer in this case) + + for(; sit != peers.end(); ++sit) + { + const RsPeerId& peerId = *sit; + + + // now see if you have an updateTS so optimise whether you need + // to get a new list of peer data + RsGxsMsgUpdateItem* mui = NULL; + + ClientMsgMap::const_iterator cit = mClientMsgUpdateMap.find(peerId); + + if(cit != mClientMsgUpdateMap.end()) + mui = cit->second; + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(peerId) << " syncing messages with peer " << peerId << std::endl; +#endif + + GrpMetaMap::const_iterator mmit = toRequest.begin(); + for(; mmit != toRequest.end(); ++mmit) + { + const RsGxsGrpMetaData* meta = mmit->second; + const RsGxsGroupId& grpId = mmit->first; + + if(!checkCanRecvMsgFromPeer(peerId, *meta)) + continue; + + // On default, the info has never been received so the TS is 0, meaning the peer has sent that it had no information. + + uint32_t updateTS = 0; + + if(mui) + { + std::map::const_iterator cit2 = mui->msgUpdateInfos.find(grpId); + + if(cit2 != mui->msgUpdateInfos.end()) + updateTS = cit2->second.time_stamp; + } + + RsNxsSyncMsg* msg = new RsNxsSyncMsg(mServType); + msg->clear(); + msg->PeerId(peerId); + msg->grpId = grpId; + msg->updateTS = updateTS; + + NxsBandwidthRecorder::recordEvent(mServType,msg) ; + + if(RSRandom::random_f32() < sending_probability) + { + sendItem(msg); +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_PG(*sit,grpId) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself" << std::endl; +#endif + } + else + { + delete msg ; +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(*sit,grpId) << " cancel RsNxsSyncMsg req (last local update TS for group+peer) for grpId=" << grpId << " to peer " << *sit << ": not enough bandwidth." << std::endl; +#endif + } + } + } + +#endif +} + +void RsGxsNetService::syncGrpStatistics() +{ + RS_STACK_MUTEX(mNxsMutex) ; + +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG___<< "Sync-ing group statistics." << std::endl; +#endif + RsGxsMetaDataTemporaryMap grpMeta; + + mDataStore->retrieveGxsGrpMetaData(grpMeta); + + std::set online_peers; + mNetMgr->getOnlineList(mServiceInfo.mServiceType, online_peers); + + // Go through group statistics and groups without information are re-requested to random peers selected + // among the ones who provided the group info. + + time_t now = time(NULL) ; + + for(std::map::const_iterator it(grpMeta.begin());it!=grpMeta.end();++it) + { + RsGroupNetworkStatsRecord& rec(mGroupNetworkStats[it->first]) ; +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG__G(it->first) << " group " << it->first ; +#endif + + if(rec.update_TS + GROUP_STATS_UPDATE_DELAY < now && rec.suppliers.size() > 0) + { +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG__G(it->first) << " needs update. Randomly asking to some friends" << std::endl; +#endif + // randomly select GROUP_STATS_UPDATE_NB_PEERS friends among the suppliers of this group + + uint32_t n = RSRandom::random_u32() % rec.suppliers.size() ; + + std::set::const_iterator rit = rec.suppliers.begin(); + for(uint32_t i=0;ifirst) << " asking friend " << peer_id << " for an update of stats for group " << it->first << std::endl; +#endif + + RsNxsSyncGrpStats *grs = new RsNxsSyncGrpStats(mServType) ; + + grs->request_type = RsNxsSyncGrpStats::GROUP_INFO_TYPE_REQUEST ; + grs->grpId = it->first ; + grs->PeerId(peer_id) ; + + sendItem(grs) ; + } + } + } +#ifdef NXS_NET_DEBUG_6 + else + GXSNETDEBUG__G(it->first) << " up to date." << std::endl; +#endif + } +} + +void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStats *grs) +{ + if(grs->request_type == RsNxsSyncGrpStats::GROUP_INFO_TYPE_REQUEST) + { +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << "Received Grp update stats Request for group " << grs->grpId << " from friend " << grs->PeerId() << std::endl; +#endif + RsGxsMetaDataTemporaryMap grpMetas; + grpMetas[grs->grpId] = NULL; + + mDataStore->retrieveGxsGrpMetaData(grpMetas); + + RsGxsGrpMetaData* grpMeta = grpMetas[grs->grpId]; + + if(grpMeta == NULL) + { +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " Group is unknown. Not reponding." << std::endl; +#endif + return ; + } + + // check if we're subscribed or not + + if(! (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )) + { +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " Group is not subscribed. Not reponding." << std::endl; +#endif + return ; + } + + // now count available messages + + GxsMsgReq reqIds; + reqIds[grs->grpId] = std::vector(); + GxsMsgMetaResult result; + +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " retrieving message information." << std::endl; +#endif + mDataStore->retrieveGxsMsgMetaData(reqIds, result); + + const std::vector& vec(result[grs->grpId]) ; + + if(vec.empty()) // that means we don't have any, or there isn't any, but since the default is always 0, no need to send. + return ; + + RsNxsSyncGrpStats *grs_resp = new RsNxsSyncGrpStats(mServType) ; + grs_resp->request_type = RsNxsSyncGrpStats::GROUP_INFO_TYPE_RESPONSE ; + grs_resp->number_of_posts = vec.size(); + grs_resp->grpId = grs->grpId; + grs_resp->PeerId(grs->PeerId()) ; + + grs_resp->last_post_TS = 0 ; + + for(uint32_t i=0;ilast_post_TS < vec[i]->mPublishTs) + grs_resp->last_post_TS = vec[i]->mPublishTs; + + delete vec[i] ; + } +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " sending back statistics item with " << vec.size() << " elements." << std::endl; +#endif + + sendItem(grs_resp) ; + } + else if(grs->request_type == RsNxsSyncGrpStats::GROUP_INFO_TYPE_RESPONSE) + { +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << "Received Grp update stats item from peer " << grs->PeerId() << " for group " << grs->grpId << ", reporting " << grs->number_of_posts << " posts." << std::endl; +#endif + RS_STACK_MUTEX(mNxsMutex) ; + RsGroupNetworkStatsRecord& rec(mGroupNetworkStats[grs->grpId]) ; + + int32_t old_count = rec.max_visible_count ; + int32_t old_suppliers_count = rec.suppliers.size() ; + + rec.suppliers.insert(grs->PeerId()) ; + rec.max_visible_count = std::max(rec.max_visible_count,grs->number_of_posts) ; + rec.update_TS = time(NULL) ; + + if (old_count != rec.max_visible_count || old_suppliers_count != rec.suppliers.size()) + mObserver->notifyChangedGroupStats(grs->grpId); + } + else + std::cerr << "(EE) RsGxsNetService::handleRecvSyncGrpStatistics(): unknown item type " << grs->request_type << " found. This is a bug." << std::endl; +} + +void RsGxsNetService::subscribeStatusChanged(const RsGxsGroupId& grpId,bool subscribed) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + if(!subscribed) + return ; + + // When we subscribe, we reset the time stamps, so that the entire group list + // gets requested once again, for a proper update. + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG__G(grpId) << "Changing subscribe status for grp " << grpId << " to " << subscribed << ": reseting all server msg time stamps for this group, and server global TS." << std::endl; +#endif + std::map::iterator it = mServerMsgUpdateMap.find(grpId) ; + + if(mServerMsgUpdateMap.end() == it) + { + RsGxsServerMsgUpdateItem *item = new RsGxsServerMsgUpdateItem(mServType) ; + item->grpId = grpId ; + item->msgUpdateTS = 0 ; + } + else + it->second->msgUpdateTS = 0 ; // reset! + + // no need to update mGrpServerUpdateItem since the ::updateServerSyncTS() call will do it. +} + +bool RsGxsNetService::fragmentMsg(RsNxsMsg& msg, MsgFragments& msgFragments) const +{ + // first determine how many fragments + uint32_t msgSize = msg.msg.TlvSize(); + uint32_t dataLeft = msgSize; + uint8_t nFragments = ceil(float(msgSize)/FRAGMENT_SIZE); + char buffer[FRAGMENT_SIZE]; + int currPos = 0; + + + for(uint8_t i=0; i < nFragments; ++i) + { + RsNxsMsg* msgFrag = new RsNxsMsg(mServType); + msgFrag->grpId = msg.grpId; + msgFrag->msgId = msg.msgId; + msgFrag->meta = msg.meta; + msgFrag->transactionNumber = msg.transactionNumber; + msgFrag->pos = i; + msgFrag->PeerId(msg.PeerId()); + msgFrag->count = nFragments; + uint32_t fragSize = std::min(dataLeft, FRAGMENT_SIZE); + + memcpy(buffer, ((char*)msg.msg.bin_data) + currPos, fragSize); + msgFrag->msg.setBinData(buffer, fragSize); + + currPos += fragSize; + dataLeft -= fragSize; + msgFragments.push_back(msgFrag); + } + + return true; +} + +bool RsGxsNetService::fragmentGrp(RsNxsGrp& grp, GrpFragments& grpFragments) const +{ + // first determine how many fragments + uint32_t grpSize = grp.grp.TlvSize(); + uint32_t dataLeft = grpSize; + uint8_t nFragments = ceil(float(grpSize)/FRAGMENT_SIZE); + char buffer[FRAGMENT_SIZE]; + int currPos = 0; + + + for(uint8_t i=0; i < nFragments; ++i) + { + RsNxsGrp* grpFrag = new RsNxsGrp(mServType); + grpFrag->grpId = grp.grpId; + grpFrag->meta = grp.meta; + grpFrag->pos = i; + grpFrag->count = nFragments; + uint32_t fragSize = std::min(dataLeft, FRAGMENT_SIZE); + + memcpy(buffer, ((char*)grp.grp.bin_data) + currPos, fragSize); + grpFrag->grp.setBinData(buffer, fragSize); + + currPos += fragSize; + dataLeft -= fragSize; + grpFragments.push_back(grpFrag); + } + + return true; +} + +RsNxsMsg* RsGxsNetService::deFragmentMsg(MsgFragments& msgFragments) const +{ + if(msgFragments.empty()) return NULL; + + // if there is only one fragment with a count 1 or less then + // the fragment is the msg + if(msgFragments.size() == 1) + { + RsNxsMsg* m = msgFragments.front(); + if(m->count > 1) + return NULL; + else + return m; + } + + // first determine total size for binary data + MsgFragments::iterator mit = msgFragments.begin(); + uint32_t datSize = 0; + + for(; mit != msgFragments.end(); ++mit) + datSize += (*mit)->msg.bin_len; + + char* data = new char[datSize]; + uint32_t currPos = 0; + + for(mit = msgFragments.begin(); mit != msgFragments.end(); ++mit) + { + RsNxsMsg* msg = *mit; + memcpy(data + (currPos), msg->msg.bin_data, msg->msg.bin_len); + currPos += msg->msg.bin_len; + } + + RsNxsMsg* msg = new RsNxsMsg(mServType); + const RsNxsMsg& m = *(*(msgFragments.begin())); + msg->msg.setBinData(data, datSize); + msg->msgId = m.msgId; + msg->grpId = m.grpId; + msg->transactionNumber = m.transactionNumber; + msg->meta = m.meta; + + delete[] data; + return msg; +} + +RsNxsGrp* RsGxsNetService::deFragmentGrp(GrpFragments& grpFragments) const +{ + if(grpFragments.empty()) return NULL; + + // first determine total size for binary data + GrpFragments::iterator mit = grpFragments.begin(); + uint32_t datSize = 0; + + for(; mit != grpFragments.end(); ++mit) + datSize += (*mit)->grp.bin_len; + + char* data = new char[datSize]; + uint32_t currPos = 0; + + for(mit = grpFragments.begin(); mit != grpFragments.end(); ++mit) + { + RsNxsGrp* grp = *mit; + memcpy(data + (currPos), grp->grp.bin_data, grp->grp.bin_len); + currPos += grp->grp.bin_len; + } + + RsNxsGrp* grp = new RsNxsGrp(mServType); + const RsNxsGrp& g = *(*(grpFragments.begin())); + grp->grp.setBinData(data, datSize); + grp->grpId = g.grpId; + grp->transactionNumber = g.transactionNumber; + grp->meta = g.meta; + + delete[] data; + + return grp; +} + +struct GrpFragCollate +{ + RsGxsGroupId mGrpId; + GrpFragCollate(const RsGxsGroupId& grpId) : mGrpId(grpId){ } + bool operator()(RsNxsGrp* grp) { return grp->grpId == mGrpId;} +}; + +void RsGxsNetService::locked_createTransactionFromPending( MsgRespPending* msgPend) +{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(msgPend->mPeerId) << "locked_createTransactionFromPending()" << std::endl; +#endif + MsgAuthorV::const_iterator cit = msgPend->mMsgAuthV.begin(); + std::list reqList; + uint32_t transN = locked_getTransactionId(); + for(; cit != msgPend->mMsgAuthV.end(); ++cit) + { + const MsgAuthEntry& entry = *cit; + + if(entry.mPassedVetting) + { + RsNxsSyncMsgItem* msgItem = new RsNxsSyncMsgItem(mServType); + msgItem->grpId = entry.mGrpId; + msgItem->msgId = entry.mMsgId; + msgItem->authorId = entry.mAuthorId; + msgItem->flag = RsNxsSyncMsgItem::FLAG_REQUEST; + msgItem->transactionNumber = transN; + msgItem->PeerId(msgPend->mPeerId); + reqList.push_back(msgItem); + } +#ifdef NXS_NET_DEBUG_1 + else + GXSNETDEBUG_PG(msgPend->mPeerId,entry.mGrpId) << " entry failed vetting: grpId=" << entry.mGrpId << ", msgId=" << entry.mMsgId << ", peerId=" << msgPend->mPeerId << std::endl; +#endif + } + + if(!reqList.empty()) + locked_pushMsgTransactionFromList(reqList, msgPend->mPeerId, transN); +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(msgPend->mPeerId) << " added " << reqList.size() << " items to transaction." << std::endl; +#endif +} + +void RsGxsNetService::locked_createTransactionFromPending(GrpRespPending* grpPend) +{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(grpPend->mPeerId) << "locked_createTransactionFromPending() from peer " << grpPend->mPeerId << std::endl; +#endif + GrpAuthorV::const_iterator cit = grpPend->mGrpAuthV.begin(); + std::list reqList; + uint32_t transN = locked_getTransactionId(); + for(; cit != grpPend->mGrpAuthV.end(); ++cit) + { + const GrpAuthEntry& entry = *cit; + + if(entry.mPassedVetting) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(grpPend->mPeerId,entry.mGrpId) << " entry Group Id: " << entry.mGrpId << " PASSED" << std::endl; +#endif + RsNxsSyncGrpItem* msgItem = new RsNxsSyncGrpItem(mServType); + msgItem->grpId = entry.mGrpId; + msgItem->authorId = entry.mAuthorId; + msgItem->flag = RsNxsSyncMsgItem::FLAG_REQUEST; + msgItem->transactionNumber = transN; + msgItem->PeerId(grpPend->mPeerId); + reqList.push_back(msgItem); + } +#ifdef NXS_NET_DEBUG_1 + else + GXSNETDEBUG_PG(grpPend->mPeerId,entry.mGrpId) << " entry failed vetting: grpId=" << entry.mGrpId << ", peerId=" << grpPend->mPeerId << std::endl; +#endif + } + + if(!reqList.empty()) + locked_pushGrpTransactionFromList(reqList, grpPend->mPeerId, transN); +} + + +void RsGxsNetService::locked_createTransactionFromPending(GrpCircleIdRequestVetting* grpPend) +{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(grpPend->mPeerId) << "locked_createTransactionFromPending(GrpCircleIdReq)" << std::endl; +#endif + std::vector::iterator cit = grpPend->mGrpCircleV.begin(); + uint32_t transN = locked_getTransactionId(); + std::list itemL; + for(; cit != grpPend->mGrpCircleV.end(); ++cit) + { + const GrpIdCircleVet& entry = *cit; + if(entry.mCleared) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(grpPend->mPeerId,entry.mGroupId) << " Group Id: " << entry.mGroupId << " PASSED" << std::endl; +#endif + RsNxsSyncGrpItem* gItem = new RsNxsSyncGrpItem(mServType); + gItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE; + gItem->grpId = entry.mGroupId; + gItem->publishTs = 0; + gItem->PeerId(grpPend->mPeerId); + gItem->transactionNumber = transN; + gItem->authorId = entry.mAuthorId; + // why it authorId not set here??? + itemL.push_back(gItem); + } +#ifdef NXS_NET_DEBUG_1 + else + GXSNETDEBUG_PG(grpPend->mPeerId,entry.mGroupId) << " Group Id: " << entry.mGroupId << " FAILED" << std::endl; +#endif + } + + if(!itemL.empty()) + locked_pushGrpRespFromList(itemL, grpPend->mPeerId, transN); +} + +void RsGxsNetService::locked_createTransactionFromPending(MsgCircleIdsRequestVetting* msgPend) +{ + std::vector::iterator vit = msgPend->mMsgs.begin(); + std::list itemL; + + uint32_t transN = locked_getTransactionId(); + RsGxsGroupId grp_id ; + + for(; vit != msgPend->mMsgs.end(); ++vit) + { + MsgIdCircleVet& mic = *vit; + RsNxsSyncMsgItem* mItem = new + RsNxsSyncMsgItem(mServType); + mItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE; + mItem->grpId = msgPend->mGrpId; + mItem->msgId = mic.mMsgId; + mItem->authorId = mic.mAuthorId; + mItem->PeerId(msgPend->mPeerId); + mItem->transactionNumber = transN; + itemL.push_back(mItem); + + grp_id = msgPend->mGrpId ; + } + + if(!itemL.empty()) + locked_pushMsgRespFromList(itemL, msgPend->mPeerId,grp_id, transN); +} + +/*bool RsGxsNetService::locked_canReceive(const RsGxsGrpMetaData * const grpMeta + , const RsPeerId& peerId ) +{ + + double timeDelta = 0.2; + + if(grpMeta->mCircleType == GXS_CIRCLE_TYPE_EXTERNAL) { + int i=0; + mCircles->loadCircle(grpMeta->mCircleId); + + // check 5 times at most + // spin for 1 second at most + while(i < 5) { + + if(mCircles->isLoaded(grpMeta->mCircleId)) { + const RsPgpId& pgpId = mPgpUtils->getPGPId(peerId); + return mCircles->canSend(grpMeta->mCircleId, pgpId); + }//if(mCircles->isLoaded(grpMeta->mCircleId)) + + usleep((int) (timeDelta * 1000 * 1000));// timeDelta sec + i++; + }//while(i < 5) + + } else {//if(grpMeta->mCircleType == GXS_CIRCLE_TYPE_EXTERNAL) + return true; + }//else (grpMeta->mCircleType == GXS_CIRCLE_TYPE_EXTERNAL) + + return false; +}*/ + +void RsGxsNetService::collateGrpFragments(GrpFragments fragments, + std::map& partFragments) const +{ + // get all unique grpIds; + GrpFragments::iterator vit = fragments.begin(); + std::set grpIds; + + for(; vit != fragments.end(); ++vit) + grpIds.insert( (*vit)->grpId ); + + std::set::iterator sit = grpIds.begin(); + + for(; sit != grpIds.end(); ++sit) + { + const RsGxsGroupId& grpId = *sit; + GrpFragments::iterator bound = std::partition( + fragments.begin(), fragments.end(), + GrpFragCollate(grpId)); + + // something will always be found for a group id + for(vit = fragments.begin(); vit != bound; ) + { + partFragments[grpId].push_back(*vit); + vit = fragments.erase(vit); + } + + GrpFragments& f = partFragments[grpId]; + RsNxsGrp* grp = *(f.begin()); + + // if counts of fragments is incorrect remove + // from coalescion + if(grp->count != f.size()) + { + GrpFragments::iterator vit2 = f.begin(); + + for(; vit2 != f.end(); ++vit2) + delete *vit2; + + partFragments.erase(grpId); + } + } + + fragments.clear(); +} + +struct MsgFragCollate +{ + RsGxsMessageId mMsgId; + MsgFragCollate(const RsGxsMessageId& msgId) : mMsgId(msgId){ } + bool operator()(RsNxsMsg* msg) { return msg->msgId == mMsgId;} +}; + +void RsGxsNetService::collateMsgFragments(MsgFragments fragments, std::map& partFragments) const +{ + // get all unique message Ids; + MsgFragments::iterator vit = fragments.begin(); + std::set msgIds; + + for(; vit != fragments.end(); ++vit) + msgIds.insert( (*vit)->msgId ); + + + std::set::iterator sit = msgIds.begin(); + + for(; sit != msgIds.end(); ++sit) + { + const RsGxsMessageId& msgId = *sit; + MsgFragments::iterator bound = std::partition( + fragments.begin(), fragments.end(), + MsgFragCollate(msgId)); + + // something will always be found for a group id + for(vit = fragments.begin(); vit != bound; ++vit ) + { + partFragments[msgId].push_back(*vit); + } + + fragments.erase(fragments.begin(), bound); + MsgFragments& f = partFragments[msgId]; + RsNxsMsg* msg = *(f.begin()); + + // if counts of fragments is incorrect remove + // from coalescion + if(msg->count != f.size()) + { + MsgFragments::iterator vit2 = f.begin(); + + for(; vit2 != f.end(); ++vit2) + delete *vit2; + + partFragments.erase(msgId); + } + } + + fragments.clear(); +} + +class StoreHere +{ +public: + + StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap& cmm, RsGxsNetService::ServerMsgMap& smm, RsGxsServerGrpUpdateItem*& sgm) + : mClientGrpMap(cgm), mClientMsgMap(cmm), mServerMsgMap(smm), mServerGrpUpdateItem(sgm) + {} + + void operator() (RsItem* item) + { + RsGxsMsgUpdateItem* mui; + RsGxsGrpUpdateItem* gui; + RsGxsServerGrpUpdateItem* gsui; + RsGxsServerMsgUpdateItem* msui; + + if((mui = dynamic_cast(item)) != NULL) + mClientMsgMap.insert(std::make_pair(mui->peerId, mui)); + else if((gui = dynamic_cast(item)) != NULL) + mClientGrpMap.insert(std::make_pair(gui->peerId, gui)); + else if((msui = dynamic_cast(item)) != NULL) + mServerMsgMap.insert(std::make_pair(msui->grpId, msui)); + else if((gsui = dynamic_cast(item)) != NULL) + { + if(mServerGrpUpdateItem == NULL) + mServerGrpUpdateItem = gsui; + else + { + std::cerr << "Error! More than one server group update item exists!" << std::endl; + delete gsui; + } + } + else + { + std::cerr << "Type not expected!" << std::endl; + delete item ; + } + } + +private: + + RsGxsNetService::ClientGrpMap& mClientGrpMap; + RsGxsNetService::ClientMsgMap& mClientMsgMap; + RsGxsNetService::ServerMsgMap& mServerMsgMap; + RsGxsServerGrpUpdateItem*& mServerGrpUpdateItem; + +}; + +bool RsGxsNetService::loadList(std::list &load) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + // The delete is done in StoreHere, if necessary + + std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mGrpServerUpdateItem)); + time_t now = time(NULL); + + for(ClientMsgMap::iterator it = mClientMsgUpdateMap.begin();it!=mClientMsgUpdateMap.end();++it) + for(std::map::const_iterator it2(it->second->msgUpdateInfos.begin());it2!=it->second->msgUpdateInfos.end();++it2) + { + RsGroupNetworkStatsRecord& gnsr = mGroupNetworkStats[it2->first] ; + + // At each reload, divide the last count by 2. This gradually flushes old information away. + + gnsr.max_visible_count = std::max(it2->second.message_count,gnsr.max_visible_count/2) ; + gnsr.update_TS = now - GROUP_STATS_UPDATE_DELAY + (RSRandom::random_u32()%(GROUP_STATS_UPDATE_DELAY/10)) ; + + // Similarly, we remove some of the suppliers randomly. If they are + // actual suppliers, they will come back automatically. If they are + // not, they will be forgotten. + + if(RSRandom::random_f32() > 0.2) + gnsr.suppliers.insert(it->first) ; + } + return true; +} + +#include + +template +struct get_second : public std::unary_function +{ + RsItem* operator()(const typename UpdateMap::value_type& value) const + { + return value.second; + } +}; + +bool RsGxsNetService::saveList(bool& cleanup, std::list& save) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + // hardcore templates + std::transform(mClientGrpUpdateMap.begin(), mClientGrpUpdateMap.end(), std::back_inserter(save), get_second()); + std::transform(mClientMsgUpdateMap.begin(), mClientMsgUpdateMap.end(), std::back_inserter(save), get_second()); + std::transform(mServerMsgUpdateMap.begin(), mServerMsgUpdateMap.end(), std::back_inserter(save), get_second()); + + save.push_back(mGrpServerUpdateItem); + + cleanup = false; + return true; +} + +RsSerialiser *RsGxsNetService::setupSerialiser() +{ + + RsSerialiser *rss = new RsSerialiser; + rss->addSerialType(new RsGxsUpdateSerialiser(mServType)); + + return rss; +} + +void RsGxsNetService::recvNxsItemQueue() +{ + RsItem *item ; + + while(NULL != (item=recvItem())) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetService Item:" << (void*)item << std::endl ; +#endif + // RsNxsItem needs dynamic_cast, since they have derived siblings. + // + RsNxsItem *ni = dynamic_cast(item) ; + if(ni != NULL) + { + // a live transaction has a non zero value + if(ni->transactionNumber != 0) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(item->PeerId()) << " recvNxsItemQueue() handlingTransaction, transN " << ni->transactionNumber << std::endl; +#endif + + if(!handleTransaction(ni)) + delete ni; + + continue; + } + + + switch(ni->PacketSubType()) + { + case RS_PKT_SUBTYPE_NXS_SYNC_GRP_STATS: handleRecvSyncGrpStatistics (dynamic_cast(ni)) ; break ; + case RS_PKT_SUBTYPE_NXS_SYNC_GRP: handleRecvSyncGroup (dynamic_cast(ni)) ; break ; + case RS_PKT_SUBTYPE_NXS_SYNC_MSG: handleRecvSyncMessage (dynamic_cast(ni)) ; break ; + case RS_PKT_SUBTYPE_NXS_GRP_PUBLISH_KEY: handleRecvPublishKeys (dynamic_cast(ni)) ; break ; + default: + std::cerr << "Unhandled item subtype " << (uint32_t) ni->PacketSubType() << " in RsGxsNetService: " << std::endl; break; + } + delete item ; + } + else + { + std::cerr << "Not a RsNxsItem, deleting!" << std::endl; + delete(item); + } + } +} + + +bool RsGxsNetService::handleTransaction(RsNxsItem* item) +{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(item->PeerId()) << "handleTransaction(RsNxsItem) number=" << item->transactionNumber << std::endl; +#endif + + /*! + * This attempts to handle a transaction + * It first checks if this transaction id already exists + * If it does then check this not a initiating transactions + */ + + RS_STACK_MUTEX(mNxsMutex) ; + + const RsPeerId& peer = item->PeerId(); + + RsNxsTransac* transItem = dynamic_cast(item); + + // if this is a RsNxsTransac item process + if(transItem) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(item->PeerId()) << " this is a RsNxsTransac item. callign process." << std::endl; +#endif + return locked_processTransac(transItem); + } + + + // then this must be transaction content to be consumed + // first check peer exist for transaction + bool peerTransExists = mTransactions.find(peer) != mTransactions.end(); + + // then check transaction exists + + NxsTransaction* tr = NULL; + uint32_t transN = item->transactionNumber; + + if(peerTransExists) + { + TransactionIdMap& transMap = mTransactions[peer]; + + if(transMap.find(transN) != transMap.end()) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(item->PeerId()) << " Consuming Transaction content, transN: " << item->transactionNumber << std::endl; + GXSNETDEBUG_P_(item->PeerId()) << " Consuming Transaction content, from Peer: " << item->PeerId() << std::endl; +#endif + + tr = transMap[transN]; + tr->mItems.push_back(item); + + return true; + } + } + + return false; +} + +bool RsGxsNetService::locked_processTransac(RsNxsTransac* item) +{ + + /*! + * To process the transaction item + * It can either be initiating a transaction + * or ending one that already exists + * + * For initiating an incoming transaction the peer + * and transaction item need not exists + * as the peer will be added and transaction number + * added thereafter + * + * For commencing/starting an outgoing transaction + * the transaction must exist already + * + * For ending a transaction the + */ + + RsPeerId peer; + + // for outgoing transaction use own id + if(item->transactFlag & (RsNxsTransac::FLAG_BEGIN_P2 | RsNxsTransac::FLAG_END_SUCCESS)) + peer = mOwnId; + else + peer = item->PeerId(); + + uint32_t transN = item->transactionNumber; + item->timestamp = time(NULL); // register time received + NxsTransaction* tr = NULL; + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << "locked_processTransac() " << std::endl; + GXSNETDEBUG_P_(peer) << " Received transaction item: " << transN << std::endl; + GXSNETDEBUG_P_(peer) << " With peer: " << item->PeerId() << std::endl; + GXSNETDEBUG_P_(peer) << " trans type: " << item->transactFlag << std::endl; +#endif + + bool peerTrExists = mTransactions.find(peer) != mTransactions.end(); + bool transExists = false; + + if(peerTrExists) + { + TransactionIdMap& transMap = mTransactions[peer]; + // record whether transaction exists already + transExists = transMap.find(transN) != transMap.end(); + } + + // initiating an incoming transaction + if(item->transactFlag & RsNxsTransac::FLAG_BEGIN_P1) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << " initiating Incoming transaction." << std::endl; +#endif + + if(transExists) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << " transaction already exist! ERROR" << std::endl; +#endif + return false; // should not happen! + } + + // create a transaction if the peer does not exist + if(!peerTrExists) + mTransactions[peer] = TransactionIdMap(); + + TransactionIdMap& transMap = mTransactions[peer]; + + + // create new transaction + tr = new NxsTransaction(); + transMap[transN] = tr; + tr->mTransaction = item; + tr->mTimeOut = item->timestamp + mTransactionTimeOut; +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << " Setting timeout of " << mTransactionTimeOut << " secs, which is " << tr->mTimeOut - time(NULL) << " secs from now." << std::endl; +#endif + + // note state as receiving, commencement item + // is sent on next run() loop + tr->mFlag = NxsTransaction::FLAG_STATE_STARTING; + return true; + // commencement item for outgoing transaction + } + else if(item->transactFlag & RsNxsTransac::FLAG_BEGIN_P2) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << " initiating outgoign transaction." << std::endl; +#endif + // transaction must exist + if(!peerTrExists || !transExists) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << " transaction does not exist. Cancelling!" << std::endl; +#endif + + return false; + } + + + // alter state so transaction content is sent on + // next run() loop + TransactionIdMap& transMap = mTransactions[mOwnId]; + NxsTransaction* tr = transMap[transN]; + tr->mFlag = NxsTransaction::FLAG_STATE_SENDING; + delete item; + return true; + // end transac item for outgoing transaction + } + else if(item->transactFlag & RsNxsTransac::FLAG_END_SUCCESS) + { + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << " marking this transaction succeed" << std::endl; +#endif + // transaction does not exist + if(!peerTrExists || !transExists) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << " transaction does not exist. Cancelling!" << std::endl; +#endif + return false; + } + + // alter state so that transaction is removed + // on next run() loop + TransactionIdMap& transMap = mTransactions[mOwnId]; + NxsTransaction* tr = transMap[transN]; + tr->mFlag = NxsTransaction::FLAG_STATE_COMPLETED; + delete item; + return true; + } + else + return false; +} + +void RsGxsNetService::data_tick() +{ + static const double timeDelta = 0.5; + + //Start waiting as nothing to do in runup + usleep((int) (timeDelta * 1000 * 1000)); // timeDelta sec + + if(mUpdateCounter >= 120) // 60 seconds + { + updateServerSyncTS(); +#ifdef TO_REMOVE + updateClientSyncTS(); +#endif + mUpdateCounter = 1; + } + else + mUpdateCounter++; + + if(mUpdateCounter % 20 == 0) // dump the full shit every 20 secs + debugDump() ; + + // process active transactions + processTransactions(); + + // process completed transactions + processCompletedTransactions(); + + // vetting of id and circle info + runVetting(); + + processExplicitGroupRequests(); +} + +void RsGxsNetService::debugDump() +{ +#ifdef NXS_NET_DEBUG_0 + RS_STACK_MUTEX(mNxsMutex) ; + time_t now = time(NULL) ; + + GXSNETDEBUG___<< "RsGxsNetService::debugDump():" << std::endl; + + if(mGrpServerUpdateItem != NULL) + GXSNETDEBUG___<< " mGrpServerUpdateItem time stamp: " << nice_time_stamp(time(NULL) , mGrpServerUpdateItem->grpUpdateTS) << " (is the last local modification time over all groups of this service)" << std::endl; + else + GXSNETDEBUG___<< " mGrpServerUpdateItem time stamp: not inited yet (is the last local modification time over all groups of this service)" << std::endl; + + GXSNETDEBUG___<< " mServerMsgUpdateMap: (is for each subscribed group, the last local modification time)" << std::endl; + + for(std::map::const_iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it) + GXSNETDEBUG__G(it->first) << " Grp:" << it->first << " last local modification (secs ago): " << nice_time_stamp(time(NULL),it->second->msgUpdateTS) << std::endl; + + GXSNETDEBUG___<< " mClientGrpUpdateMap: (is for each friend, last modif time of group meta data at that friend, all groups included, sent by the friend himself)" << std::endl; + + for(std::map::const_iterator it(mClientGrpUpdateMap.begin());it!=mClientGrpUpdateMap.end();++it) + GXSNETDEBUG_P_(it->first) << " From peer: " << it->first << " - last updated at peer (secs ago): " << nice_time_stamp(time(NULL),it->second->grpUpdateTS) << std::endl; + + GXSNETDEBUG___<< " mClientMsgUpdateMap: (is for each friend, the modif time for each group (e.g. last message received), sent by the friend himself)" << std::endl; + + for(std::map::const_iterator it(mClientMsgUpdateMap.begin());it!=mClientMsgUpdateMap.end();++it) + { + GXSNETDEBUG_P_(it->first) << " From peer: " << it->first << std::endl; + + for(std::map::const_iterator it2(it->second->msgUpdateInfos.begin());it2!=it->second->msgUpdateInfos.end();++it2) + GXSNETDEBUG_PG(it->first,it2->first) << " group " << it2->first << " - last updated at peer (secs ago): " << nice_time_stamp(time(NULL),it2->second.time_stamp) << ". Message count=" << it2->second.message_count << std::endl; + } + + GXSNETDEBUG___<< " List of rejected message ids: " << mRejectedMessages.size() << std::endl; +#endif +} + +#ifdef TO_REMOVE +// This method is normally not needed, but we use it to correct possible inconsistencies in the updte time stamps +// on the client side. + +void RsGxsNetService::updateClientSyncTS() +{ + RS_STACK_MUTEX(mNxsMutex) ; + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG___<< "updateClientSyncTS(): checking last modification time stamps of local data w.r.t. client's modification times" << std::endl; +#endif + + if(mGrpServerUpdateItem == NULL) + mGrpServerUpdateItem = new RsGxsServerGrpUpdateItem(mServType); + + for(ClientGrpMap::iterator it = mClientGrpUpdateMap.begin();it!=mClientGrpUpdateMap.end();++it) + if(it->second->grpUpdateTS > SECURITY_DELAY_TO_FORCE_CLIENT_REUPDATE + mGrpServerUpdateItem->grpUpdateTS) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(it->first) << " last global client GRP modification time known for peer (" << nice_time_stamp(time(NULL),it->second->grpUpdateTS) << " is quite more recent than our own server modification time (" << nice_time_stamp(time(NULL),mGrpServerUpdateItem->grpUpdateTS) << ". Forcing update! " << std::endl; +#endif + it->second->grpUpdateTS = 0 ; + } + + for(ClientMsgMap::iterator it = mClientMsgUpdateMap.begin();it!=mClientMsgUpdateMap.end();++it) + for(std::map::iterator it2 = it->second->msgUpdateInfos.begin();it2!=it->second->msgUpdateInfos.end();++it2) + { + std::map::const_iterator mmit = mServerMsgUpdateMap.find(it2->first) ; + + if(mmit != mServerMsgUpdateMap.end() && it2->second.time_stamp > SECURITY_DELAY_TO_FORCE_CLIENT_REUPDATE + mmit->second->msgUpdateTS) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(it->first,it2->first) << " last group msg modification time known for peer (" << nice_time_stamp(time(NULL),it2->second.time_stamp) << " and group " << it2->first << " is quite more recent than our own server modification time (" << nice_time_stamp(time(NULL),mmit->second->msgUpdateTS) << ". Forcing update! " << std::endl; +#endif + it2->second.time_stamp = 0 ; + } + } +} +#endif + +void RsGxsNetService::updateServerSyncTS() +{ + RS_STACK_MUTEX(mNxsMutex) ; + + RsGxsMetaDataTemporaryMap gxsMap; + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG___<< "updateServerSyncTS(): updating last modification time stamp of local data." << std::endl; +#endif + + // retrieve all grps and update TS + mDataStore->retrieveGxsGrpMetaData(gxsMap); + + // as a grp list server also note this is the latest item you have + if(mGrpServerUpdateItem == NULL) + mGrpServerUpdateItem = new RsGxsServerGrpUpdateItem(mServType); + + // First reset it. That's important because it will re-compute correct TS in case + // we have unsubscribed a group. + + mGrpServerUpdateItem->grpUpdateTS = 0 ; + bool change = false; + + // then remove from mServerMsgUpdateMap, all items that are not in the group list! + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG___ << " cleaning server map of groups with no data:" << std::endl; +#endif + + for(std::map::iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();) + if(gxsMap.find(it->first) == gxsMap.end()) + { + // not found! Removing server update info for this group + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG__G(it->first) << " removing server update info for group " << it->first << std::endl; +#endif + std::map::iterator tmp(it) ; + ++tmp ; + mServerMsgUpdateMap.erase(it) ; + it = tmp ; + } + else + ++it; + +#ifdef NXS_NET_DEBUG_0 + if(gxsMap.empty()) + GXSNETDEBUG___<< " database seems to be empty. The modification timestamp will be reset." << std::endl; +#endif + // finally, update timestamps. + + for(std::map::const_iterator mit = gxsMap.begin();mit != gxsMap.end(); ++mit) + { + const RsGxsGroupId& grpId = mit->first; + const RsGxsGrpMetaData* grpMeta = mit->second; + ServerMsgMap::iterator mapIT = mServerMsgUpdateMap.find(grpId); + RsGxsServerMsgUpdateItem* msui = NULL; + + // That accounts for modification of the meta data. + + if(mGrpServerUpdateItem->grpUpdateTS < grpMeta->mPublishTs) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG__G(grpId) << " publish time stamp of group " << grpId << " has changed to " << time(NULL)-grpMeta->mPublishTs << " secs ago. updating!" << std::endl; +#endif + mGrpServerUpdateItem->grpUpdateTS = grpMeta->mPublishTs; + } + + if(mapIT == mServerMsgUpdateMap.end()) + { + msui = new RsGxsServerMsgUpdateItem(mServType); + msui->grpId = grpMeta->mGroupId; + + mServerMsgUpdateMap.insert(std::make_pair(msui->grpId, msui)); +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG__G(grpId) << " created new entry for group " << grpId << std::endl; +#endif + } + else + msui = mapIT->second; + + if(grpMeta->mLastPost > msui->msgUpdateTS ) + { + change = true; + msui->msgUpdateTS = grpMeta->mLastPost; +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG__G(grpId) << " updated msgUpdateTS to last post = " << time(NULL) - grpMeta->mLastPost << " secs ago for group "<< grpId << std::endl; +#endif + } + + // This might be very inefficient with time. This is needed because an old message might have been received, so the last modification time + // needs to account for this so that a friend who hasn't + + if(mGrpServerUpdateItem->grpUpdateTS < grpMeta->mRecvTS) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG__G(grpId) << " updated msgUpdateTS to last RecvTS = " << time(NULL) - grpMeta->mRecvTS << " secs ago for group "<< grpId << std::endl; +#endif + mGrpServerUpdateItem->grpUpdateTS = grpMeta->mRecvTS; + change = true; + } + } + + // actual change in config settings, then save configuration + if(change) + IndicateConfigChanged(); +} +bool RsGxsNetService::locked_checkTransacTimedOut(NxsTransaction* tr) +{ + return tr->mTimeOut < ((uint32_t) time(NULL)); +} + +void RsGxsNetService::processTransactions() +{ +#ifdef NXS_NET_DEBUG_1 + if(!mTransactions.empty()) + GXSNETDEBUG___ << "processTransactions()" << std::endl; +#endif + RS_STACK_MUTEX(mNxsMutex) ; + + TransactionsPeerMap::iterator mit = mTransactions.begin(); + + for(; mit != mTransactions.end(); ++mit) + { + TransactionIdMap& transMap = mit->second; + TransactionIdMap::iterator mmit = transMap.begin(), mmit_end = transMap.end(); + +#ifdef NXS_NET_DEBUG_1 + if(mmit != mmit_end) + GXSNETDEBUG_P_(mit->first) << " peerId=" << mit->first << std::endl; +#endif + // transaction to be removed + std::list toRemove; + + /*! + * Transactions owned by peer + */ + if(mit->first == mOwnId) + { + for(; mmit != mmit_end; ++mmit) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " type: outgoing " << std::endl; + GXSNETDEBUG_P_(mit->first) << " transN = " << mmit->second->mTransaction->transactionNumber << std::endl; +#endif + NxsTransaction* tr = mmit->second; + uint16_t flag = tr->mFlag; + std::list::iterator lit, lit_end; + uint32_t transN = tr->mTransaction->transactionNumber; + + // first check transaction has not expired + if(locked_checkTransacTimedOut(tr)) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " timeout! " << std::endl; + GXSNETDEBUG_P_(mit->first) << std::dec ; + int total_transaction_time = (int)time(NULL) - (tr->mTimeOut - mTransactionTimeOut) ; + GXSNETDEBUG_P_(mit->first) << " Outgoing Transaction has failed, tranN: " << transN << ", Peer: " << mit->first ; + GXSNETDEBUG_P_(mit->first) << ", age: " << total_transaction_time << ", nItems=" << tr->mTransaction->nItems << ". tr->mTimeOut = " << tr->mTimeOut << ", now = " << (uint32_t) time(NULL) << std::endl; +#endif + + tr->mFlag = NxsTransaction::FLAG_STATE_FAILED; + toRemove.push_back(transN); + mComplTransactions.push_back(tr); + continue; + } +#ifdef NXS_NET_DEBUG_1 + else + GXSNETDEBUG_P_(mit->first) << " still on time." << std::endl; +#endif + + // send items requested + if(flag & NxsTransaction::FLAG_STATE_SENDING) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first)<< " Sending Transaction content, transN: " << transN << " with peer: " << tr->mTransaction->PeerId() << std::endl; +#endif + lit = tr->mItems.begin(); + lit_end = tr->mItems.end(); + + for(; lit != lit_end; ++lit){ + sendItem(*lit); + } + + tr->mItems.clear(); // clear so they don't get deleted in trans cleaning + tr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; + + } + else if(flag & NxsTransaction::FLAG_STATE_WAITING_CONFIRM) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first)<< " Waiting confirm! returning." << std::endl; +#endif + continue; + + } + else if(flag & NxsTransaction::FLAG_STATE_COMPLETED) + { + +#ifdef NXS_NET_DEBUG_1 + int total_transaction_time = (int)time(NULL) - (tr->mTimeOut - mTransactionTimeOut) ; + + GXSNETDEBUG_P_(mit->first)<< " Outgoing completed " << tr->mTransaction->nItems << " items transaction in " << total_transaction_time << " seconds." << std::endl; +#endif + // move to completed transactions + toRemove.push_back(transN); + mComplTransactions.push_back(tr); + }else{ + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first)<< " Unknown flag for active transaction, transN: " << transN << ", Peer: " << mit->first<< std::endl; +#endif + + toRemove.push_back(transN); + tr->mFlag = NxsTransaction::FLAG_STATE_FAILED; + mComplTransactions.push_back(tr); + } + } + + }else{ + + /*! + * Essentially these are incoming transactions + * Several states are dealth with + * Receiving: waiting to receive items from peer's transaction + * and checking if all have been received + * Completed: remove transaction from active and tell peer + * involved in transaction + * Starting: this is a new transaction and need to teell peer + * involved in transaction + */ + + for(; mmit != mmit_end; ++mmit){ + + NxsTransaction* tr = mmit->second; + uint16_t flag = tr->mFlag; + uint32_t transN = tr->mTransaction->transactionNumber; + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " type: incoming " << std::endl; + GXSNETDEBUG_P_(mit->first) << " transN = " << mmit->second->mTransaction->transactionNumber << std::endl; +#endif + // first check transaction has not expired + if(locked_checkTransacTimedOut(tr)) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " timeout!" << std::endl; + GXSNETDEBUG_P_(mit->first) << std::dec ; + int total_transaction_time = (int)time(NULL) - (tr->mTimeOut - mTransactionTimeOut) ; + GXSNETDEBUG_P_(mit->first) << " Incoming Transaction has failed, tranN: " << transN << ", Peer: " << mit->first ; + GXSNETDEBUG_P_(mit->first) << ", age: " << total_transaction_time << ", nItems=" << tr->mTransaction->nItems << ". tr->mTimeOut = " << tr->mTimeOut << ", now = " << (uint32_t) time(NULL) << std::endl; +#endif + + tr->mFlag = NxsTransaction::FLAG_STATE_FAILED; + toRemove.push_back(transN); + mComplTransactions.push_back(tr); + continue; + } + + if(flag & NxsTransaction::FLAG_STATE_RECEIVING) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " received " << tr->mItems.size() << " item over a total of " << tr->mTransaction->nItems << std::endl; +#endif + + // if the number it item received equal that indicated + // then transaction is marked as completed + // to be moved to complete transations + // check if done + if(tr->mItems.size() == tr->mTransaction->nItems) + { + tr->mFlag = NxsTransaction::FLAG_STATE_COMPLETED; +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " completed!" << std::endl; +#endif + } + + }else if(flag & NxsTransaction::FLAG_STATE_COMPLETED) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " transaction is completed!" << std::endl; + GXSNETDEBUG_P_(mit->first) << " sending success!" << std::endl; +#endif + + // send completion msg + RsNxsTransac* trans = new RsNxsTransac(mServType); + trans->clear(); + trans->transactFlag = RsNxsTransac::FLAG_END_SUCCESS; + trans->transactionNumber = transN; + trans->PeerId(tr->mTransaction->PeerId()); + sendItem(trans); + + // move to completed transactions + mComplTransactions.push_back(tr); +#ifdef NXS_NET_DEBUG_1 + int total_transaction_time = (int)time(NULL) - (tr->mTimeOut - mTransactionTimeOut) ; + GXSNETDEBUG_P_(mit->first) << " incoming completed " << tr->mTransaction->nItems << " items transaction in " << total_transaction_time << " seconds." << std::endl; +#endif + + // transaction processing done + // for this id, add to removal list + toRemove.push_back(mmit->first); + } + else if(flag & NxsTransaction::FLAG_STATE_STARTING) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " transaction is starting!" << std::endl; + GXSNETDEBUG_P_(mit->first) << " setting state to Receiving" << std::endl; +#endif + // send item to tell peer your are ready to start + RsNxsTransac* trans = new RsNxsTransac(mServType); + trans->clear(); + trans->transactFlag = RsNxsTransac::FLAG_BEGIN_P2 | + (tr->mTransaction->transactFlag & RsNxsTransac::FLAG_TYPE_MASK); + trans->transactionNumber = transN; + trans->PeerId(tr->mTransaction->PeerId()); + sendItem(trans); + tr->mFlag = NxsTransaction::FLAG_STATE_RECEIVING; + + } + else{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(mit->first) << " transaction is in unknown state. ERROR!" << std::endl; + GXSNETDEBUG_P_(mit->first) << " transaction FAILS!" << std::endl; +#endif + + std::cerr << " Unknown flag for active transaction, transN: " << transN << ", Peer: " << mit->first << std::endl; + toRemove.push_back(mmit->first); + mComplTransactions.push_back(tr); + tr->mFlag = NxsTransaction::FLAG_STATE_FAILED; // flag as a failed transaction + } + } + } + + std::list::iterator lit = toRemove.begin(); + + for(; lit != toRemove.end(); ++lit) + { + transMap.erase(*lit); + } + + } +} + +bool RsGxsNetService::getGroupNetworkStats(const RsGxsGroupId& gid,RsGroupNetworkStats& stats) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + std::map::const_iterator it = mGroupNetworkStats.find(gid) ; + + if(it == mGroupNetworkStats.end()) + return false ; + + stats.mSuppliers = it->second.suppliers.size(); + stats.mMaxVisibleCount = it->second.max_visible_count ; + + return true ; +} + +void RsGxsNetService::processCompletedTransactions() +{ + RS_STACK_MUTEX(mNxsMutex) ; + /*! + * Depending on transaction we may have to respond to peer + * responsible for transaction + */ + while(mComplTransactions.size()>0) + { + + NxsTransaction* tr = mComplTransactions.front(); + + bool outgoing = tr->mTransaction->PeerId() == mOwnId; + + if(outgoing){ + locked_processCompletedOutgoingTrans(tr); + }else{ + locked_processCompletedIncomingTrans(tr); + } + + + delete tr; + mComplTransactions.pop_front(); + } +} + +void RsGxsNetService::locked_processCompletedIncomingTrans(NxsTransaction* tr) +{ + uint16_t flag = tr->mTransaction->transactFlag; + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "Processing complete Incoming transaction with " << tr->mTransaction->nItems << " items." << std::endl; + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " flags = " << flag << std::endl; + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " peerId= " << tr->mTransaction->PeerId() << std::endl; +#endif + if(tr->mFlag & NxsTransaction::FLAG_STATE_COMPLETED) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " transaction has completed." << std::endl; +#endif + // for a completed list response transaction + // one needs generate requests from this + if(flag & RsNxsTransac::FLAG_TYPE_MSG_LIST_RESP) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = msg list response." << std::endl; + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " => generate msg request based on it." << std::endl; +#endif + // generate request based on a peers response + locked_genReqMsgTransaction(tr); + + }else if(flag & RsNxsTransac::FLAG_TYPE_GRP_LIST_RESP) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = grp list response." << std::endl; + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " => generate group transaction request based on it." << std::endl; +#endif + locked_genReqGrpTransaction(tr); + } + // you've finished receiving request information now gen + else if(flag & RsNxsTransac::FLAG_TYPE_MSG_LIST_REQ) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = msg list request." << std::endl; + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " => generate msg list based on it." << std::endl; +#endif + locked_genSendMsgsTransaction(tr); + } + else if(flag & RsNxsTransac::FLAG_TYPE_GRP_LIST_REQ) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = grp list request." << std::endl; + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " => generate grp list based on it." << std::endl; +#endif + locked_genSendGrpsTransaction(tr); + } + else if(flag & RsNxsTransac::FLAG_TYPE_GRPS) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = groups." << std::endl; +#endif + std::vector grps; + + while(tr->mItems.size() != 0) + { + RsNxsGrp* grp = dynamic_cast(tr->mItems.front()); + + if(grp) + { + tr->mItems.pop_front(); + grps.push_back(grp); +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grp->grpId) << " adding new group " << grp->grpId << " to incoming list!" << std::endl; +#endif + } + else + std::cerr << " /!\\ item did not caste to grp" << std::endl; + } + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " ...and notifying observer " << std::endl; +#endif +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_P_ (tr->mTransaction->PeerId()) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - Received new groups meta data from peer " << tr->mTransaction->PeerId() << std::endl; + for(uint32_t i=0;imTransaction->PeerId(),grps[i]->grpId) ; +#endif + // notify listener of grps + for(uint32_t i=0;imTransaction->PeerId(); + uint32_t updateTS = tr->mTransaction->updateTS; + + ClientGrpMap::iterator it = mClientGrpUpdateMap.find(peerFrom); + + RsGxsGrpUpdateItem* item = NULL; + + if(it != mClientGrpUpdateMap.end()) + { + item = it->second; + }else + { + item = new RsGxsGrpUpdateItem(mServType); + mClientGrpUpdateMap.insert(std::make_pair(peerFrom, item)); + } +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " and updating mClientGrpUpdateMap for peer " << peerFrom << " of new time stamp " << nice_time_stamp(time(NULL),updateTS) << std::endl; +#endif + +#warning should not we conservatively use the most recent one, in case the peer has reset its mServerGrpUpdate time?? What happens if the peer unsubscribed a recent group? + item->grpUpdateTS = updateTS; + item->peerId = peerFrom; + + IndicateConfigChanged(); + + + }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) + { + + std::vector msgs; +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " type = msgs." << std::endl; +#endif + RsGxsGroupId grpId; + while(tr->mItems.size() > 0) + { + RsNxsMsg* msg = dynamic_cast(tr->mItems.front()); + if(msg) + { + if(grpId.isNull()) + grpId = msg->grpId; + + tr->mItems.pop_front(); + msgs.push_back(msg); +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),msg->grpId) << " pushing grpId="<< msg->grpId << ", msgsId=" << msg->msgId << " to list of incoming messages" << std::endl; +#endif + } + else + std::cerr << "RsGxsNetService::processCompletedTransactions(): item did not caste to msg" << std::endl; + } + +#ifdef NSXS_FRAG + std::map collatedMsgs; + collateMsgFragments(msgs, collatedMsgs); + + msgs.clear(); + + std::map::iterator mit = collatedMsgs.begin(); + for(; mit != collatedMsgs.end(); ++mit) + { + MsgFragments& f = mit->second; + RsNxsMsg* msg = deFragmentMsg(f); + + if(msg) + msgs.push_back(msg); + } +#endif +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << " ...and notifying observer of " << msgs.size() << " new messages." << std::endl; +#endif +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_PG (tr->mTransaction->PeerId(),grpId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - Received new messages from peer " << tr->mTransaction->PeerId() << " for group " << grpId << std::endl; + for(uint32_t i=0;imTransaction->PeerId(),grpId) << " " << msgs[i]->msgId << std::endl ; +#endif + // notify listener of msgs + for(uint32_t i=0;imTransaction, grpId); + + } + } + else if(tr->mFlag == NxsTransaction::FLAG_STATE_FAILED) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " transaction has failed. Wasting it." << std::endl; +#endif + // don't do anything transaction will simply be cleaned + } + return; +} + +void RsGxsNetService::locked_doMsgUpdateWork(const RsNxsTransac *nxsTrans, const RsGxsGroupId &grpId) +{ +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(nxsTrans->PeerId(),grpId) << "updating MsgUpdate time stamps for peerId=" << nxsTrans->PeerId() << ", grpId=" << grpId << std::endl; +#endif + // firts check if peer exists + const RsPeerId& peerFrom = nxsTrans->PeerId(); + + ClientMsgMap::iterator it = mClientMsgUpdateMap.find(peerFrom); + + RsGxsMsgUpdateItem* mui = NULL; + + // now update the peer's entry for this grp id + if(it != mClientMsgUpdateMap.end()) + { + mui = it->second; + } + else + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(nxsTrans->PeerId(),grpId) << " created new entry." << std::endl; +#endif + mui = new RsGxsMsgUpdateItem(mServType); + mClientMsgUpdateMap.insert(std::make_pair(peerFrom, mui)); + } + + mui->peerId = peerFrom; + + if(mPartialMsgUpdates[peerFrom].find(grpId) != mPartialMsgUpdates[peerFrom].end()) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(nxsTrans->PeerId(),grpId) << " this is a partial update. Not using new time stamp." << std::endl; +#endif + } + else + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(nxsTrans->PeerId(),grpId) << " this is a full update. Updating time stamp." << std::endl; +#endif + mui->msgUpdateInfos[grpId].time_stamp = nxsTrans->updateTS; + IndicateConfigChanged(); + } +} + +void RsGxsNetService::locked_processCompletedOutgoingTrans(NxsTransaction* tr) +{ + uint16_t flag = tr->mTransaction->transactFlag; + +#ifdef NXS_NET_DEBUG_0 + RsNxsTransac *nxsTrans = tr->mTransaction; + GXSNETDEBUG_P_(nxsTrans->PeerId()) << "locked_processCompletedOutgoingTrans(): tr->flags = " << flag << std::endl; +#endif + + if(tr->mFlag & NxsTransaction::FLAG_STATE_COMPLETED) + { + // for a completed list response transaction + // one needs generate requests from this + if(flag & RsNxsTransac::FLAG_TYPE_MSG_LIST_RESP) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Msg List Response, transN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + }else if(flag & RsNxsTransac::FLAG_TYPE_GRP_LIST_RESP) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Grp Response, transN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + } + // you've finished sending a request so don't do anything + else if( (flag & RsNxsTransac::FLAG_TYPE_MSG_LIST_REQ) || + (flag & RsNxsTransac::FLAG_TYPE_GRP_LIST_REQ) ) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Msg/Grp Request, transN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + + }else if(flag & RsNxsTransac::FLAG_TYPE_GRPS) + { + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Grp Data, transN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + + }else if(flag & RsNxsTransac::FLAG_TYPE_MSGS) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(nxsTrans->PeerId())<< " complete Sending Msg Data, transN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + } + }else if(tr->mFlag == NxsTransaction::FLAG_STATE_FAILED){ +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(nxsTrans->PeerId())<< " Failed transaction! transN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + }else{ + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(nxsTrans->PeerId())<< " Serious error unrecognised trans Flag! transN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + } +} + + +void RsGxsNetService::locked_pushMsgTransactionFromList(std::list& reqList, const RsPeerId& peerId, const uint32_t& transN) +{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peerId) << "locked_pushMsgTransactionFromList()" << std::endl; + GXSNETDEBUG_P_(peerId) << " nelems = " << reqList.size() << std::endl; + GXSNETDEBUG_P_(peerId) << " peerId = " << peerId << std::endl; + GXSNETDEBUG_P_(peerId) << " transN = " << transN << std::endl; +#endif +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_P_ (peerId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending message request to peer " + << peerId << " for " << reqList.size() << " messages" << std::endl; +#endif + RsNxsTransac* transac = new RsNxsTransac(mServType); + transac->transactFlag = RsNxsTransac::FLAG_TYPE_MSG_LIST_REQ + | RsNxsTransac::FLAG_BEGIN_P1; + transac->timestamp = 0; + transac->nItems = reqList.size(); + transac->PeerId(peerId); + transac->transactionNumber = transN; + NxsTransaction* newTrans = new NxsTransaction(); + newTrans->mItems = reqList; + newTrans->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; + newTrans->mTimeOut = time(NULL) + mTransactionTimeOut; + // create transaction copy with your id to indicate + // its an outgoing transaction + newTrans->mTransaction = new RsNxsTransac(*transac); + newTrans->mTransaction->PeerId(mOwnId); + sendItem(transac); + + if (!locked_addTransaction(newTrans)) + delete newTrans; + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peerId) << " Requested new transaction for " << reqList.size() << " items." << std::endl; +#endif +} + +void RsGxsNetService::locked_genReqMsgTransaction(NxsTransaction* tr) +{ + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG___ << "RsGxsNetService::genReqMsgTransaction()" << std::endl; +#endif + + // to create a transaction you need to know who you are transacting with + // then what msgs to request + // then add an active Transaction for request + + std::list msgItemL; + std::list::iterator lit = tr->mItems.begin(); + + // first get item list sent from transaction + for(; lit != tr->mItems.end(); ++lit) + { + RsNxsSyncMsgItem* item = dynamic_cast(*lit); + if(item) + { + msgItemL.push_back(item); + }else + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(item->PeerId()) << "RsGxsNetService::genReqMsgTransaction(): item failed cast to RsNxsSyncMsgItem* " << std::endl; +#endif + } + } +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " found " << msgItemL.size()<< " messages in this transaction." << std::endl; +#endif + + if(msgItemL.empty()) + return; + + // get grp id for this transaction + RsNxsSyncMsgItem* item = msgItemL.front(); + const RsGxsGroupId& grpId = item->grpId; + + // store the count for the peer who sent the message list + uint32_t mcount = msgItemL.size() ; + RsPeerId pid = msgItemL.front()->PeerId() ; + + RsGroupNetworkStatsRecord& gnsr = mGroupNetworkStats[grpId]; + + std::set::size_type oldSuppliersCount = gnsr.suppliers.size(); + uint32_t oldVisibleCount = gnsr.max_visible_count; + + gnsr.suppliers.insert(pid) ; + gnsr.max_visible_count = std::max(gnsr.max_visible_count, mcount) ; + + if (oldVisibleCount != gnsr.max_visible_count || oldSuppliersCount != gnsr.suppliers.size()) + mObserver->notifyChangedGroupStats(grpId); + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " grpId = " << grpId << std::endl; + GXSNETDEBUG_PG(item->PeerId(),grpId) << " retrieving grp mesta data..." << std::endl; +#endif + RsGxsMetaDataTemporaryMap grpMetaMap; + grpMetaMap[grpId] = NULL; + + mDataStore->retrieveGxsGrpMetaData(grpMetaMap); + RsGxsGrpMetaData* grpMeta = grpMetaMap[grpId]; + +#warning TODO: what if grpMeta is NULL? + if(! (grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )) + { + // For unsubscribed groups, we update the timestamp something more recent, so that the group content will not be asked to the same + // peer again, unless the peer has new info about it. It's important to use the same clock (this is peer's clock) so that + // we never compare times from different (and potentially badly sync-ed clocks) + + std::cerr << "(EE) stepping in part of the code (" << __PRETTY_FUNCTION__ << ") where we shouldn't. This is a bug." << std::endl; +#ifdef TO_REMOVE + locked_stampPeerGroupUpdateTime(pid,grpId,tr->mTransaction->updateTS,msgItemL.size()) ; +#endif + + return ; + } + + int cutoff = 0; + if(grpMeta != NULL) + cutoff = grpMeta->mReputationCutOff; + + GxsMsgReq reqIds; + reqIds[grpId] = std::vector(); + GxsMsgMetaResult result; + mDataStore->retrieveGxsMsgMetaData(reqIds, result); + std::vector &msgMetaV = result[grpId]; + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " retrieving grp message list..." << std::endl; + GXSNETDEBUG_PG(item->PeerId(),grpId) << " grp locally contains " << msgMetaV.size() << " messsages." << std::endl; +#endif + std::vector::const_iterator vit = msgMetaV.begin(); + std::set msgIdSet; + + // put ids in set for each searching + for(; vit != msgMetaV.end(); ++vit) + { + msgIdSet.insert((*vit)->mMsgId); + delete(*vit); + } + msgMetaV.clear(); + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " grp locally contains " << msgIdSet.size() << " unique messsages." << std::endl; +#endif + // get unique id for this transaction + uint32_t transN = locked_getTransactionId(); + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " new transaction ID: " << transN << std::endl; +#endif + // add msgs that you don't have to request list + std::list::iterator llit = msgItemL.begin(); + std::list reqList; + int reqListSize = 0 ; + + const RsPeerId peerFrom = tr->mTransaction->PeerId(); + + MsgAuthorV toVet; + + std::list peers; + peers.push_back(tr->mTransaction->PeerId()); + bool reqListSizeExceeded = false ; + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " sorting items..." << std::endl; +#endif + for(; llit != msgItemL.end(); ++llit) + { + RsNxsSyncMsgItem*& syncItem = *llit; + const RsGxsMessageId& msgId = syncItem->msgId; + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " msg ID = " << msgId ; +#endif + if(reqListSize >= MAX_REQLIST_SIZE) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << ". reqlist too big. Pruning out this item for now." << std::endl; +#endif + reqListSizeExceeded = true ; + continue ; // we should actually break, but we need to print some debug info. + } + + if(reqListSize < MAX_REQLIST_SIZE && msgIdSet.find(msgId) == msgIdSet.end()) + { + + // if reputation is in reputations cache then proceed + // or if there isn't an author (note as author requirement is + // enforced at service level, if no author is needed then reputation + // filtering is optional) + bool noAuthor = syncItem->authorId.isNull(); + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << ", reqlist size=" << reqListSize << ", message not present." ; +#endif + // grp meta must be present if author present + if(!noAuthor && grpMeta == NULL) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << ", no group meta found. Givign up." << std::endl; +#endif + continue; + } + + if(rsReputations->isIdentityBanned(syncItem->authorId)) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << ", Identity " << syncItem->authorId << " is banned. Not requesting message!" << std::endl; +#endif + continue ; + } + + if(mRejectedMessages.find(msgId) != mRejectedMessages.end()) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << ", message has been recently rejected. Not requesting message!" << std::endl; +#endif + continue ; + } + + + if(mReputations->haveReputation(syncItem->authorId) || noAuthor) + { + GixsReputation rep; + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << ", author Id=" << syncItem->authorId << ". Reputation: " ; +#endif + if(!noAuthor) + mReputations->getReputation(syncItem->authorId, rep); + + // if author is required for this message, it will simply get dropped + // at genexchange side of things + if(rep.score >= (int)grpMeta->mReputationCutOff || noAuthor) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << ", passed! Adding message to req list." << std::endl; +#endif + RsNxsSyncMsgItem* msgItem = new RsNxsSyncMsgItem(mServType); + msgItem->grpId = grpId; + msgItem->msgId = msgId; + msgItem->flag = RsNxsSyncMsgItem::FLAG_REQUEST; + msgItem->transactionNumber = transN; + msgItem->PeerId(peerFrom); + reqList.push_back(msgItem); + ++reqListSize ; + } +#ifdef NXS_NET_DEBUG_1 + else + GXSNETDEBUG_PG(item->PeerId(),grpId) << ", failed!" << std::endl; +#endif + } + else + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << ", no author/no reputation. Pushed to Vetting list." << std::endl; +#endif + // preload for speed + mReputations->loadReputation(syncItem->authorId, peers); + MsgAuthEntry entry; + entry.mAuthorId = syncItem->authorId; + entry.mGrpId = syncItem->grpId; + entry.mMsgId = syncItem->msgId; + toVet.push_back(entry); + } + } +#ifdef NXS_NET_DEBUG_1 + else + GXSNETDEBUG_PG(item->PeerId(),grpId) << ". already here." << std::endl; +#endif + } + + if(!toVet.empty()) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " Vetting list: " << toVet.size() << " elements." << std::endl; +#endif + MsgRespPending* mrp = new MsgRespPending(mReputations, tr->mTransaction->PeerId(), toVet, cutoff); + mPendingResp.push_back(mrp); + } + + if(!reqList.empty()) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " Request list: " << reqList.size() << " elements." << std::endl; +#endif + locked_pushMsgTransactionFromList(reqList, tr->mTransaction->PeerId(), transN); + + if(reqListSizeExceeded) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " Marking update operation as unfinished." << std::endl; +#endif + mPartialMsgUpdates[tr->mTransaction->PeerId()].insert(item->grpId) ; + } + else + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " Marking update operation as terminal." << std::endl; +#endif + mPartialMsgUpdates[tr->mTransaction->PeerId()].erase(item->grpId) ; + } + } + else + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(item->PeerId(),grpId) << " Request list is empty. Not doing anything. " << std::endl; +#endif + // The list to req is empty. That means we already have all messages that this peer can + // provide. So we can stamp the group from this peer to be up to date. + + // Part of this is already achieved in two other places: + // - the GroupStats exchange system, which counts the messages at each peer. It could also supply TS for the messages, but it does not for the time being + // - client TS are updated when receiving messages + + locked_stampPeerGroupUpdateTime(pid,grpId,tr->mTransaction->updateTS,msgItemL.size()) ; + } +} + +void RsGxsNetService::locked_stampPeerGroupUpdateTime(const RsPeerId& pid,const RsGxsGroupId& grpId,time_t tm,uint32_t n_messages) +{ + std::map::iterator it = mClientMsgUpdateMap.find(pid) ; + + RsGxsMsgUpdateItem *pitem; + + if(it == mClientMsgUpdateMap.end()) + { + pitem = new RsGxsMsgUpdateItem(mServType) ; + pitem->peerId = pid ; + + mClientMsgUpdateMap[pid] = pitem ; + } + else + pitem = it->second ; + + pitem->msgUpdateInfos[grpId].time_stamp = tm; + pitem->msgUpdateInfos[grpId].message_count = std::max(n_messages, pitem->msgUpdateInfos[grpId].message_count) ; + + IndicateConfigChanged(); +} + +void RsGxsNetService::locked_pushGrpTransactionFromList( std::list& reqList, const RsPeerId& peerId, const uint32_t& transN) +{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peerId) << "locked_pushGrpTransactionFromList()" << std::endl; + GXSNETDEBUG_P_(peerId) << " nelems = " << reqList.size() << std::endl; + GXSNETDEBUG_P_(peerId) << " peerId = " << peerId << std::endl; + GXSNETDEBUG_P_(peerId) << " transN = " << transN << std::endl; +#endif +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_P_ (peerId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending group request to peer " + << peerId << " for " << reqList.size() << " groups" << std::endl; +#endif + RsNxsTransac* transac = new RsNxsTransac(mServType); + transac->transactFlag = RsNxsTransac::FLAG_TYPE_GRP_LIST_REQ + | RsNxsTransac::FLAG_BEGIN_P1; + transac->timestamp = 0; + transac->nItems = reqList.size(); + transac->PeerId(peerId); + transac->transactionNumber = transN; + NxsTransaction* newTrans = new NxsTransaction(); + newTrans->mItems = reqList; + newTrans->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; + newTrans->mTimeOut = time(NULL) + mTransactionTimeOut; + newTrans->mTransaction = new RsNxsTransac(*transac); + newTrans->mTransaction->PeerId(mOwnId); + sendItem(transac); + if (!locked_addTransaction(newTrans)) + delete newTrans; +} +void RsGxsNetService::addGroupItemToList(NxsTransaction*& tr, const RsGxsGroupId& grpId, uint32_t& transN, std::list& reqList) +{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << "RsGxsNetService::addGroupItemToList() Added GroupID: << grpId" << std::endl; +#endif + + RsNxsSyncGrpItem* grpItem = new RsNxsSyncGrpItem(mServType); + grpItem->PeerId(tr->mTransaction->PeerId()); + grpItem->grpId = grpId; + grpItem->flag = RsNxsSyncMsgItem::FLAG_REQUEST; + grpItem->transactionNumber = transN; + reqList.push_back(grpItem); +} + +void RsGxsNetService::locked_genReqGrpTransaction(NxsTransaction* tr) +{ + // to create a transaction you need to know who you are transacting with + // then what grps to request + // then add an active Transaction for request + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genReqGrpTransaction(): " << std::endl; +#endif + + RsGxsMetaDataTemporaryMap grpMetaMap; + + std::list grpItemL; + + for(std::list::iterator lit = tr->mItems.begin(); lit != tr->mItems.end(); ++lit) + { + RsNxsSyncGrpItem* item = dynamic_cast(*lit); + if(item) + { + grpItemL.push_back(item); + grpMetaMap[item->grpId] = NULL; + }else + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),item->grpId) << "RsGxsNetService::genReqGrpTransaction(): item failed to caste to RsNxsSyncMsgItem* " << std::endl; +#endif + } + } + + if (grpItemL.empty()) + return; + + mDataStore->retrieveGxsGrpMetaData(grpMetaMap); + + // now do compare and add loop + std::list::iterator llit = grpItemL.begin(); + std::list reqList; + + uint32_t transN = locked_getTransactionId(); + + GrpAuthorV toVet; + std::list peers; + peers.push_back(tr->mTransaction->PeerId()); + + for(; llit != grpItemL.end(); ++llit) + { + RsNxsSyncGrpItem*& grpSyncItem = *llit; + const RsGxsGroupId& grpId = grpSyncItem->grpId; + + std::map::const_iterator metaIter = grpMetaMap.find(grpId); + bool haveItem = false; + bool latestVersion = false; + + if (metaIter != grpMetaMap.end() && metaIter->second) + { + haveItem = true; + latestVersion = grpSyncItem->publishTs > metaIter->second->mPublishTs; + } + + if(!grpSyncItem->authorId.isNull() && rsReputations->isIdentityBanned(grpSyncItem->authorId)) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << " Identity " << grpSyncItem->authorId << " is banned. Not syncing group." << std::endl; +#endif + continue ; + } + + if( (mGrpAutoSync && !haveItem) || latestVersion) + { + // determine if you need to check reputation + bool checkRep = !grpSyncItem->authorId.isNull(); + + // check if you have reputation, if you don't then + // place in holding pen + if(checkRep) + { + if(mReputations->haveReputation(grpSyncItem->authorId)) + { + GixsReputation rep; + mReputations->getReputation(grpSyncItem->authorId, rep); + + if(rep.score >= GIXS_CUT_OFF) + { + addGroupItemToList(tr, grpId, transN, reqList); +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId)<< " reputation cut off: limit=" << GIXS_CUT_OFF << " value=" << rep.score << ": allowed." << std::endl; +#endif + } +#ifdef NXS_NET_DEBUG_0 + else + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId)<< " reputation cut off: limit=" << GIXS_CUT_OFF << " value=" << rep.score << ": you shall not pass." << std::endl; +#endif + } + else + { + // preload reputation for later + mReputations->loadReputation(grpSyncItem->authorId, peers); + GrpAuthEntry entry; + entry.mAuthorId = grpSyncItem->authorId; + entry.mGrpId = grpSyncItem->grpId; + toVet.push_back(entry); + } + } + else + { + addGroupItemToList(tr, grpId, transN, reqList); + } + } + } + + if(!toVet.empty()) + { + RsPeerId peerId = tr->mTransaction->PeerId(); + GrpRespPending* grp = new GrpRespPending(mReputations, peerId, toVet); + mPendingResp.push_back(grp); + } + + + if(!reqList.empty()) + locked_pushGrpTransactionFromList(reqList, tr->mTransaction->PeerId(), transN); + else + { + ClientGrpMap::iterator it = mClientGrpUpdateMap.find(tr->mTransaction->PeerId()); + RsGxsGrpUpdateItem* item = NULL; + if(it != mClientGrpUpdateMap.end()) + item = it->second; + else + { + item = new RsGxsGrpUpdateItem(mServType); + mClientGrpUpdateMap.insert(std::make_pair(tr->mTransaction->PeerId(), item)); + } +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << " reqList is empty, updating anyway ClientGrpUpdate TS for peer " << tr->mTransaction->PeerId() << " to: " << tr->mTransaction->updateTS << std::endl; +#endif + item->grpUpdateTS = tr->mTransaction->updateTS; + item->peerId = tr->mTransaction->PeerId(); + IndicateConfigChanged(); + } +} + +void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) +{ + +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendGrpsTransaction() Generating Grp data send fron TransN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + + // go groups requested in transaction tr + + std::list::iterator lit = tr->mItems.begin(); + + RsGxsMetaDataTemporaryMap grps ; + + for(;lit != tr->mItems.end(); ++lit) + { + RsNxsSyncGrpItem* item = dynamic_cast(*lit); + if (item) + grps[item->grpId] = NULL; + else + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),item->grpId) << "RsGxsNetService::locked_genSendGrpsTransaction(): item failed to caste to RsNxsSyncGrpItem* " << std::endl; +#endif + } + } + + if(!grps.empty()) + mDataStore->retrieveNxsGrps(grps, false, false); + else + return; + + NxsTransaction* newTr = new NxsTransaction(); + newTr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; + + uint32_t transN = locked_getTransactionId(); + + // store grp items to send in transaction + std::map::iterator mit = grps.begin(); + RsPeerId peerId = tr->mTransaction->PeerId(); + for(;mit != grps.end(); ++mit) + { + mit->second->PeerId(peerId); // set so it gets sent to right peer + mit->second->transactionNumber = transN; + newTr->mItems.push_back(mit->second); + mit->second = NULL ; // avoids deletion + } + + if(newTr->mItems.empty()){ + delete newTr; + return; + } + + uint32_t updateTS = 0; + if(mGrpServerUpdateItem) + updateTS = mGrpServerUpdateItem->grpUpdateTS; + +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_P_ (tr->mTransaction->PeerId()) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending global group TS " + << updateTS << " to peer " << tr->mTransaction->PeerId() << std::endl; +#endif + RsNxsTransac* ntr = new RsNxsTransac(mServType); + ntr->transactionNumber = transN; + ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_GRPS; + ntr->updateTS = updateTS; + ntr->nItems = grps.size(); + ntr->PeerId(tr->mTransaction->PeerId()); + + newTr->mTransaction = new RsNxsTransac(*ntr); + newTr->mTransaction->PeerId(mOwnId); + newTr->mTimeOut = time(NULL) + mTransactionTimeOut; + + ntr->PeerId(tr->mTransaction->PeerId()); + sendItem(ntr); + + locked_addTransaction(newTr); + + return; +} + +void RsGxsNetService::runVetting() +{ + RS_STACK_MUTEX(mNxsMutex) ; + + std::vector::iterator vit = mPendingResp.begin(); + + for(; vit != mPendingResp.end(); ) + { + AuthorPending* ap = *vit; + + if(ap->accepted() || ap->expired()) + { + // add to transactions + if(AuthorPending::MSG_PEND == ap->getType()) + { + MsgRespPending* mrp = static_cast(ap); + locked_createTransactionFromPending(mrp); + } + else if(AuthorPending::GRP_PEND == ap->getType()) + { + GrpRespPending* grp = static_cast(ap); + locked_createTransactionFromPending(grp); + }else + std::cerr << "RsGxsNetService::runVetting(): Unknown pending type! Type: " << ap->getType() << std::endl; + + delete ap; + vit = mPendingResp.erase(vit); + } + else + { + ++vit; + } + + } + + + // now lets do circle vetting + std::vector::iterator vit2 = mPendingCircleVets.begin(); + for(; vit2 != mPendingCircleVets.end(); ) + { + GrpCircleVetting*& gcv = *vit2; + if(gcv->cleared() || gcv->expired()) + { + if(gcv->getType() == GrpCircleVetting::GRP_ID_PEND) + { + GrpCircleIdRequestVetting* gcirv = + static_cast(gcv); + + locked_createTransactionFromPending(gcirv); + } + else if(gcv->getType() == GrpCircleVetting::MSG_ID_SEND_PEND) + { + MsgCircleIdsRequestVetting* mcirv = + static_cast(gcv); + + if(mcirv->cleared()) + locked_createTransactionFromPending(mcirv); + } + else + { +#ifdef NXS_NET_DEBUG_4 + std::cerr << "RsGxsNetService::runVetting(): Unknown Circle pending type! Type: " << gcv->getType() << std::endl; +#endif + } + + delete gcv; + vit2 = mPendingCircleVets.erase(vit2); + } + else + { + ++vit2; + } + } +} + +void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) +{ + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendMsgsTransaction() Generating Msg data send fron TransN: " << tr->mTransaction->transactionNumber << std::endl; +#endif + + // go groups requested in transaction tr + + std::list::iterator lit = tr->mItems.begin(); + + GxsMsgReq msgIds; + GxsMsgResult msgs; + + if(tr->mItems.empty()){ + return; + } + + // hacky assumes a transaction only consist of a single grpId + RsGxsGroupId grpId; + + for(;lit != tr->mItems.end(); ++lit) + { + RsNxsSyncMsgItem* item = dynamic_cast(*lit); + if (item) + { + msgIds[item->grpId].push_back(item->msgId); + + if(grpId.isNull()) + grpId = item->grpId; + } + else + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),grpId) << "RsGxsNetService::locked_genSendMsgsTransaction(): item failed to caste to RsNxsSyncMsgItem* " << std::endl; +#endif + } + } + + mDataStore->retrieveNxsMsgs(msgIds, msgs, false, false); + + NxsTransaction* newTr = new NxsTransaction(); + newTr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; + + uint32_t transN = locked_getTransactionId(); + + // store msg items to send in transaction + GxsMsgResult::iterator mit = msgs.begin(); + RsPeerId peerId = tr->mTransaction->PeerId(); + uint32_t msgSize = 0; + + for(;mit != msgs.end(); ++mit) + { + std::vector& msgV = mit->second; + std::vector::iterator vit = msgV.begin(); + + for(; vit != msgV.end(); ++vit) + { + RsNxsMsg* msg = *vit; + msg->PeerId(peerId); + msg->transactionNumber = transN; + +#ifndef NXS_FRAG + newTr->mItems.push_back(msg); + msgSize++; +#else + MsgFragments fragments; + fragmentMsg(*msg, fragments); + + MsgFragments::iterator mit = fragments.begin(); + + for(; mit != fragments.end(); ++mit) + { + newTr->mItems.push_back(*mit); + msgSize++; + } +#endif + } + } + + if(newTr->mItems.empty()){ + delete newTr; + return; + } + + uint32_t updateTS = 0; + + ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(grpId); + + if(cit != mServerMsgUpdateMap.end()) + updateTS = cit->second->msgUpdateTS; + + RsNxsTransac* ntr = new RsNxsTransac(mServType); + ntr->transactionNumber = transN; + ntr->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | + RsNxsTransac::FLAG_TYPE_MSGS; + ntr->updateTS = updateTS; + ntr->nItems = msgSize; + ntr->PeerId(peerId); + + newTr->mTransaction = new RsNxsTransac(*ntr); + newTr->mTransaction->PeerId(mOwnId); + newTr->mTimeOut = time(NULL) + mTransactionTimeOut; + +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_PG (peerId,grpId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending message update to peer " + << peerId << " for group " << grpId << " with TS=" << nice_time_stamp(time(NULL),updateTS) <<" (secs ago)" << std::endl; +#endif + ntr->PeerId(tr->mTransaction->PeerId()); + sendItem(ntr); + + locked_addTransaction(newTr); + + return; +} +uint32_t RsGxsNetService::locked_getTransactionId() +{ + return ++mTransactionN; +} +bool RsGxsNetService::locked_addTransaction(NxsTransaction* tr) +{ + const RsPeerId& peer = tr->mTransaction->PeerId(); + uint32_t transN = tr->mTransaction->transactionNumber; + TransactionIdMap& transMap = mTransactions[peer]; + bool transNumExist = transMap.find(transN) + != transMap.end(); + + + if(transNumExist){ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << "locked_addTransaction() " << std::endl; + GXSNETDEBUG_P_(peer) << "Transaction number exist already, transN: " << transN << std::endl; +#endif + return false; + }else{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(peer) << "locked_addTransaction() " << std::endl; + GXSNETDEBUG_P_(peer) << "Added transaction number " << transN << std::endl; +#endif + transMap[transN] = tr; + return true; + } +} + +void RsGxsNetService::cleanTransactionItems(NxsTransaction* tr) const +{ + std::list::iterator lit = tr->mItems.begin(); + + for(; lit != tr->mItems.end(); ++lit) + { + delete *lit; + } + + tr->mItems.clear(); +} + +void RsGxsNetService::locked_pushGrpRespFromList(std::list& respList, const RsPeerId& peer, const uint32_t& transN) +{ +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(peer) << "locked_pushGrpResponseFromList()" << std::endl; + GXSNETDEBUG_P_(peer) << " nelems = " << respList.size() << std::endl; + GXSNETDEBUG_P_(peer) << " peerId = " << peer << std::endl; + GXSNETDEBUG_P_(peer) << " transN = " << transN << std::endl; +#endif + NxsTransaction* tr = new NxsTransaction(); + tr->mItems = respList; + + tr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; + RsNxsTransac* trItem = new RsNxsTransac(mServType); + trItem->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 + | RsNxsTransac::FLAG_TYPE_GRP_LIST_RESP; + trItem->nItems = respList.size(); + trItem->timestamp = 0; + trItem->PeerId(peer); + trItem->transactionNumber = transN; +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_ (peer) << "Setting tr->mTransaction->updateTS to " << mGrpServerUpdateItem->grpUpdateTS << std::endl; +#endif + trItem->updateTS = mGrpServerUpdateItem->grpUpdateTS; + // also make a copy for the resident transaction + tr->mTransaction = new RsNxsTransac(*trItem); + tr->mTransaction->PeerId(mOwnId); + tr->mTimeOut = time(NULL) + mTransactionTimeOut; + // signal peer to prepare for transaction +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_P_ (peer) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending group response to peer " + << peer << " with " << respList.size() << " groups " << std::endl; +#endif + sendItem(trItem); + locked_addTransaction(tr); +} + +bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncGrp *item) +{ + // Do we have new updates for this peer? + + // This is one of the few places where we compare a local time stamp (mGrpServerUpdateItem->grpUpdateTS) to a peer's time stamp. + // Because this is the global modification time for groups, async-ed computers will eventually figure out that their data needs + // to be synced. + + if(mGrpServerUpdateItem) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(item->PeerId()) << " local modification time stamp: " << std::dec<< time(NULL) - mGrpServerUpdateItem->grpUpdateTS << " secs ago. Update sent: " << + ((item->updateTS < mGrpServerUpdateItem->grpUpdateTS)?"YES":"NO") << std::endl; +#endif + return item->updateTS < mGrpServerUpdateItem->grpUpdateTS; + } +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(item->PeerId()) << " no local time stamp. This will be fixed after updateServerSyncTS(). Not sending for now. " << std::endl; +#endif + + return false; +} + +void RsGxsNetService::handleRecvSyncGroup(RsNxsSyncGrp* item) +{ + if (!item) + return; + + RS_STACK_MUTEX(mNxsMutex) ; + + RsPeerId peer = item->PeerId(); +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(peer) << "HandleRecvSyncGroup(): Service: " << mServType << " from " << peer << ", Last update TS (from myself) sent from peer is T = " << std::dec<< time(NULL) - item->updateTS << " secs ago" << std::endl; +#endif + + if(!locked_CanReceiveUpdate(item)) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(peer) << " RsGxsNetService::handleRecvSyncGroup() update will not be sent." << std::endl; +#endif + return; + } + + RsGxsMetaDataTemporaryMap grp; + mDataStore->retrieveGxsGrpMetaData(grp); + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(peer) << " RsGxsNetService::handleRecvSyncGroup() retrieving local list of groups..." << std::endl; +#endif + if(grp.empty()) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(peer) << " RsGxsNetService::handleRecvSyncGroup() Grp Empty" << std::endl; +#endif + return; + } + + std::list itemL; + + uint32_t transN = locked_getTransactionId(); + + std::vector toVet; +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(peer) << " Group list beings being sent: " << std::endl; +#endif + + for(std::map::iterator mit = grp.begin(); mit != grp.end(); ++mit) + { + RsGxsGrpMetaData* grpMeta = mit->second; + + // Only send info about subscribed groups. + + if(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) + { + + // check if you can send this id to peer + // or if you need to add to the holding + // pen for peer to be vetted + if(canSendGrpId(peer, *grpMeta, toVet)) + { + RsNxsSyncGrpItem* gItem = new RsNxsSyncGrpItem(mServType); + gItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE; + gItem->grpId = mit->first; + gItem->publishTs = mit->second->mPublishTs; + gItem->authorId = grpMeta->mAuthorId; + gItem->PeerId(peer); + gItem->transactionNumber = transN; + itemL.push_back(gItem); +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(peer,mit->first) << " sending item for Grp " << mit->first << " name=" << grpMeta->mGroupName << ", publishTS=" << std::dec<< time(NULL) - mit->second->mPublishTs << " secs ago to peer ID " << peer << std::endl; +#endif + } + } + } + + if(!toVet.empty()) + { + mPendingCircleVets.push_back(new GrpCircleIdRequestVetting(mCircles, mPgpUtils, toVet, peer)); + } + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_P_(peer) << " final list sent (after vetting): " << itemL.size() << " elements." << std::endl; +#endif + locked_pushGrpRespFromList(itemL, peer, transN); + + return; +} + + + +bool RsGxsNetService::canSendGrpId(const RsPeerId& sslId, RsGxsGrpMetaData& grpMeta, std::vector& toVet) +{ +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId()"<< std::endl; +#endif + // first do the simple checks + uint8_t circleType = grpMeta.mCircleType; + + if(circleType == GXS_CIRCLE_TYPE_LOCAL) + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId)<< "RsGxsNetService::canSendGrpId() LOCAL_CIRCLE, cannot send"<< std::endl; +#endif + return false; + } + + if(circleType == GXS_CIRCLE_TYPE_PUBLIC) + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId)<< "RsGxsNetService::canSendGrpId() PUBLIC_CIRCLE, can send"<< std::endl; +#endif + return true; + } + + if(circleType == GXS_CIRCLE_TYPE_EXTERNAL) + { + const RsGxsCircleId& circleId = grpMeta.mCircleId; + if(circleId.isNull()) + { + std::cerr << "RsGxsNetService::canSendGrpId() ERROR; EXTERNAL_CIRCLE missing NULL CircleId: " << grpMeta.mGroupId<< std::endl; + + // ERROR, will never be shared. + return false; + } + + if(mCircles->isLoaded(circleId)) + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId)<< "RsGxsNetService::canSendGrpId() EXTERNAL_CIRCLE, checking mCircles->canSend"<< std::endl; +#endif + const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); + return mCircles->canSend(circleId, pgpId); + } + + toVet.push_back(GrpIdCircleVet(grpMeta.mGroupId, circleId, grpMeta.mAuthorId)); + return false; + } + + if(circleType == GXS_CIRCLE_TYPE_YOUREYESONLY) + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId)<< "RsGxsNetService::canSendGrpId() YOUREYESONLY, checking further"<< std::endl; +#endif + // a non empty internal circle id means this + // is the personal circle owner + if(!grpMeta.mInternalCircle.isNull()) + { + const RsGxsCircleId& internalCircleId = grpMeta.mInternalCircle; +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() have mInternalCircle - we are Group creator" << std::endl; + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() mCircleId: " << grpMeta.mCircleId << std::endl; + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() mInternalCircle: " << grpMeta.mInternalCircle << std::endl; +#endif + + if(mCircles->isLoaded(internalCircleId)) + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() circle Loaded - checking mCircles->canSend" << std::endl; +#endif + const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); + return mCircles->canSend(internalCircleId, pgpId); + } + +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() Circle Not Loaded - add to vetting"<< std::endl; +#endif + toVet.push_back(GrpIdCircleVet(grpMeta.mGroupId, internalCircleId, grpMeta.mAuthorId)); + return false; + } + else + { + // an empty internal circle id means this peer can only + // send circle related info from peer he received it +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() mInternalCircle not set, someone else's personal circle"<< std::endl; +#endif + if(grpMeta.mOriginator == sslId) + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() Originator matches -> can send"<< std::endl; +#endif + return true; + } + else + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId() Originator doesn't match -> cannot send"<< std::endl; +#endif + return false; + } + } + } + + return true; +} + +bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxsGrpMetaData& grpMeta) +{ + + #ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::checkCanRecvMsgFromPeer()"; + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " peer Id = " << sslId << ", grpId=" << grpMeta.mGroupId <isLoaded(circleId)) + { + #ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " EXTERNAL_CIRCLE, checking mCircles->canSend" << std::endl; + #endif + const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); + return mCircles->canSend(circleId, pgpId); + } + else + mCircles->loadCircle(circleId); // simply request for next pass + + return false; + } + + if(circleType == GXS_CIRCLE_TYPE_YOUREYESONLY) // do not attempt to sync msg unless to originator or those permitted + { + #ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " YOUREYESONLY, checking further" << std::endl; + #endif + // a non empty internal circle id means this + // is the personal circle owner + if(!grpMeta.mInternalCircle.isNull()) + { + const RsGxsCircleId& internalCircleId = grpMeta.mInternalCircle; + #ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " have mInternalCircle - we are Group creator" << std::endl; + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " mCircleId: " << grpMeta.mCircleId << std::endl; + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " mInternalCircle: " << grpMeta.mInternalCircle << std::endl; + #endif + + if(mCircles->isLoaded(internalCircleId)) + { + #ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " circle Loaded - checking mCircles->canSend" << std::endl; + #endif + const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); + return mCircles->canSend(internalCircleId, pgpId); + } + else + mCircles->loadCircle(internalCircleId); // request for next pass + + return false; + } + else + { + // an empty internal circle id means this peer can only + // send circle related info from peer he received it + #ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " mInternalCircle not set, someone else's personal circle" << std::endl; + #endif + if(grpMeta.mOriginator == sslId) + { + #ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Originator matches -> can send" << std::endl; + #endif + return true; + } + else + { + #ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Originator doesn't match -> cannot send"<< std::endl; + #endif + return false; + } + } + } + + return true; +} + +bool RsGxsNetService::locked_CanReceiveUpdate(const RsNxsSyncMsg *item) +{ + // Do we have new updates for this peer? + // Here we compare times in the same clock: the friend's clock, so it should be fine. + + ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(item->grpId); + + if(cit != mServerMsgUpdateMap.end()) + { + const RsGxsServerMsgUpdateItem *msui = cit->second; + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " local time stamp: " << std::dec<< time(NULL) - msui->msgUpdateTS << " secs ago. Update sent: " << (item->updateTS < msui->msgUpdateTS) ; +#endif + return item->updateTS < msui->msgUpdateTS ; + } +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " no local time stamp for this grp. " ; +#endif + + return false; +} +void RsGxsNetService::handleRecvSyncMessage(RsNxsSyncMsg* item) +{ + if (!item) + return; + + RS_STACK_MUTEX(mNxsMutex) ; + + const RsPeerId& peer = item->PeerId(); + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << "handleRecvSyncMsg(): Received last update TS of group " << item->grpId << ", for peer " << peer << ", TS = " << time(NULL) - item->updateTS << " secs ago." ; +#endif + + if(!locked_CanReceiveUpdate(item)) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " no update will be sent." << std::endl; +#endif + return; + } + + GxsMsgMetaResult metaResult; + GxsMsgReq req; + + RsGxsMetaDataTemporaryMap grpMetas; + grpMetas[item->grpId] = NULL; + + mDataStore->retrieveGxsGrpMetaData(grpMetas); + RsGxsGrpMetaData* grpMeta = grpMetas[item->grpId]; + + if(grpMeta == NULL) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " Grp is unknown." << std::endl; +#endif + return; + } + if(!(grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED )) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " Grp is not subscribed." << std::endl; +#endif + return ; + } + + req[item->grpId] = std::vector(); + mDataStore->retrieveGxsMsgMetaData(req, metaResult); + std::vector& msgMetas = metaResult[item->grpId]; + +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " retrieving message meta data." << std::endl; +#endif + if(req.empty()) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " No msg meta data.." << std::endl; +#endif + } +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " Sending MSG meta data!" << std::endl; +#endif + + std::list itemL; + + uint32_t transN = locked_getTransactionId(); + + if(canSendMsgIds(msgMetas, *grpMeta, peer)) + { + std::vector::iterator vit = msgMetas.begin(); + + for(; vit != msgMetas.end(); ++vit) + { + RsGxsMsgMetaData* m = *vit; + + RsNxsSyncMsgItem* mItem = new RsNxsSyncMsgItem(mServType); + mItem->flag = RsNxsSyncGrpItem::FLAG_RESPONSE; + mItem->grpId = m->mGroupId; + mItem->msgId = m->mMsgId; + mItem->authorId = m->mAuthorId; + mItem->PeerId(peer); + mItem->transactionNumber = transN; + itemL.push_back(mItem); +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " sending info item for msg id " << mItem->msgId << std::endl; +#endif + } + + if(!itemL.empty()) + { +#ifdef NXS_NET_DEBUG_0 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " sending final msg info list of " << itemL.size() << " items." << std::endl; +#endif + locked_pushMsgRespFromList(itemL, peer, item->grpId,transN); + } + } +#ifdef NXS_NET_DEBUG_0 + else + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " vetting forbids sending. Nothing will be sent." << itemL.size() << " items." << std::endl; +#endif + + std::vector::iterator vit = msgMetas.begin(); + // release meta resource + for(vit = msgMetas.begin(); vit != msgMetas.end(); ++vit) + delete *vit; +} + +void RsGxsNetService::locked_pushMsgRespFromList(std::list& itemL, const RsPeerId& sslId, const RsGxsGroupId& grp_id,const uint32_t& transN) +{ +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(sslId,grp_id) << "locked_pushMsgResponseFromList()" << std::endl; + GXSNETDEBUG_PG(sslId,grp_id) << " nelems = " << itemL.size() << std::endl; + GXSNETDEBUG_PG(sslId,grp_id) << " peerId = " << sslId << std::endl; + GXSNETDEBUG_PG(sslId,grp_id) << " transN = " << transN << std::endl; +#endif + NxsTransaction* tr = new NxsTransaction(); + tr->mItems = itemL; + tr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; + RsNxsTransac* trItem = new RsNxsTransac(mServType); + trItem->transactFlag = RsNxsTransac::FLAG_BEGIN_P1 | RsNxsTransac::FLAG_TYPE_MSG_LIST_RESP; + trItem->nItems = itemL.size(); + trItem->timestamp = 0 ; + trItem->PeerId(sslId); + trItem->transactionNumber = transN; + + ServerMsgMap::const_iterator cit = mServerMsgUpdateMap.find(grp_id); + + // This time stamp is not supposed to be used on the other side. We just set it to avoid sending an uninitialiszed value. + + if(cit != mServerMsgUpdateMap.end()) + trItem->updateTS = cit->second->msgUpdateTS; + else + { + std::cerr << "(EE) cannot find a server TS for message of group " << grp_id << " in locked_pushMsgRespFromList. This is weird." << std::endl; + trItem->updateTS = 0 ; + } + + // also make a copy for the resident transaction + tr->mTransaction = new RsNxsTransac(*trItem); + tr->mTransaction->PeerId(mOwnId); + tr->mTimeOut = time(NULL) + mTransactionTimeOut; + +#ifdef NXS_NET_DEBUG_5 + GXSNETDEBUG_P_ (sslId) << "Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " - sending messages response to peer " + << sslId << " with " << itemL.size() << " messages " << std::endl; +#endif + // signal peer to prepare for transaction + sendItem(trItem); + + locked_addTransaction(tr); +} + +bool RsGxsNetService::canSendMsgIds(const std::vector& msgMetas, + const RsGxsGrpMetaData& grpMeta, const RsPeerId& sslId) +{ +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG__G(grpMeta.mGroupId) << "RsGxsNetService::canSendMsgIds() CIRCLE VETTING" << std::endl; +#endif + + // first do the simple checks + uint8_t circleType = grpMeta.mCircleType; + + if(circleType == GXS_CIRCLE_TYPE_LOCAL) + return false; + + if(circleType == GXS_CIRCLE_TYPE_PUBLIC) + return true; + + const RsGxsCircleId& circleId = grpMeta.mCircleId; + + if(circleType == GXS_CIRCLE_TYPE_EXTERNAL) + { + if(mCircles->isLoaded(circleId)) + { + const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); + return mCircles->canSend(circleId, pgpId); + } + + std::vector toVet; + std::vector::const_iterator vit = msgMetas.begin(); + + for(; vit != msgMetas.end(); ++vit) + { + const RsGxsMsgMetaData* const& meta = *vit; + + MsgIdCircleVet mic(meta->mMsgId, meta->mAuthorId); + toVet.push_back(mic); + } + + if(!toVet.empty()) + mPendingCircleVets.push_back(new MsgCircleIdsRequestVetting(mCircles, mPgpUtils, toVet, grpMeta.mGroupId, + sslId, grpMeta.mCircleId)); + + return false; + } + + if(circleType == GXS_CIRCLE_TYPE_YOUREYESONLY) + { + // a non empty internal circle id means this + // is the personal circle owner + if(!grpMeta.mInternalCircle.isNull()) + { + const RsGxsCircleId& internalCircleId = grpMeta.mInternalCircle; + if(mCircles->isLoaded(internalCircleId)) + { + const RsPgpId& pgpId = mPgpUtils->getPGPId(sslId); + return mCircles->canSend(internalCircleId, pgpId); + } + + std::vector toVet; + std::vector::const_iterator vit = msgMetas.begin(); + + for(; vit != msgMetas.end(); ++vit) + { + const RsGxsMsgMetaData* const& meta = *vit; + + MsgIdCircleVet mic(meta->mMsgId, meta->mAuthorId); + toVet.push_back(mic); + } + + if(!toVet.empty()) + mPendingCircleVets.push_back(new MsgCircleIdsRequestVetting(mCircles, mPgpUtils, + toVet, grpMeta.mGroupId, + sslId, grpMeta.mCircleId)); + + return false; + } + else + { + // an empty internal circle id means this peer can only + // send circle related info from peer he received it + if(grpMeta.mOriginator == sslId) + return true; + else + return false; + } + } + + return true; +} + +/** inherited methods **/ + +void RsGxsNetService::pauseSynchronisation(bool /* enabled */) +{ + +} + +void RsGxsNetService::setSyncAge(uint32_t /* age */) +{ + +} + +int RsGxsNetService::requestGrp(const std::list& grpId, const RsPeerId& peerId) +{ + RS_STACK_MUTEX(mNxsMutex) ; + mExplicitRequest[peerId].assign(grpId.begin(), grpId.end()); + return 1; +} + +void RsGxsNetService::processExplicitGroupRequests() +{ + RS_STACK_MUTEX(mNxsMutex) ; + + std::map >::const_iterator cit = mExplicitRequest.begin(); + + for(; cit != mExplicitRequest.end(); ++cit) + { + const RsPeerId& peerId = cit->first; + const std::list& groupIdList = cit->second; + + std::list grpSyncItems; + std::list::const_iterator git = groupIdList.begin(); + uint32_t transN = locked_getTransactionId(); + for(; git != groupIdList.end(); ++git) + { + RsNxsSyncGrpItem* item = new RsNxsSyncGrpItem(mServType); + item->grpId = *git; + item->PeerId(peerId); + item->flag = RsNxsSyncGrpItem::FLAG_REQUEST; + item->transactionNumber = transN; + grpSyncItems.push_back(item); + } + + if(!grpSyncItems.empty()) + locked_pushGrpTransactionFromList(grpSyncItems, peerId, transN); + } + + mExplicitRequest.clear(); +} + +int RsGxsNetService::sharePublishKey(const RsGxsGroupId& grpId,const std::set& peers) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + mPendingPublishKeyRecipients[grpId] = peers ; +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG__G(grpId) << "RsGxsNetService::sharePublishKeys() " << (void*)this << " adding publish keys for grp " << grpId << " to sending list" << std::endl; +#endif + + return true ; +} + +void RsGxsNetService::sharePublishKeysPending() +{ + RS_STACK_MUTEX(mNxsMutex) ; + + if(mPendingPublishKeyRecipients.empty()) + return ; + +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG___ << "RsGxsNetService::sharePublishKeys() " << (void*)this << std::endl; +#endif + // get list of peers that are online + + std::set peersOnline; + std::list toDelete; + std::map >::iterator mit ; + + mNetMgr->getOnlineList(mServiceInfo.mServiceType, peersOnline); + +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG___ << " " << peersOnline.size() << " peers online." << std::endl; +#endif + /* send public key to peers online */ + + for(mit = mPendingPublishKeyRecipients.begin(); mit != mPendingPublishKeyRecipients.end(); ++mit) + { + // Compute the set of peers to send to. We start with this, to avoid retrieving the data for nothing. + + std::list recipients ; + std::set offline_recipients ; + + for(std::set::const_iterator it(mit->second.begin());it!=mit->second.end();++it) + if(peersOnline.find(*it) != peersOnline.end()) + { +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG_P_(*it) << " " << *it << ": online. Adding." << std::endl; +#endif + recipients.push_back(*it) ; + } + else + { +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG_P_(*it) << " " << *it << ": offline. Keeping for next try." << std::endl; +#endif + offline_recipients.insert(*it) ; + } + + // If empty, skip + + if(recipients.empty()) + { +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG___ << " No recipients online. Skipping." << std::endl; +#endif + continue ; + } + + // Get the meta data for this group Id + // + RsGxsMetaDataTemporaryMap grpMetaMap; + grpMetaMap[mit->first] = NULL; + mDataStore->retrieveGxsGrpMetaData(grpMetaMap); + + // Find the publish keys in the retrieved info + + RsGxsGrpMetaData *grpMeta = grpMetaMap[mit->first] ; + + if(grpMeta == NULL) + { + std::cerr << "(EE) RsGxsNetService::sharePublishKeys() Publish keys cannot be found for group " << mit->first << std::endl; + continue ; + } + + const RsTlvSecurityKeySet& keys = grpMeta->keys; + + std::map::const_iterator kit = keys.keys.begin(), kit_end = keys.keys.end(); + bool publish_key_found = false; + RsTlvSecurityKey publishKey ; + + for(; kit != kit_end && !publish_key_found; ++kit) + { + publish_key_found = (kit->second.keyFlags == (RSTLV_KEY_DISTRIB_PUBLISH | RSTLV_KEY_TYPE_FULL)); + publishKey = kit->second ; + } + + if(!publish_key_found) + { + std::cerr << "(EE) no publish key in group " << mit->first << ". Cannot share!" << std::endl; + continue ; + } + + +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG__G(grpMeta->mGroupId) << " using publish key ID=" << publishKey.keyId << ", flags=" << publishKey.keyFlags << std::endl; +#endif + for(std::list::const_iterator it(recipients.begin());it!=recipients.end();++it) + { + /* Create publish key sharing item */ + RsNxsGroupPublishKeyItem *publishKeyItem = new RsNxsGroupPublishKeyItem(mServType); + + publishKeyItem->clear(); + publishKeyItem->grpId = mit->first; + + publishKeyItem->key = publishKey ; + publishKeyItem->PeerId(*it); + + sendItem(publishKeyItem); +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG_PG(*it,grpMeta->mGroupId) << " sent key item to " << *it << std::endl; +#endif + } + + mit->second = offline_recipients ; + + // If given peers have all received key(s) then stop sending for group + if(offline_recipients.empty()) + toDelete.push_back(mit->first); + } + + // delete pending peer list which are done with + for(std::list::const_iterator lit = toDelete.begin(); lit != toDelete.end(); ++lit) + mPendingPublishKeyRecipients.erase(*lit); +} + +void RsGxsNetService::handleRecvPublishKeys(RsNxsGroupPublishKeyItem *item) +{ +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << "RsGxsNetService::sharePublishKeys() " << std::endl; +#endif + + if (!item) + return; + + RS_STACK_MUTEX(mNxsMutex) ; + +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " PeerId : " << item->PeerId() << std::endl; + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " GrpId: " << item->grpId << std::endl; + GXSNETDEBUG_PG(item->PeerId(),item->grpId) << " Got key Item: " << item->key.keyId << std::endl; +#endif + + // Get the meta data for this group Id + // + RsGxsMetaDataTemporaryMap grpMetaMap; + grpMetaMap[item->grpId] = NULL; + + mDataStore->retrieveGxsGrpMetaData(grpMetaMap); + + // update the publish keys in this group meta info + + RsGxsGrpMetaData *grpMeta = grpMetaMap[item->grpId] ; + + // Check that the keys correspond, and that FULL keys are supplied, etc. + +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " Key received: " << std::endl; +#endif + + bool admin = (item->key.keyFlags & RSTLV_KEY_DISTRIB_ADMIN) && (item->key.keyFlags & RSTLV_KEY_TYPE_FULL) ; + bool publi = (item->key.keyFlags & RSTLV_KEY_DISTRIB_PUBLISH) && (item->key.keyFlags & RSTLV_KEY_TYPE_FULL) ; + +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " Key id = " << item->key.keyId << " admin=" << admin << ", publish=" << publi << " ts=" << item->key.endTS << std::endl; +#endif + + if(!(!admin && publi)) + { + std::cerr << " Key is not a publish private key. Discarding!" << std::endl; + return ; + } + // Also check that we don't already have full keys for that group. + + std::map::iterator it = grpMeta->keys.keys.find(item->key.keyId) ; + + if(it == grpMeta->keys.keys.end()) + { + std::cerr << " (EE) Key not found in known group keys. This is an inconsistency." << std::endl; + return ; + } + + if((it->second.keyFlags & RSTLV_KEY_DISTRIB_PUBLISH) && (it->second.keyFlags & RSTLV_KEY_TYPE_FULL)) + { +#ifdef NXS_NET_DEBUG_3 + GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " (EE) Publish key already present in database. Discarding message." << std::endl; +#endif + return ; + } + + // Store/update the info. + + it->second = item->key ; + bool ret = mDataStore->updateGroupKeys(item->grpId,grpMeta->keys, grpMeta->mSubscribeFlags | GXS_SERV::GROUP_SUBSCRIBE_PUBLISH) ; + + if(ret) + { +#ifdef NXS_NET_DEBUG + GXSNETDEBUG_PG(item->PeerId(),item->grpId)<< " updated database with new publish keys." << std::endl; +#endif + mObserver->notifyReceivePublishKey(item->grpId); + } + else + { + std::cerr << "(EE) could not update database. Something went wrong." << std::endl; + } +} diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 1719371e0..7cdb764ce 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -1,536 +1,538 @@ -#ifndef RSGXSNETSERVICE_H -#define RSGXSNETSERVICE_H - -/* - * libretroshare/src/gxs: rsgxnetservice.h - * - * Access to rs network and synchronisation service implementation - * - * Copyright 2012-2012 by Christopher Evi-Parker - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License Version 2 as published by the Free Software Foundation. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the Free Software - * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 - * USA. - * - * Please report all bugs and problems to "retroshare@lunamutt.com". - * - */ - -#include -#include - -#include "rsnxs.h" -#include "rsgds.h" -#include "rsnxsobserver.h" -#include "pqi/p3linkmgr.h" -#include "serialiser/rsnxsitems.h" -#include "serialiser/rsgxsupdateitems.h" -#include "rsgxsnetutils.h" -#include "pqi/p3cfgmgr.h" -#include "rsgixs.h" -#include "util/rssharedptr.h" - -/// keep track of transaction number -typedef std::map TransactionIdMap; - -/// to keep track of peers active transactions -typedef std::map TransactionsPeerMap; - -class PgpAuxUtils; - -class RsGroupNetworkStatsRecord -{ - public: - RsGroupNetworkStatsRecord() { max_visible_count= 0 ; update_TS=0; } - - std::set suppliers ; - uint32_t max_visible_count ; - time_t update_TS ; -}; - -/*! - * This class implements the RsNetWorkExchangeService - * using transactions to handle synchrnisation of Nxs items between - * peers in a network - * Transactions requires the maintenance of several states between peers - * - * Thus a data structure maintains: peers, and their active transactions - * Then for each transaction it needs to be noted if this is an outgoing or incoming transaction - * Outgoing transaction are in 3 different states: - * 1. START 2. INITIATED 3. SENDING 4. END - * Incoming transaction are in 3 different states - * 1. START 2. RECEIVING 3. END - */ -class RsGxsNetService : public RsNetworkExchangeService, public p3ThreadedService, - public p3Config -{ -public: - - typedef RsSharedPtr pointer; - - static const uint32_t FRAGMENT_SIZE; - /*! - * only one observer is allowed - * @param servType service type - * @param gds The data service which allows read access to a service/store - * @param nxsObs observer will be notified whenever new messages/grps - * @param nxsObs observer will be notified whenever new messages/grps - * arrive - */ - RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, - RsNxsNetMgr *netMgr, - RsNxsObserver *nxsObs, // used to be = NULL. - const RsServiceInfo serviceInfo, - RsGixsReputation* reputations = NULL, RsGcxs* circles = NULL, - PgpAuxUtils *pgpUtils = NULL, - bool grpAutoSync = true, bool msgAutoSync = true); - - virtual ~RsGxsNetService(); - - virtual RsServiceInfo getServiceInfo() { return mServiceInfo; } - -public: - - - /*! - * Use this to set how far back synchronisation of messages should take place - * @param age the max age a sync item can to be allowed in a synchronisation - */ - // NOT IMPLEMENTED - virtual void setSyncAge(uint32_t age); - - /*! - * pauses synchronisation of subscribed groups and request for group id - * from peers - * @param enabled set to false to disable pause, and true otherwise - */ - // NOT IMPLEMENTED - virtual void pauseSynchronisation(bool enabled); - - - /*! - * Request for this message is sent through to peers on your network - * and how many hops from them you've indicated - * @param msgId the messages to retrieve - * @return request token to be redeemed - */ - virtual int requestMsg(const RsGxsGrpMsgIdPair& /* msgId */){ return 0;} - - /*! - * Request for this group is sent through to peers on your network - * and how many hops from them you've indicated - * @param enabled set to false to disable pause, and true otherwise - * @return request token to be redeemed - */ - virtual int requestGrp(const std::list& grpId, const RsPeerId& peerId); - - /*! - * share publish keys for the specified group with the peers in the specified list. - */ - - virtual int sharePublishKey(const RsGxsGroupId& grpId,const std::set& peers) ; - - /*! - * Returns statistics for the group networking activity: popularity (number of friends subscribers) and max_visible_msg_count, - * that is the max nnumber of messages reported by a friend. - */ - virtual bool getGroupNetworkStats(const RsGxsGroupId& id,RsGroupNetworkStats& stats) ; - - /*! - * Used to inform the net service that we changed subscription status. That helps - * optimising data transfer when e.g. unsubsribed groups are updated less often, etc - */ - virtual void subscribeStatusChanged(const RsGxsGroupId& id,bool subscribed) ; - - virtual void rejectMessage(const RsGxsMessageId& msg_id) ; - - /* p3Config methods */ -public: - - bool loadList(std::list& load); - bool saveList(bool &cleanup, std::list&); - RsSerialiser *setupSerialiser(); - -public: - - /*! - * initiates synchronisation - */ - int tick(); - - /*! - * Processes transactions and job queue - */ - virtual void data_tick(); -private: - - /*! - * called when - * items are deemed to be waiting in p3Service item queue - */ - void recvNxsItemQueue(); - - - /** S: Transaction processing **/ - - /*! - * These process transactions which are in a wait state - * Also moves transaction which have been completed to - * the completed transactions list - */ - void processTransactions(); - - /*! - * Process completed transaction, which either simply - * retires a transaction or additionally generates a response - * to the completed transaction - */ - void processCompletedTransactions(); - - /*! - * Process transaction owned/started by user - * @param tr transaction to process, ownership stays with callee - */ - void locked_processCompletedOutgoingTrans(NxsTransaction* tr); - - /*! - * Process transactions started/owned by other peers - * @param tr transaction to process, ownership stays with callee - */ - void locked_processCompletedIncomingTrans(NxsTransaction* tr); - - - /*! - * Process a transaction item, assumes a general lock - * @param item the transaction item to process - * @return false ownership of item left with callee - */ - bool locked_processTransac(RsNxsTransac* item); - - /*! - * This adds a transaction - * completeted transaction list - * If this is an outgoing transaction, transaction id is - * decrement - * @param trans transaction to add - */ - void locked_completeTransaction(NxsTransaction* trans); - - /*! - * This retrieves a unique transaction id that - * can be used in an outgoing transaction - */ - uint32_t locked_getTransactionId(); - - /*! - * This attempts to push the transaction id counter back if you have - * active outgoing transactions in play - */ - bool attemptRecoverIds(); - - /*! - * The cb listener is the owner of the grps - * @param grps - */ - //void notifyListenerGrps(std::list& grps); - - /*! - * The cb listener is the owner of the msgs - * @param msgs - */ - //void notifyListenerMsgs(std::list& msgs); - - /*! - * Generates new transaction to send msg requests based on list - * of msgs received from peer stored in passed transaction - * @param tr transaction responsible for generating msg request - */ - void locked_genReqMsgTransaction(NxsTransaction* tr); - - /*! - * Generates new transaction to send grp requests based on list - * of grps received from peer stored in passed transaction - * @param tr transaction responsible for generating grp request - */ - void locked_genReqGrpTransaction(NxsTransaction* tr); - - /*! - * This first checks if one can send a grpId based circles - * If it can send, then it call locked_genSendMsgsTransaction - * @param tr transaction responsible for generating grp request - * @see locked_genSendMsgsTransaction - */ - void locked_checkSendMsgsTransaction(NxsTransaction* tr); - - /*! - * Generates new transaction to send msg data based on list - * of grpids received from peer stored in passed transaction - * @param tr transaction responsible for generating grp request - */ - void locked_genSendMsgsTransaction(NxsTransaction* tr); - - /*! - * Generates new transaction to send grp data based on list - * of grps received from peer stored in passed transaction - * @param tr transaction responsible for generating grp request - */ - void locked_genSendGrpsTransaction(NxsTransaction* tr); - - /*! - * convenience function to add a transaction to list - * @param tr transaction to add - */ - bool locked_addTransaction(NxsTransaction* tr); - - void cleanTransactionItems(NxsTransaction* tr) const; - - /*! - * @param tr the transaction to check for timeout - * @return false if transaction has timed out, true otherwise - */ - bool locked_checkTransacTimedOut(NxsTransaction* tr); - - /** E: Transaction processing **/ - - /** S: item handlers **/ - - /*! - * This attempts handles transaction items - * ownership of item is left with callee if this method returns false - * @param item transaction item to handle - * @return false if transaction could not be handled, ownership of item is left with callee - */ - bool handleTransaction(RsNxsItem* item); - - /*! - * Handles an nxs item for group synchronisation - * by startin a transaction and sending a list - * of groups held by user - * @param item contains grp sync info - */ - void handleRecvSyncGroup(RsNxsSyncGrp* item); - - /*! - * Handles an nxs item for group statistics - * @param item contaims update time stamp and number of messages - */ - void handleRecvSyncGrpStatistics(RsNxsSyncGrpStats *grs); - - /*! - * Handles an nxs item for msgs synchronisation - * @param item contaims msg sync info - */ - void handleRecvSyncMessage(RsNxsSyncMsg* item); - - /*! - * Handles an nxs item for group publish key - * @param item contaims keys/grp info - */ - void handleRecvPublishKeys(RsNxsGroupPublishKeyItem*) ; - - /** E: item handlers **/ - - - void runVetting(); - - /*! - * @param peerId The peer to vet to see if they can receive this groupid - * @param grpMeta this is the meta item to determine if it can be sent to given peer - * @param toVet groupid/peer to vet are stored here if their circle id is not cached - * @return false, if you cannot send to this peer, true otherwise - */ - bool canSendGrpId(const RsPeerId& sslId, RsGxsGrpMetaData& grpMeta, std::vector& toVet); - - - bool canSendMsgIds(const std::vector& msgMetas, const RsGxsGrpMetaData&, const RsPeerId& sslId); - - bool checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxsGrpMetaData& meta); - - void locked_createTransactionFromPending(MsgRespPending* grpPend); - void locked_createTransactionFromPending(GrpRespPending* msgPend); - void locked_createTransactionFromPending(GrpCircleIdRequestVetting* grpPend); - void locked_createTransactionFromPending(MsgCircleIdsRequestVetting* grpPend); - - void locked_pushMsgTransactionFromList(std::list& reqList, const RsPeerId& peerId, const uint32_t& transN); - void locked_pushGrpTransactionFromList(std::list& reqList, const RsPeerId& peerId, const uint32_t& transN); - void locked_pushGrpRespFromList(std::list& respList, const RsPeerId& peer, const uint32_t& transN); - void locked_pushMsgRespFromList(std::list& itemL, const RsPeerId& sslId, const RsGxsGroupId &grp_id, const uint32_t& transN); - void syncWithPeers(); - void syncGrpStatistics(); - void addGroupItemToList(NxsTransaction*& tr, - const RsGxsGroupId& grpId, uint32_t& transN, - std::list& reqList); - - //bool locked_canReceive(const RsGxsGrpMetaData * const grpMeta, const RsPeerId& peerId); - - void processExplicitGroupRequests(); - - void locked_doMsgUpdateWork(const RsNxsTransac* nxsTrans, const RsGxsGroupId& grpId); - - void updateServerSyncTS(); - void updateClientSyncTS(); - - bool locked_CanReceiveUpdate(const RsNxsSyncGrp* item); - bool locked_CanReceiveUpdate(const RsNxsSyncMsg* item); - -private: - - typedef std::vector GrpFragments; - typedef std::vector MsgFragments; - - /*! - * Loops over pending publish key orders. - */ - void sharePublishKeysPending() ; - - /*! - * Fragment a message into individual fragments which are at most 150kb - * @param msg message to fragment - * @param msgFragments fragmented message - * @return false if fragmentation fails true otherwise - */ - bool fragmentMsg(RsNxsMsg& msg, MsgFragments& msgFragments) const; - - /*! - * Fragment a group into individual fragments which are at most 150kb - * @param grp group to fragment - * @param grpFragments fragmented group - * @return false if fragmentation fails true other wise - */ - bool fragmentGrp(RsNxsGrp& grp, GrpFragments& grpFragments) const; - - /*! - * Fragment a message into individual fragments which are at most 150kb - * @param msg message to fragment - * @param msgFragments fragmented message - * @return NULL if not possible to reconstruct message from fragment, - * pointer to defragments nxs message is possible - */ - RsNxsMsg* deFragmentMsg(MsgFragments& msgFragments) const; - - /*! - * Fragment a group into individual fragments which are at most 150kb - * @param grp group to fragment - * @param grpFragments fragmented group - * @return NULL if not possible to reconstruct group from fragment, - * pointer to defragments nxs group is possible - */ - RsNxsGrp* deFragmentGrp(GrpFragments& grpFragments) const; - - - /*! - * Note that if all fragments for a message are not found then its fragments are dropped - * @param fragments message fragments which are not necessarily from the same message - * @param partFragments the partitioned fragments (into message ids) - */ - void collateMsgFragments(MsgFragments fragments, std::map& partFragments) const; - - /*! - * Note that if all fragments for a group are not found then its fragments are dropped - * @param fragments group fragments which are not necessarily from the same group - * @param partFragments the partitioned fragments (into message ids) - */ - void collateGrpFragments(GrpFragments fragments, std::map& partFragments) const; - - /*! - * stamp the group info from that particular peer at the given time. - */ - - void locked_stampPeerGroupUpdateTime(const RsPeerId& pid,const RsGxsGroupId& grpId,time_t tm,uint32_t n_messages) ; - - void cleanRejectedMessages(); - void processObserverNotifications(); -private: - - - /*** transactions ***/ - - /// active transactions - TransactionsPeerMap mTransactions; - - /// completed transactions - std::list mComplTransactions; - - /// transaction id counter - uint32_t mTransactionN; - - /*** transactions ***/ - - /*** synchronisation ***/ - std::list mSyncGrp; - std::list mSyncMsg; - /*** synchronisation ***/ - - RsNxsObserver* mObserver; - RsGeneralDataService* mDataStore; - uint16_t mServType; - - // how much time must elapse before a timeout failure - // for an active transaction - uint32_t mTransactionTimeOut; - - RsPeerId mOwnId; - - RsNxsNetMgr* mNetMgr; - - /// for other members save transactions - RsMutex mNxsMutex; - - uint32_t mSyncTs; - uint32_t mLastKeyPublishTs; - uint32_t mLastCleanRejectedMessages; - - const uint32_t mSYNC_PERIOD; - int mUpdateCounter ; - - RsGcxs* mCircles; - RsGixsReputation* mReputations; - PgpAuxUtils *mPgpUtils; - bool mGrpAutoSync; - bool mAllowMsgSync; - - // need to be verfied - std::vector mPendingResp; - std::vector mPendingCircleVets; - std::map > mPendingPublishKeyRecipients ; - std::map > mExplicitRequest; - std::map > mPartialMsgUpdates ; - - // nxs sync optimisation - // can pull dynamically the latest timestamp for each message - -public: - - typedef std::map ClientMsgMap; - typedef std::map ServerMsgMap; - typedef std::map ClientGrpMap; - -private: - - ClientMsgMap mClientMsgUpdateMap; - ServerMsgMap mServerMsgUpdateMap; - ClientGrpMap mClientGrpUpdateMap; - - std::map mGroupNetworkStats ; - - RsGxsServerGrpUpdateItem* mGrpServerUpdateItem; - RsServiceInfo mServiceInfo; - - std::map mRejectedMessages; - std::vector mNewGroupsToNotify ; - std::vector mNewMessagesToNotify ; - - void debugDump(); -}; - -#endif // RSGXSNETSERVICE_H +#ifndef RSGXSNETSERVICE_H +#define RSGXSNETSERVICE_H + +/* + * libretroshare/src/gxs: rsgxnetservice.h + * + * Access to rs network and synchronisation service implementation + * + * Copyright 2012-2012 by Christopher Evi-Parker + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#include +#include + +#include "rsnxs.h" +#include "rsgds.h" +#include "rsnxsobserver.h" +#include "pqi/p3linkmgr.h" +#include "serialiser/rsnxsitems.h" +#include "serialiser/rsgxsupdateitems.h" +#include "rsgxsnetutils.h" +#include "pqi/p3cfgmgr.h" +#include "rsgixs.h" +#include "util/rssharedptr.h" + +/// keep track of transaction number +typedef std::map TransactionIdMap; + +/// to keep track of peers active transactions +typedef std::map TransactionsPeerMap; + +class PgpAuxUtils; + +class RsGroupNetworkStatsRecord +{ + public: + RsGroupNetworkStatsRecord() { max_visible_count= 0 ; update_TS=0; } + + std::set suppliers ; + uint32_t max_visible_count ; + time_t update_TS ; +}; + +/*! + * This class implements the RsNetWorkExchangeService + * using transactions to handle synchrnisation of Nxs items between + * peers in a network + * Transactions requires the maintenance of several states between peers + * + * Thus a data structure maintains: peers, and their active transactions + * Then for each transaction it needs to be noted if this is an outgoing or incoming transaction + * Outgoing transaction are in 3 different states: + * 1. START 2. INITIATED 3. SENDING 4. END + * Incoming transaction are in 3 different states + * 1. START 2. RECEIVING 3. END + */ +class RsGxsNetService : public RsNetworkExchangeService, public p3ThreadedService, + public p3Config +{ +public: + + typedef RsSharedPtr pointer; + + static const uint32_t FRAGMENT_SIZE; + /*! + * only one observer is allowed + * @param servType service type + * @param gds The data service which allows read access to a service/store + * @param nxsObs observer will be notified whenever new messages/grps + * @param nxsObs observer will be notified whenever new messages/grps + * arrive + */ + RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, + RsNxsNetMgr *netMgr, + RsNxsObserver *nxsObs, // used to be = NULL. + const RsServiceInfo serviceInfo, + RsGixsReputation* reputations = NULL, RsGcxs* circles = NULL, + PgpAuxUtils *pgpUtils = NULL, + bool grpAutoSync = true, bool msgAutoSync = true); + + virtual ~RsGxsNetService(); + + virtual RsServiceInfo getServiceInfo() { return mServiceInfo; } + +public: + + + /*! + * Use this to set how far back synchronisation of messages should take place + * @param age the max age a sync item can to be allowed in a synchronisation + */ + // NOT IMPLEMENTED + virtual void setSyncAge(uint32_t age); + + /*! + * pauses synchronisation of subscribed groups and request for group id + * from peers + * @param enabled set to false to disable pause, and true otherwise + */ + // NOT IMPLEMENTED + virtual void pauseSynchronisation(bool enabled); + + + /*! + * Request for this message is sent through to peers on your network + * and how many hops from them you've indicated + * @param msgId the messages to retrieve + * @return request token to be redeemed + */ + virtual int requestMsg(const RsGxsGrpMsgIdPair& /* msgId */){ return 0;} + + /*! + * Request for this group is sent through to peers on your network + * and how many hops from them you've indicated + * @param enabled set to false to disable pause, and true otherwise + * @return request token to be redeemed + */ + virtual int requestGrp(const std::list& grpId, const RsPeerId& peerId); + + /*! + * share publish keys for the specified group with the peers in the specified list. + */ + + virtual int sharePublishKey(const RsGxsGroupId& grpId,const std::set& peers) ; + + /*! + * Returns statistics for the group networking activity: popularity (number of friends subscribers) and max_visible_msg_count, + * that is the max nnumber of messages reported by a friend. + */ + virtual bool getGroupNetworkStats(const RsGxsGroupId& id,RsGroupNetworkStats& stats) ; + + /*! + * Used to inform the net service that we changed subscription status. That helps + * optimising data transfer when e.g. unsubsribed groups are updated less often, etc + */ + virtual void subscribeStatusChanged(const RsGxsGroupId& id,bool subscribed) ; + + virtual void rejectMessage(const RsGxsMessageId& msg_id) ; + + /* p3Config methods */ +public: + + bool loadList(std::list& load); + bool saveList(bool &cleanup, std::list&); + RsSerialiser *setupSerialiser(); + +public: + + /*! + * initiates synchronisation + */ + int tick(); + + /*! + * Processes transactions and job queue + */ + virtual void data_tick(); +private: + + /*! + * called when + * items are deemed to be waiting in p3Service item queue + */ + void recvNxsItemQueue(); + + + /** S: Transaction processing **/ + + /*! + * These process transactions which are in a wait state + * Also moves transaction which have been completed to + * the completed transactions list + */ + void processTransactions(); + + /*! + * Process completed transaction, which either simply + * retires a transaction or additionally generates a response + * to the completed transaction + */ + void processCompletedTransactions(); + + /*! + * Process transaction owned/started by user + * @param tr transaction to process, ownership stays with callee + */ + void locked_processCompletedOutgoingTrans(NxsTransaction* tr); + + /*! + * Process transactions started/owned by other peers + * @param tr transaction to process, ownership stays with callee + */ + void locked_processCompletedIncomingTrans(NxsTransaction* tr); + + + /*! + * Process a transaction item, assumes a general lock + * @param item the transaction item to process + * @return false ownership of item left with callee + */ + bool locked_processTransac(RsNxsTransac* item); + + /*! + * This adds a transaction + * completeted transaction list + * If this is an outgoing transaction, transaction id is + * decrement + * @param trans transaction to add + */ + void locked_completeTransaction(NxsTransaction* trans); + + /*! + * This retrieves a unique transaction id that + * can be used in an outgoing transaction + */ + uint32_t locked_getTransactionId(); + + /*! + * This attempts to push the transaction id counter back if you have + * active outgoing transactions in play + */ + bool attemptRecoverIds(); + + /*! + * The cb listener is the owner of the grps + * @param grps + */ + //void notifyListenerGrps(std::list& grps); + + /*! + * The cb listener is the owner of the msgs + * @param msgs + */ + //void notifyListenerMsgs(std::list& msgs); + + /*! + * Generates new transaction to send msg requests based on list + * of msgs received from peer stored in passed transaction + * @param tr transaction responsible for generating msg request + */ + void locked_genReqMsgTransaction(NxsTransaction* tr); + + /*! + * Generates new transaction to send grp requests based on list + * of grps received from peer stored in passed transaction + * @param tr transaction responsible for generating grp request + */ + void locked_genReqGrpTransaction(NxsTransaction* tr); + + /*! + * This first checks if one can send a grpId based circles + * If it can send, then it call locked_genSendMsgsTransaction + * @param tr transaction responsible for generating grp request + * @see locked_genSendMsgsTransaction + */ + void locked_checkSendMsgsTransaction(NxsTransaction* tr); + + /*! + * Generates new transaction to send msg data based on list + * of grpids received from peer stored in passed transaction + * @param tr transaction responsible for generating grp request + */ + void locked_genSendMsgsTransaction(NxsTransaction* tr); + + /*! + * Generates new transaction to send grp data based on list + * of grps received from peer stored in passed transaction + * @param tr transaction responsible for generating grp request + */ + void locked_genSendGrpsTransaction(NxsTransaction* tr); + + /*! + * convenience function to add a transaction to list + * @param tr transaction to add + */ + bool locked_addTransaction(NxsTransaction* tr); + + void cleanTransactionItems(NxsTransaction* tr) const; + + /*! + * @param tr the transaction to check for timeout + * @return false if transaction has timed out, true otherwise + */ + bool locked_checkTransacTimedOut(NxsTransaction* tr); + + /** E: Transaction processing **/ + + /** S: item handlers **/ + + /*! + * This attempts handles transaction items + * ownership of item is left with callee if this method returns false + * @param item transaction item to handle + * @return false if transaction could not be handled, ownership of item is left with callee + */ + bool handleTransaction(RsNxsItem* item); + + /*! + * Handles an nxs item for group synchronisation + * by startin a transaction and sending a list + * of groups held by user + * @param item contains grp sync info + */ + void handleRecvSyncGroup(RsNxsSyncGrp* item); + + /*! + * Handles an nxs item for group statistics + * @param item contaims update time stamp and number of messages + */ + void handleRecvSyncGrpStatistics(RsNxsSyncGrpStats *grs); + + /*! + * Handles an nxs item for msgs synchronisation + * @param item contaims msg sync info + */ + void handleRecvSyncMessage(RsNxsSyncMsg* item); + + /*! + * Handles an nxs item for group publish key + * @param item contaims keys/grp info + */ + void handleRecvPublishKeys(RsNxsGroupPublishKeyItem*) ; + + /** E: item handlers **/ + + + void runVetting(); + + /*! + * @param peerId The peer to vet to see if they can receive this groupid + * @param grpMeta this is the meta item to determine if it can be sent to given peer + * @param toVet groupid/peer to vet are stored here if their circle id is not cached + * @return false, if you cannot send to this peer, true otherwise + */ + bool canSendGrpId(const RsPeerId& sslId, RsGxsGrpMetaData& grpMeta, std::vector& toVet); + + + bool canSendMsgIds(const std::vector& msgMetas, const RsGxsGrpMetaData&, const RsPeerId& sslId); + + bool checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxsGrpMetaData& meta); + + void locked_createTransactionFromPending(MsgRespPending* grpPend); + void locked_createTransactionFromPending(GrpRespPending* msgPend); + void locked_createTransactionFromPending(GrpCircleIdRequestVetting* grpPend); + void locked_createTransactionFromPending(MsgCircleIdsRequestVetting* grpPend); + + void locked_pushMsgTransactionFromList(std::list& reqList, const RsPeerId& peerId, const uint32_t& transN); + void locked_pushGrpTransactionFromList(std::list& reqList, const RsPeerId& peerId, const uint32_t& transN); + void locked_pushGrpRespFromList(std::list& respList, const RsPeerId& peer, const uint32_t& transN); + void locked_pushMsgRespFromList(std::list& itemL, const RsPeerId& sslId, const RsGxsGroupId &grp_id, const uint32_t& transN); + void syncWithPeers(); + void syncGrpStatistics(); + void addGroupItemToList(NxsTransaction*& tr, + const RsGxsGroupId& grpId, uint32_t& transN, + std::list& reqList); + + //bool locked_canReceive(const RsGxsGrpMetaData * const grpMeta, const RsPeerId& peerId); + + void processExplicitGroupRequests(); + + void locked_doMsgUpdateWork(const RsNxsTransac* nxsTrans, const RsGxsGroupId& grpId); + + void updateServerSyncTS(); +#ifdef TO_REMOVE + void updateClientSyncTS(); +#endif + + bool locked_CanReceiveUpdate(const RsNxsSyncGrp* item); + bool locked_CanReceiveUpdate(const RsNxsSyncMsg* item); + +private: + + typedef std::vector GrpFragments; + typedef std::vector MsgFragments; + + /*! + * Loops over pending publish key orders. + */ + void sharePublishKeysPending() ; + + /*! + * Fragment a message into individual fragments which are at most 150kb + * @param msg message to fragment + * @param msgFragments fragmented message + * @return false if fragmentation fails true otherwise + */ + bool fragmentMsg(RsNxsMsg& msg, MsgFragments& msgFragments) const; + + /*! + * Fragment a group into individual fragments which are at most 150kb + * @param grp group to fragment + * @param grpFragments fragmented group + * @return false if fragmentation fails true other wise + */ + bool fragmentGrp(RsNxsGrp& grp, GrpFragments& grpFragments) const; + + /*! + * Fragment a message into individual fragments which are at most 150kb + * @param msg message to fragment + * @param msgFragments fragmented message + * @return NULL if not possible to reconstruct message from fragment, + * pointer to defragments nxs message is possible + */ + RsNxsMsg* deFragmentMsg(MsgFragments& msgFragments) const; + + /*! + * Fragment a group into individual fragments which are at most 150kb + * @param grp group to fragment + * @param grpFragments fragmented group + * @return NULL if not possible to reconstruct group from fragment, + * pointer to defragments nxs group is possible + */ + RsNxsGrp* deFragmentGrp(GrpFragments& grpFragments) const; + + + /*! + * Note that if all fragments for a message are not found then its fragments are dropped + * @param fragments message fragments which are not necessarily from the same message + * @param partFragments the partitioned fragments (into message ids) + */ + void collateMsgFragments(MsgFragments fragments, std::map& partFragments) const; + + /*! + * Note that if all fragments for a group are not found then its fragments are dropped + * @param fragments group fragments which are not necessarily from the same group + * @param partFragments the partitioned fragments (into message ids) + */ + void collateGrpFragments(GrpFragments fragments, std::map& partFragments) const; + + /*! + * stamp the group info from that particular peer at the given time. + */ + + void locked_stampPeerGroupUpdateTime(const RsPeerId& pid,const RsGxsGroupId& grpId,time_t tm,uint32_t n_messages) ; + + void cleanRejectedMessages(); + void processObserverNotifications(); +private: + + + /*** transactions ***/ + + /// active transactions + TransactionsPeerMap mTransactions; + + /// completed transactions + std::list mComplTransactions; + + /// transaction id counter + uint32_t mTransactionN; + + /*** transactions ***/ + + /*** synchronisation ***/ + std::list mSyncGrp; + std::list mSyncMsg; + /*** synchronisation ***/ + + RsNxsObserver* mObserver; + RsGeneralDataService* mDataStore; + uint16_t mServType; + + // how much time must elapse before a timeout failure + // for an active transaction + uint32_t mTransactionTimeOut; + + RsPeerId mOwnId; + + RsNxsNetMgr* mNetMgr; + + /// for other members save transactions + RsMutex mNxsMutex; + + uint32_t mSyncTs; + uint32_t mLastKeyPublishTs; + uint32_t mLastCleanRejectedMessages; + + const uint32_t mSYNC_PERIOD; + int mUpdateCounter ; + + RsGcxs* mCircles; + RsGixsReputation* mReputations; + PgpAuxUtils *mPgpUtils; + bool mGrpAutoSync; + bool mAllowMsgSync; + + // need to be verfied + std::vector mPendingResp; + std::vector mPendingCircleVets; + std::map > mPendingPublishKeyRecipients ; + std::map > mExplicitRequest; + std::map > mPartialMsgUpdates ; + + // nxs sync optimisation + // can pull dynamically the latest timestamp for each message + +public: + + typedef std::map ClientMsgMap; + typedef std::map ServerMsgMap; + typedef std::map ClientGrpMap; + +private: + + ClientMsgMap mClientMsgUpdateMap; + ServerMsgMap mServerMsgUpdateMap; + ClientGrpMap mClientGrpUpdateMap; + + std::map mGroupNetworkStats ; + + RsGxsServerGrpUpdateItem* mGrpServerUpdateItem; + RsServiceInfo mServiceInfo; + + std::map mRejectedMessages; + std::vector mNewGroupsToNotify ; + std::vector mNewMessagesToNotify ; + + void debugDump(); +}; + +#endif // RSGXSNETSERVICE_H diff --git a/libretroshare/src/gxstunnel/p3gxstunnel.cc b/libretroshare/src/gxstunnel/p3gxstunnel.cc index ccd20087c..43ff5b0b3 100644 --- a/libretroshare/src/gxstunnel/p3gxstunnel.cc +++ b/libretroshare/src/gxstunnel/p3gxstunnel.cc @@ -856,7 +856,7 @@ void p3GxsTunnelService::handleRecvDHPublicKey(RsGxsTunnelDHPublicKeyItem *item) #endif uint32_t pubkey_size = BN_num_bytes(item->public_key) ; - unsigned char *data = (unsigned char *)malloc(pubkey_size) ; + RsTemporaryMemory data(pubkey_size) ; BN_bn2bin(item->public_key, data) ; RsTlvSecurityKey signature_key ; @@ -901,7 +901,7 @@ void p3GxsTunnelService::handleRecvDHPublicKey(RsGxsTunnelDHPublicKeyItem *item) signature_key = item->gxs_key ; } - if(!GxsSecurity::validateSignature((char*)data,pubkey_size,signature_key,item->signature)) + if(!GxsSecurity::validateSignature((char*)(unsigned char*)data,pubkey_size,signature_key,item->signature)) { std::cerr << "(SS) Signature was verified and it doesn't check! This is a security issue!" << std::endl; return ; @@ -939,7 +939,7 @@ void p3GxsTunnelService::handleRecvDHPublicKey(RsGxsTunnelDHPublicKeyItem *item) // Looks for the DH params. If not there yet, create them. // int size = DH_size(it->second.dh) ; - unsigned char *key_buff = new unsigned char[size] ; + RsTemporaryMemory key_buff(size) ; if(size != DH_compute_key(key_buff,item->public_key,it->second.dh)) { @@ -959,7 +959,6 @@ void p3GxsTunnelService::handleRecvDHPublicKey(RsGxsTunnelDHPublicKeyItem *item) assert(GXS_TUNNEL_AES_KEY_SIZE <= Sha1CheckSum::SIZE_IN_BYTES) ; memcpy(pinfo.aes_key, RsDirUtil::sha1sum(key_buff,size).toByteArray(),GXS_TUNNEL_AES_KEY_SIZE) ; - delete[] key_buff ; pinfo.last_contact = time(NULL) ; pinfo.last_keep_alive_sent = time(NULL) ; @@ -1036,7 +1035,15 @@ bool p3GxsTunnelService::locked_sendDHPublicKey(const DH *dh,const RsGxsId& own_ uint32_t error_status ; uint32_t size = BN_num_bytes(dhitem->public_key) ; - unsigned char *data = (unsigned char *)malloc(size) ; + + RsTemporaryMemory data(size) ; + + if(data == NULL) + { + delete(dhitem); + return false ; + } + BN_bn2bin(dhitem->public_key, data) ; if(!mGixs->signData((unsigned char*)data,size,own_gxs_id,signature,error_status)) @@ -1048,11 +1055,9 @@ bool p3GxsTunnelService::locked_sendDHPublicKey(const DH *dh,const RsGxsId& own_ default: std::cerr << "(EE) Unknown error when signing" << std::endl; break ; } - free(data) ; delete(dhitem); return false; } - free(data) ; if(!mGixs->getKey(own_gxs_id,signature_key_public)) { @@ -1136,8 +1141,13 @@ bool p3GxsTunnelService::locked_sendClearTunnelData(RsGxsTunnelDHPublicKeyItem * uint32_t rssize = item->serial_size() ; gitem->data_size = rssize + 8 ; - gitem->data_bytes = malloc(rssize+8) ; + gitem->data_bytes = rs_malloc(rssize+8) ; + if(gitem->data_bytes == NULL) + { + delete gitem ; + return NULL ; + } // by convention, we use a IV of 0 for unencrypted data. memset(gitem->data_bytes,0,8) ; @@ -1221,8 +1231,11 @@ bool p3GxsTunnelService::locked_sendEncryptedTunnelData(RsGxsTunnelItem *item) RsTurtleGenericDataItem *gitem = new RsTurtleGenericDataItem ; gitem->data_size = encrypted_size + GXS_TUNNEL_ENCRYPTION_IV_SIZE + GXS_TUNNEL_ENCRYPTION_HMAC_SIZE ; - gitem->data_bytes = malloc(gitem->data_size) ; + gitem->data_bytes = rs_malloc(gitem->data_size) ; + if(gitem->data_bytes == NULL) + return false ; + memcpy(& ((uint8_t*)gitem->data_bytes)[0] ,&IV,8) ; unsigned int md_len = GXS_TUNNEL_ENCRYPTION_HMAC_SIZE ; @@ -1316,7 +1329,11 @@ bool p3GxsTunnelService::sendData(const RsGxsTunnelId &tunnel_id, uint32_t servi item->flags = 0; // not used yet. item->service_id = service_id; item->data_size = size; // encrypted data size - item->data = (uint8_t*)malloc(size); // encrypted data + item->data = (uint8_t*)rs_malloc(size); // encrypted data + + if(item->data == NULL) + delete item ; + item->PeerId(RsPeerId(tunnel_id)) ; memcpy(item->data,data,size) ; diff --git a/libretroshare/src/gxstunnel/rsgxstunnelitems.cc b/libretroshare/src/gxstunnel/rsgxstunnelitems.cc index 414901df3..8898a352a 100644 --- a/libretroshare/src/gxstunnel/rsgxstunnelitems.cc +++ b/libretroshare/src/gxstunnel/rsgxstunnelitems.cc @@ -29,6 +29,7 @@ #include "serialiser/rsbaseserial.h" #include "serialiser/rstlvbase.h" #include "util/rsprint.h" +#include "util/rsmemory.h" #include "gxstunnel/rsgxstunnelitems.h" @@ -343,6 +344,13 @@ RsGxsTunnelDHPublicKeyItem *RsGxsTunnelSerialiser::deserialise_RsGxsTunnelDHPubl /* get mandatory parts first */ ok &= getRawUInt32(data, rssize, &offset, &s); + if(s > rssize || rssize - s < offset) + { + std::cerr << "RsGxsTunnelDHPublicKeyItem::() Size error while deserializing." << std::endl ; + delete item ; + return NULL ; + } + item->public_key = BN_bin2bn(&((unsigned char *)data)[offset],s,NULL) ; offset += s ; @@ -380,21 +388,22 @@ RsGxsTunnelDataItem *RsGxsTunnelSerialiser::deserialise_RsGxsTunnelDataItem(void ok &= getRawUInt32(dat, rssize, &offset, &item->service_id); ok &= getRawUInt32(dat, rssize, &offset, &item->data_size); - if(offset + item->data_size <= size) + if(item->data_size > rssize || rssize - item->data_size < offset) { - item->data = (unsigned char*)malloc(item->data_size) ; - - if(dat == NULL) - { - delete item ; - return NULL ; - } - - memcpy(item->data,&((uint8_t*)dat)[offset],item->data_size) ; - offset += item->data_size ; + std::cerr << "RsGxsTunnelDHPublicKeyItem::() Size error while deserializing." << std::endl ; + delete item ; + return NULL ; } - else - ok = false ; + item->data = (unsigned char*)rs_malloc(item->data_size) ; + + if(item->data == NULL) + { + delete item ; + return NULL ; + } + + memcpy(item->data,&((uint8_t*)dat)[offset],item->data_size) ; + offset += item->data_size ; if (offset != rssize) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index c6e64cd7e..d01d4c9dd 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -361,7 +361,6 @@ HEADERS += chat/distantchat.h \ HEADERS += pqi/authssl.h \ pqi/authgpg.h \ - pqi/rsmemory.h \ pgp/pgphandler.h \ pgp/pgpkeyutil.h \ pgp/rsaes.h \ @@ -475,6 +474,7 @@ HEADERS += turtle/p3turtle.h \ HEADERS += util/folderiterator.h \ util/rsdebug.h \ + util/rsmemory.h \ util/rscompress.h \ util/smallobject.h \ util/rsdir.h \ @@ -631,6 +631,7 @@ SOURCES += util/folderiterator.cc \ util/rscompress.cc \ util/smallobject.cc \ util/rsdir.cc \ + util/rsmemory.cc \ util/rsdiscspace.cc \ util/rsnet.cc \ util/rsnet_ss.cc \ diff --git a/libretroshare/src/pgp/pgphandler.cc b/libretroshare/src/pgp/pgphandler.cc index 76850b4e6..22ce28a48 100644 --- a/libretroshare/src/pgp/pgphandler.cc +++ b/libretroshare/src/pgp/pgphandler.cc @@ -26,6 +26,7 @@ extern "C" { #include "retroshare/rspeers.h" // For rsicontrol. #include "util/rsdir.h" #include "util/rsdiscspace.h" +#include "util/rsmemory.h" #include "pgp/pgpkeyutil.h" static const uint32_t PGP_CERTIFICATE_LIMIT_MAX_NAME_SIZE = 64 ; @@ -39,7 +40,11 @@ PassphraseCallback PGPHandler::_passphrase_callback = NULL ; ops_keyring_t *PGPHandler::allocateOPSKeyring() { - ops_keyring_t *kr = (ops_keyring_t*)malloc(sizeof(ops_keyring_t)) ; + ops_keyring_t *kr = (ops_keyring_t*)rs_malloc(sizeof(ops_keyring_t)) ; + + if(kr == NULL) + return NULL ; + kr->nkeys = 0 ; kr->nkeys_allocated = 0 ; kr->keys = 0 ; diff --git a/libretroshare/src/pqi/authssl.cc b/libretroshare/src/pqi/authssl.cc index 27cae90f6..f03847883 100644 --- a/libretroshare/src/pqi/authssl.cc +++ b/libretroshare/src/pqi/authssl.cc @@ -117,10 +117,10 @@ static struct CRYPTO_dynlock_value *dyn_create_function(const char */*file*/, in { struct CRYPTO_dynlock_value *value; - value = (struct CRYPTO_dynlock_value*) malloc(sizeof(struct CRYPTO_dynlock_value)); - if (!value) { + value = (struct CRYPTO_dynlock_value*) rs_malloc(sizeof(struct CRYPTO_dynlock_value)); + if (!value) return NULL; - } + pthread_mutex_init(&value->mutex, NULL); return value; @@ -166,10 +166,10 @@ static void dyn_destroy_function(struct CRYPTO_dynlock_value *l, const char */*f bool tls_init() { /* static locks area */ - mutex_buf = (pthread_mutex_t*) malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t)); - if (mutex_buf == NULL) { + mutex_buf = (pthread_mutex_t*) rs_malloc(CRYPTO_num_locks() * sizeof(pthread_mutex_t)); + if (mutex_buf == NULL) return false; - } + for (int i = 0; i < CRYPTO_num_locks(); i++) { pthread_mutex_init(&mutex_buf[i], NULL); } @@ -1259,7 +1259,11 @@ bool AuthSSLimpl::encrypt(void *&out, int &outlen, const void *in, int inlen, int out_offset = 0; int max_evp_key_size = EVP_PKEY_size(public_key); - ek = (unsigned char*)malloc(max_evp_key_size); + ek = (unsigned char*)rs_malloc(max_evp_key_size); + + if(ek == NULL) + return false ; + const EVP_CIPHER *cipher = EVP_aes_128_cbc(); int cipher_block_size = EVP_CIPHER_block_size(cipher); int size_net_ekl = sizeof(net_ekl); @@ -1273,8 +1277,13 @@ bool AuthSSLimpl::encrypt(void *&out, int &outlen, const void *in, int inlen, } // now assign memory to out accounting for data, and cipher block size, key length, and key length val - out = (unsigned char*)malloc(inlen + cipher_block_size + size_net_ekl + eklen + EVP_MAX_IV_LENGTH); + out = (unsigned char*)rs_malloc(inlen + cipher_block_size + size_net_ekl + eklen + EVP_MAX_IV_LENGTH); + if(out == NULL) + { + free(ek) ; + return false ; + } net_ekl = htonl(eklen); memcpy((unsigned char*)out + out_offset, &net_ekl, size_net_ekl); out_offset += size_net_ekl; @@ -1343,6 +1352,12 @@ bool AuthSSLimpl::decrypt(void *&out, int &outlen, const void *in, int inlen) unsigned char iv[EVP_MAX_IV_LENGTH]; int ek_mkl = EVP_PKEY_size(mOwnPrivateKey); ek = (unsigned char*)malloc(ek_mkl); + + if(ek == NULL) + { + std::cerr << "(EE) Cannot allocate memory for " << ek_mkl << " bytes in " << __PRETTY_FUNCTION__ << std::endl; + return false ; + } EVP_CIPHER_CTX_init(&ctx); int in_offset = 0, out_currOffset = 0; @@ -1380,8 +1395,13 @@ bool AuthSSLimpl::decrypt(void *&out, int &outlen, const void *in, int inlen) return false; } - out = (unsigned char*)malloc(inlen - in_offset); + out = (unsigned char*)rs_malloc(inlen - in_offset); + if(out == NULL) + { + free(ek) ; + return false ; + } if(!EVP_OpenUpdate(&ctx, (unsigned char*) out, &out_currOffset, (unsigned char*)in + in_offset, inlen - in_offset)) { free(ek); free(out) ; diff --git a/libretroshare/src/pqi/pqibin.cc b/libretroshare/src/pqi/pqibin.cc index 2aaa6e34a..dcc89b525 100644 --- a/libretroshare/src/pqi/pqibin.cc +++ b/libretroshare/src/pqi/pqibin.cc @@ -27,6 +27,7 @@ #include "pqi/authssl.h" #include "util/rsnet.h" #include "util/rsdir.h" +#include "util/rsmemory.h" // #define DEBUG_PQIBIN @@ -314,7 +315,13 @@ BinMemInterface::BinMemInterface(int defsize, int flags) :bin_flags(flags), buf(NULL), size(defsize), recvsize(0), readloc(0), hash(NULL), bcount(0) { - buf = malloc(defsize); + buf = rs_malloc(defsize); + + if(buf == NULL) + { + close() ; + return ; + } if (bin_flags & BIN_FLAGS_HASH_DATA) { hash = new pqihash(); @@ -326,7 +333,13 @@ BinMemInterface::BinMemInterface(const void *data, const int defsize, int flags) :bin_flags(flags), buf(NULL), size(defsize), recvsize(0), readloc(0), hash(NULL), bcount(0) { - buf = malloc(defsize); + buf = rs_malloc(defsize); + + if(buf == NULL) + { + close() ; + return ; + } if (bin_flags & BIN_FLAGS_HASH_DATA) { hash = new pqihash(); diff --git a/libretroshare/src/pqi/pqinetwork.cc b/libretroshare/src/pqi/pqinetwork.cc index 7f43068c9..e0cb9af90 100644 --- a/libretroshare/src/pqi/pqinetwork.cc +++ b/libretroshare/src/pqi/pqinetwork.cc @@ -25,6 +25,7 @@ #ifdef WINDOWS_SYS #include "util/rswin.h" +#include "util/rsmemory.h" #include #endif // WINDOWS_SYS @@ -290,7 +291,11 @@ bool getLocalAddresses(std::list & addrs) #ifdef WINDOWS_SYS // Seems strange to me but M$ documentation suggests to allocate this way... DWORD bf_size = 16000; - IP_ADAPTER_ADDRESSES* adapter_addresses = (IP_ADAPTER_ADDRESSES*) malloc(bf_size); + IP_ADAPTER_ADDRESSES* adapter_addresses = (IP_ADAPTER_ADDRESSES*) rs_malloc(bf_size); + + if(adapter_addresses == NULL) + return false ; + DWORD error = GetAdaptersAddresses(AF_UNSPEC, GAA_FLAG_SKIP_MULTICAST | GAA_FLAG_SKIP_DNS_SERVER | diff --git a/libretroshare/src/pqi/pqistore.cc b/libretroshare/src/pqi/pqistore.cc index 217ebdfb4..6dedbd5fc 100644 --- a/libretroshare/src/pqi/pqistore.cc +++ b/libretroshare/src/pqi/pqistore.cc @@ -46,6 +46,7 @@ #include #include "util/rsdebug.h" +#include "util/rsmemory.h" #include "util/rsstring.h" // @@ -194,7 +195,12 @@ int pqistore::writePkt(RsItem *pqi) #endif uint32_t pktsize = rsSerialiser->size(pqi); - void *ptr = malloc(pktsize); + + RsTemporaryMemory ptr(pktsize) ; + + if(ptr == NULL) + return 0 ; + if (!(rsSerialiser->serialise(pqi, ptr, &pktsize))) { #ifdef PQISTORE_DEBUG @@ -203,7 +209,6 @@ int pqistore::writePkt(RsItem *pqi) pqioutput(PQL_ALERT, pqistorezone, out); #endif - free(ptr); if (!(bio_flags & BIN_FLAGS_NO_DELETE)) delete pqi; return 0; @@ -218,7 +223,6 @@ int pqistore::writePkt(RsItem *pqi) pqi -> print_string(out); pqioutput(PQL_ALERT, pqistorezone, out); - free(ptr); if (!(bio_flags & BIN_FLAGS_NO_DELETE)) delete pqi; return 0; @@ -232,7 +236,6 @@ int pqistore::writePkt(RsItem *pqi) pqi -> print_string(out); pqioutput(PQL_ALERT, pqistorezone, out); - free(ptr); if (!(bio_flags & BIN_FLAGS_NO_DELETE)) delete pqi; @@ -250,7 +253,6 @@ int pqistore::writePkt(RsItem *pqi) pqioutput(PQL_ALERT, pqistorezone, out); #endif - free(ptr); if (!(bio_flags & BIN_FLAGS_NO_DELETE)) delete pqi; @@ -262,7 +264,6 @@ int pqistore::writePkt(RsItem *pqi) pqioutput(PQL_DEBUG_BASIC, pqistorezone, out); #endif - free(ptr); if (!(bio_flags & BIN_FLAGS_NO_DELETE)) delete pqi; @@ -288,7 +289,10 @@ int pqistore::readPkt(RsItem **item_out) // initial read size: basic packet. int blen = getRsPktBaseSize(); - void *block = malloc(blen); + void *block = rs_malloc(blen); + + if(block == NULL) + return false ; int tmplen; /* we have the header */ @@ -495,7 +499,10 @@ int pqiSSLstore::readPkt(RsItem **item_out) // initial read size: basic packet. int blen = getRsPktBaseSize(); - void *block = malloc(blen); + void *block = rs_malloc(blen); + + if(block == NULL) + return false ; int tmplen; /* we have the header */ diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 437c4d8fb..6f020a1ed 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -315,7 +315,11 @@ int pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize) /* decide which type of packet it is */ pktsize = mRsSerialiser->size(pqi); - void *ptr = malloc(pktsize); + void *ptr = rs_malloc(pktsize); + + if(ptr == NULL) + return 0 ; + #ifdef DEBUG_PQISTREAMER std::cerr << "pqistreamer::queue_outpqi() serializing packet with packet size : " << pktsize << std::endl; @@ -471,6 +475,7 @@ int pqistreamer::handleoutgoing_locked() return 0; } #define GROUP_OUTGOING_PACKETS 1 +#define PACKET_GROUPING_SIZE_LIMIT 32768 // send a out_pkt., else send out_data. unless // there is a pending packet. if (!mPkt_wpending) @@ -480,7 +485,7 @@ int pqistreamer::handleoutgoing_locked() mPkt_wpending_size = 0 ; int k=0; - while(mPkt_wpending_size < maxbytes && (dta = locked_pop_out_data())!=NULL ) + while(mPkt_wpending_size < (uint32_t)maxbytes && mPkt_wpending_size < PACKET_GROUPING_SIZE_LIMIT && (dta = locked_pop_out_data())!=NULL ) { uint32_t s = getRsItemSize(dta); mPkt_wpending = realloc(mPkt_wpending,s+mPkt_wpending_size) ; @@ -1040,7 +1045,10 @@ void pqistreamer::allocate_rpend_locked() return; mPkt_rpend_size = getRsPktMaxSize(); - mPkt_rpending = malloc(mPkt_rpend_size); + mPkt_rpending = rs_malloc(mPkt_rpend_size); + + if(mPkt_rpending == NULL) + return ; // avoid uninitialized (and random) memory read. memset(mPkt_rpending,0,mPkt_rpend_size) ; diff --git a/libretroshare/src/retroshare/rsplugin.h b/libretroshare/src/retroshare/rsplugin.h index 579b7694c..6b8363c37 100644 --- a/libretroshare/src/retroshare/rsplugin.h +++ b/libretroshare/src/retroshare/rsplugin.h @@ -160,7 +160,7 @@ class RsPlugin // creates a new resource api handler object. ownership is transferred to the caller. // the caller should supply a statetokenserver, and keep it valid until destruction // the plugin should return a entry point name. this is to make the entry point name independent from file names - virtual resource_api::ResourceRouter* new_resource_api_handler(const RsPlugInInterfaces& ifaces, resource_api::StateTokenServer* sts, std::string &entrypoint) const { return 0;} + virtual resource_api::ResourceRouter* new_resource_api_handler(const RsPlugInInterfaces& /* ifaces */, resource_api::StateTokenServer* /* sts */, std::string & /*entrypoint*/) const { return 0;} // Shutdown virtual void stop() {} diff --git a/libretroshare/src/rsserver/rsloginhandler.cc b/libretroshare/src/rsserver/rsloginhandler.cc index 4236709d7..b2a3c23f4 100644 --- a/libretroshare/src/rsserver/rsloginhandler.cc +++ b/libretroshare/src/rsserver/rsloginhandler.cc @@ -273,7 +273,13 @@ bool RsLoginHandler::tryAutoLogin(const RsPeerId& ssl_id,std::string& ssl_passwd fseek(fp, 0, SEEK_END); datalen = ftell(fp); fseek(fp, 0, SEEK_SET); - dataptr = (char *) malloc(datalen); + dataptr = (char *) rs_malloc(datalen); + + if(dataptr == NULL) + { + fclose(fp); + return false; + } fread(dataptr, 1, datalen, fp); fclose(fp); diff --git a/libretroshare/src/serialiser/rsbaseserial.cc b/libretroshare/src/serialiser/rsbaseserial.cc index fd19461f1..e38dcb8e0 100644 --- a/libretroshare/src/serialiser/rsbaseserial.cc +++ b/libretroshare/src/serialiser/rsbaseserial.cc @@ -40,6 +40,7 @@ bool getRawUInt8(void *data, uint32_t size, uint32_t *offset, uint8_t *out) /* first check there is space */ if (size < *offset + 1) { + std::cerr << "(EE) Cannot deserialise uint8_t: not enough size." << std::endl; return false; } void *buf = (void *) &(((uint8_t *) data)[*offset]); @@ -56,6 +57,7 @@ bool setRawUInt8(void *data, uint32_t size, uint32_t *offset, uint8_t in) /* first check there is space */ if (size < *offset + 1) { + std::cerr << "(EE) Cannot serialise uint8_t: not enough size." << std::endl; return false; } @@ -74,6 +76,7 @@ bool getRawUInt16(void *data, uint32_t size, uint32_t *offset, uint16_t *out) /* first check there is space */ if (size < *offset + 2) { + std::cerr << "(EE) Cannot deserialise uint16_t: not enough size." << std::endl; return false; } void *buf = (void *) &(((uint8_t *) data)[*offset]); @@ -92,6 +95,7 @@ bool setRawUInt16(void *data, uint32_t size, uint32_t *offset, uint16_t in) /* first check there is space */ if (size < *offset + 2) { + std::cerr << "(EE) Cannot serialise uint16_t: not enough size." << std::endl; return false; } @@ -114,6 +118,7 @@ bool getRawUInt32(void *data, uint32_t size, uint32_t *offset, uint32_t *out) /* first check there is space */ if (size < *offset + 4) { + std::cerr << "(EE) Cannot deserialise uint32_t: not enough size." << std::endl; return false; } void *buf = (void *) &(((uint8_t *) data)[*offset]); @@ -132,6 +137,7 @@ bool setRawUInt32(void *data, uint32_t size, uint32_t *offset, uint32_t in) /* first check there is space */ if (size < *offset + 4) { + std::cerr << "(EE) Cannot serialise uint32_t: not enough size." << std::endl; return false; } @@ -154,6 +160,7 @@ bool getRawUInt64(void *data, uint32_t size, uint32_t *offset, uint64_t *out) /* first check there is space */ if (size < *offset + 8) { + std::cerr << "(EE) Cannot deserialise uint64_t: not enough size." << std::endl; return false; } void *buf = (void *) &(((uint8_t *) data)[*offset]); @@ -172,6 +179,7 @@ bool setRawUInt64(void *data, uint32_t size, uint32_t *offset, uint64_t in) /* first check there is space */ if (size < *offset + 8) { + std::cerr << "(EE) Cannot serialise uint64_t: not enough size." << std::endl; return false; } @@ -231,12 +239,13 @@ bool getRawString(void *data, uint32_t size, uint32_t *offset, std::string &outS } /* check there is space for string */ - if (size < *offset + len) + if(len > size || size-len < *offset) // better than if(size < *offset + len) because it avoids integer overflow { std::cerr << "getRawString() not enough size" << std::endl; return false; } uint8_t *buf = &(((uint8_t *) data)[*offset]); + for (uint32_t i = 0; i < len; i++) { outStr += buf[i]; @@ -250,11 +259,10 @@ bool setRawString(void *data, uint32_t size, uint32_t *offset, const std::string { uint32_t len = inStr.length(); /* first check there is space */ - if (size < *offset + 4 + len) + + if(size < 4 || len > size-4 || size-len-4 < *offset) // better than if(size < *offset + len + 4) because it avoids integer overflow { -//#ifdef RSSERIAL_DEBUG std::cerr << "setRawString() Not enough size" << std::endl; -//#endif return false; } diff --git a/libretroshare/src/serialiser/rsserial.h b/libretroshare/src/serialiser/rsserial.h index dadf9a3d6..47b0666fc 100644 --- a/libretroshare/src/serialiser/rsserial.h +++ b/libretroshare/src/serialiser/rsserial.h @@ -34,6 +34,7 @@ #include #include +#include "util/rsmemory.h" #include "retroshare/rstypes.h" /******************************************************************* @@ -181,7 +182,7 @@ class RsRawItem: public RsItem RsRawItem(uint32_t t, uint32_t size) :RsItem(t), len(size) { - data = malloc(len); + data = rs_malloc(len); } virtual ~RsRawItem() diff --git a/libretroshare/src/serialiser/rstlvbinary.cc b/libretroshare/src/serialiser/rstlvbinary.cc index f27f97fcf..672f6c54b 100644 --- a/libretroshare/src/serialiser/rstlvbinary.cc +++ b/libretroshare/src/serialiser/rstlvbinary.cc @@ -24,6 +24,7 @@ * */ +#include "util/rsmemory.h" #include "serialiser/rstlvbinary.h" #include "serialiser/rstlvbase.h" @@ -74,7 +75,11 @@ bool RsTlvBinaryData::setBinData(const void *data, uint32_t size) return true; } - bin_data = malloc(bin_len); + bin_data = rs_malloc(bin_len); + + if(bin_data == NULL) + return false ; + memcpy(bin_data, data, bin_len); return true; } diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index d25220662..a8b7224ef 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -916,6 +916,12 @@ bool SSGxsChannelGroup::load(const std::string &input) mAutoDownload = false; mDownloadDirectory.clear(); + if(input.empty()) + { + std::cerr << "(EE) SSGxsChannelGroup::load() asked to load a null string. Weird." << std::endl; + return false ; + } + RsTemporaryMemory tmpmem(input.length()); if (1 == sscanf(input.c_str(), "D:%d", &download_val)) diff --git a/libretroshare/src/services/p3gxscommon.cc b/libretroshare/src/services/p3gxscommon.cc index 1f98072fa..ad97f1eca 100644 --- a/libretroshare/src/services/p3gxscommon.cc +++ b/libretroshare/src/services/p3gxscommon.cc @@ -98,7 +98,8 @@ void RsGxsImage::take(uint8_t *data, uint32_t size) // NB Must make sure that we always use malloc/free for this data. uint8_t *RsGxsImage::allocate(uint32_t size) { - uint8_t *val = (uint8_t *) malloc(size); + uint8_t *val = (uint8_t *) rs_malloc(size); + #ifdef DEBUG_GXSCOMMON std::cerr << "RsGxsImage()::allocate(" << (void *) val << ")"; std::cerr << std::endl; diff --git a/libretroshare/src/services/p3photoservice.cc b/libretroshare/src/services/p3photoservice.cc index 1821c8d69..759f5ad18 100644 --- a/libretroshare/src/services/p3photoservice.cc +++ b/libretroshare/src/services/p3photoservice.cc @@ -26,7 +26,11 @@ bool RsPhotoThumbnail::copyFrom(const RsPhotoThumbnail &nail) size = nail.size; type = nail.type; - data = (uint8_t *) malloc(size); + data = (uint8_t *) rs_malloc(size); + + if(data == NULL) + return false ; + memcpy(data, nail.data, size); return true; diff --git a/libretroshare/src/tcponudp/tcppacket.cc b/libretroshare/src/tcponudp/tcppacket.cc index 23b007d5b..3f308de1a 100644 --- a/libretroshare/src/tcponudp/tcppacket.cc +++ b/libretroshare/src/tcponudp/tcppacket.cc @@ -36,6 +36,7 @@ #include #include +#include #include @@ -79,8 +80,10 @@ TcpPacket::TcpPacket(uint8 *ptr, int size) if (size > 0) { datasize = size; - data = (uint8 *) malloc(datasize); - memcpy(data, (void *) ptr, size); + data = (uint8 *) rs_malloc(datasize); + + if(data != NULL) + memcpy(data, (void *) ptr, size); } return; } @@ -185,7 +188,17 @@ int TcpPacket::readPacket(void *buf, int size) free(data); } datasize = size - TCP_PSEUDO_HDR_SIZE; - data = (uint8 *) malloc(datasize); + + if(datasize == 0) // this happens! + { + data = NULL ; + return 0 ; + } + + data = (uint8 *) rs_malloc(datasize); + + if(data == NULL) + return 0 ; /* now the data */ memcpy(data, (void *) &(((uint8 *) buf)[20]), datasize); diff --git a/libretroshare/src/tcponudp/udprelay.cc b/libretroshare/src/tcponudp/udprelay.cc index cc339f07d..2f837feec 100644 --- a/libretroshare/src/tcponudp/udprelay.cc +++ b/libretroshare/src/tcponudp/udprelay.cc @@ -26,6 +26,7 @@ #include "udprelay.h" #include #include +#include /* * #define DEBUG_UDP_RELAY 1 @@ -70,7 +71,7 @@ UdpRelayReceiver::UdpRelayReceiver(UdpPublisher *pub) setRelayClassMax(UDP_RELAY_CLASS_GENERAL, UDP_RELAY_DEFAULT_GENERAL, UDP_RELAY_DEFAULT_BANDWIDTH); /* only allocate this space once */ - mTmpSendPkt = malloc(MAX_RELAY_UDP_PACKET_SIZE); + mTmpSendPkt = rs_malloc(MAX_RELAY_UDP_PACKET_SIZE); mTmpSendSize = MAX_RELAY_UDP_PACKET_SIZE; clearDataTransferred(); diff --git a/libretroshare/src/tcponudp/udpstunner.cc b/libretroshare/src/tcponudp/udpstunner.cc index 06e1748ad..421ad8f17 100644 --- a/libretroshare/src/tcponudp/udpstunner.cc +++ b/libretroshare/src/tcponudp/udpstunner.cc @@ -29,6 +29,7 @@ #include "util/rsrandom.h" #include "util/rsprint.h" +#include "util/rsmemory.h" #include "util/rsstring.h" static const int STUN_TTL = 64; @@ -535,7 +536,11 @@ bool UdpStun_generate_stun_pkt(void *stun_pkt, int *len) void *UdpStun_generate_stun_reply(struct sockaddr_in *stun_addr, int *len) { /* just the header */ - void *stun_pkt = malloc(28); + void *stun_pkt = rs_malloc(28); + + if(!stun_pkt) + return NULL ; + ((uint16_t *) stun_pkt)[0] = (uint16_t) htons(0x0101); ((uint16_t *) stun_pkt)[1] = (uint16_t) htons(28); /* only header + 8 byte addr */ /* transaction id - should be random */ diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index 4e33f76e8..8139c9ad3 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -284,19 +284,24 @@ RsTurtleRegExpSearchRequestItem::RsTurtleRegExpSearchRequestItem(void *data,uint uint32_t n =0 ; ok &= getRawUInt32(data,pktsize,&offset,&n) ; - expr._tokens.resize(n) ; + if(ok) + expr._tokens.resize(n) ; - for(uint32_t i=0;i pktsize) + throw std::runtime_error("RsTurtleTunnelOkItem::() wrong rssize (exceeds pktsize).") ; + /* add mandatory parts first */ bool ok = true ; @@ -540,7 +548,11 @@ RsTurtleGenericDataItem::RsTurtleGenericDataItem(void *data,uint32_t pktsize) #ifdef P3TURTLE_DEBUG std::cerr << " request_id=" << (void*)request_id << ", tunnel_id=" << (void*)tunnel_id << std::endl ; #endif - data_bytes = malloc(data_size) ; + + if(data_size > rssize || rssize - data_size < offset) + throw std::runtime_error("RsTurtleTunnelOkItem::() wrong data_size (exceeds rssize).") ; + + data_bytes = rs_malloc(data_size) ; if(data_bytes != NULL) { @@ -548,10 +560,7 @@ RsTurtleGenericDataItem::RsTurtleGenericDataItem(void *data,uint32_t pktsize) offset += data_size ; } else - { - std::cerr << "(EE) RsTurtleGenericDataItem: Error. Cannot allocate data for a size of " << data_size << " bytes." < #include "rscompress.h" #include "zlib.h" +#include "util/rsmemory.h" // 16K buffer size. // @@ -42,7 +43,10 @@ bool RsCompress::compress_memory_chunk(const uint8_t *input_mem,const uint32_t i uint32_t output_offset = 0 ; uint32_t input_offset = 0 ; output_size = 1024 ; - output_mem = (uint8_t*)malloc(output_size) ; + output_mem = (uint8_t*)rs_malloc(output_size) ; + + if(!output_mem) + return false ; int ret, flush; unsigned have; @@ -113,8 +117,11 @@ bool RsCompress::uncompress_memory_chunk(const uint8_t *input_mem,const uint32_t output_size = input_size ; uint32_t output_offset = 0 ; uint32_t input_offset = 0 ; - output_mem = (uint8_t*)malloc(output_size) ; + output_mem = (uint8_t*)rs_malloc(output_size) ; + if(!output_mem) + return false ; + int ret; unsigned have; z_stream strm; diff --git a/libretroshare/src/util/rsdir.cc b/libretroshare/src/util/rsdir.cc index b396b1672..bc9de2c43 100644 --- a/libretroshare/src/util/rsdir.cc +++ b/libretroshare/src/util/rsdir.cc @@ -33,6 +33,7 @@ #include "util/rsdir.h" #include "util/rsstring.h" #include "util/rsrandom.h" +#include "util/rsmemory.h" #include "retroshare/rstypes.h" #include "rsthreads.h" #include @@ -267,7 +268,14 @@ bool RsDirUtil::copyFile(const std::string& source,const std::string& dest) size_t T=0; static const int BUFF_SIZE = 10485760 ; // 10 MB buffer to speed things up. - void *buffer = malloc(BUFF_SIZE) ; + RsTemporaryMemory buffer(BUFF_SIZE) ; + + if(!buffer) + { + fclose(in) ; + fclose(out) ; + return false ; + } bool bRet = true; @@ -286,8 +294,6 @@ bool RsDirUtil::copyFile(const std::string& source,const std::string& dest) fclose(in) ; fclose(out) ; - free(buffer) ; - return true ; #endif diff --git a/libretroshare/src/util/rsmemory.cc b/libretroshare/src/util/rsmemory.cc new file mode 100644 index 000000000..46ffd4a58 --- /dev/null +++ b/libretroshare/src/util/rsmemory.cc @@ -0,0 +1,32 @@ +#include "util/rsmemory.h" + +void *rs_malloc(size_t size) +{ + static const size_t SAFE_MEMALLOC_THRESHOLD = 1024*1024*1024 ; // 1Gb should be enough for everything! + + if(size == 0) + { + std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl; + print_stacktrace() ; + return NULL ; + } + + if(size > SAFE_MEMALLOC_THRESHOLD) + { + std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl; + print_stacktrace() ; + return NULL ; + } + + void *mem = malloc(size) ; + + if(mem == NULL) + { + std::cerr << "(EE) Memory allocation error for a chunk of " << size << " bytes. Callstack:" << std::endl; + print_stacktrace() ; + return NULL ; + } + + return mem ; +} + diff --git a/libretroshare/src/util/rsmemory.h b/libretroshare/src/util/rsmemory.h index c6b88ed2b..7da4245c4 100644 --- a/libretroshare/src/util/rsmemory.h +++ b/libretroshare/src/util/rsmemory.h @@ -1,6 +1,10 @@ #pragma once #include +#include +#include + +void *rs_malloc(size_t size) ; // This is a scope guard to release the memory block when going of of the current scope. // Can be very useful to auto-delete some memory on quit without the need to call free each time. @@ -24,7 +28,7 @@ class RsTemporaryMemory public: RsTemporaryMemory(size_t s) { - _mem = (unsigned char *)malloc(s) ; + _mem = (unsigned char *)rs_malloc(s) ; if(_mem) _size = s ; @@ -53,5 +57,3 @@ private: RsTemporaryMemory& operator=(const RsTemporaryMemory&) { return *this ;} RsTemporaryMemory(const RsTemporaryMemory&) {} }; - - diff --git a/libretroshare/src/util/rsrecogn.cc b/libretroshare/src/util/rsrecogn.cc index 6712608a4..0f5fc0504 100644 --- a/libretroshare/src/util/rsrecogn.cc +++ b/libretroshare/src/util/rsrecogn.cc @@ -145,7 +145,12 @@ bool RsRecogn::loadSigningKeys(std::map &signM /* store in */ uint32_t datalen = recognSerialiser.size(item); - uint8_t *data = (uint8_t *) malloc(datalen); + + RsTemporaryMemory data(datalen) ; + + if(!data) + return false ; + uint32_t pktlen = datalen; int signOk = 0; @@ -181,8 +186,6 @@ bool RsRecogn::loadSigningKeys(std::map &signM #endif // DEBUG_RECOGN delete item; } - - free(data); } /* clean up */ @@ -233,7 +236,12 @@ bool RsRecogn::validateTagSignature(RsGxsRecognSignerItem *signer, RsGxsRecognTa RsGxsRecognSerialiser serialiser; uint32_t datalen = serialiser.size(item); - uint8_t *data = (uint8_t *) malloc(datalen); + + RsTemporaryMemory data(datalen) ; + + if(!data) + return false ; + int signOk = 0; uint32_t pktlen = datalen; @@ -262,8 +270,6 @@ bool RsRecogn::validateTagSignature(RsGxsRecognSignerItem *signer, RsGxsRecognTa EVP_MD_CTX_destroy(mdctx); EVP_PKEY_free(signKey); - - free(data); return (signOk == 1); } diff --git a/libretroshare/src/util/smallobject.cc b/libretroshare/src/util/smallobject.cc index 3d91e99f9..ff9beeb48 100644 --- a/libretroshare/src/util/smallobject.cc +++ b/libretroshare/src/util/smallobject.cc @@ -1,6 +1,7 @@ #include #include "smallobject.h" #include "util/rsthreads.h" +#include "util/rsmemory.h" using namespace RsMemoryManagement ; @@ -206,7 +207,7 @@ SmallObjectAllocator::~SmallObjectAllocator() void *SmallObjectAllocator::allocate(size_t bytes) { if(bytes > _maxObjectSize) - return malloc(bytes) ; + return rs_malloc(bytes) ; else if(_lastAlloc != NULL && _lastAlloc->blockSize() == bytes) return _lastAlloc->allocate() ; else diff --git a/openpgpsdk/src/openpgpsdk/util.c b/openpgpsdk/src/openpgpsdk/util.c index 88fe3f100..491611b8b 100644 --- a/openpgpsdk/src/openpgpsdk/util.c +++ b/openpgpsdk/src/openpgpsdk/util.c @@ -114,13 +114,16 @@ void ops_finish(void) \note Should be freed after use with free(). */ void *ops_mallocz(size_t n) - { - void *m=malloc(n); +{ + void *m=malloc(n); - memset(m,'\0',n); + if(m == NULL) + fprintf(stderr,"(EE) Cannot allocate %d bytes of memory in %s\n",n,__PRETTY_FUNCTION__) ; + else + memset(m,'\0',n); - return m; - } + return m; +} typedef struct { diff --git a/plugins/FeedReader/util/XMLWrapper.cpp b/plugins/FeedReader/util/XMLWrapper.cpp index a625a104b..c51f60b1a 100644 --- a/plugins/FeedReader/util/XMLWrapper.cpp +++ b/plugins/FeedReader/util/XMLWrapper.cpp @@ -182,7 +182,7 @@ bool XMLWrapper::readXML(const char *xml) cleanup(); handleError(true, mLastErrorString); - mDocument = xmlReadDoc(BAD_CAST xml, "", NULL, /*XML_PARSE_NOERROR | XML_PARSE_NOWARNING | */XML_PARSE_COMPACT | XML_PARSE_NOENT | XML_PARSE_NOCDATA); + mDocument = xmlReadDoc(BAD_CAST xml, "", NULL, /*XML_PARSE_NOERROR | XML_PARSE_NOWARNING | */XML_PARSE_COMPACT | XML_PARSE_NOCDATA); handleError(false, mLastErrorString); if (mDocument) { diff --git a/plugins/VOIP/gui/SpeexProcessor.cpp b/plugins/VOIP/gui/SpeexProcessor.cpp index 4322a1912..1ff7bab17 100644 --- a/plugins/VOIP/gui/SpeexProcessor.cpp +++ b/plugins/VOIP/gui/SpeexProcessor.cpp @@ -1,5 +1,6 @@ #include "SpeexProcessor.h" +#include #include #include @@ -350,7 +351,11 @@ void SpeexOutputProcessor::putNetworkPacket(QString name, QByteArray packet) { if (userJitterHash.contains(name)) { userJitter = userJitterHash.value(name); } else { - userJitter = (SpeexJitter*)malloc(sizeof(SpeexJitter)); + userJitter = (SpeexJitter*)rs_malloc(sizeof(SpeexJitter)); + + if(!userJitter) + return ; + speex_jitter_init(userJitter, speex_decoder_init(&speex_wb_mode), SAMPLING_RATE); int on = 1; speex_decoder_ctl(userJitter->dec, SPEEX_SET_ENH, &on); diff --git a/plugins/VOIP/gui/VideoProcessor.cpp b/plugins/VOIP/gui/VideoProcessor.cpp index ad0a217ec..5f4d5de0e 100644 --- a/plugins/VOIP/gui/VideoProcessor.cpp +++ b/plugins/VOIP/gui/VideoProcessor.cpp @@ -10,6 +10,8 @@ #include #include +#include "util/rsmemory.h" + #include "VideoProcessor.h" #include "QVideoDevice.h" @@ -384,7 +386,10 @@ bool JPEGVideo::encodeData(const QImage& image,uint32_t /* size_hint */,RsVOIPDa buffer.open(QIODevice::WriteOnly) ; encoded_frame.save(&buffer,"JPEG") ; - voip_chunk.data = malloc(HEADER_SIZE + qb.size()); + voip_chunk.data = rs_malloc(HEADER_SIZE + qb.size()); + + if(!voip_chunk.data) + return false ; // build header uint32_t flags = differential_frame ? JPEG_VIDEO_FLAGS_DIFFERENTIAL_FRAME : 0x0 ; @@ -679,7 +684,11 @@ bool FFmpegVideo::encodeData(const QImage& image, uint32_t target_encoding_bitra if(got_output) { - voip_chunk.data = malloc(pkt.size + HEADER_SIZE) ; + voip_chunk.data = rs_malloc(pkt.size + HEADER_SIZE) ; + + if(!voip_chunk.data) + return false ; + uint32_t flags = 0; ((unsigned char *)voip_chunk.data)[0] = VideoProcessor::VIDEO_PROCESSOR_CODEC_ID_MPEG_VIDEO & 0xff ; diff --git a/plugins/VOIP/services/p3VOIP.cc b/plugins/VOIP/services/p3VOIP.cc index d5cc4b06b..859c5bf35 100644 --- a/plugins/VOIP/services/p3VOIP.cc +++ b/plugins/VOIP/services/p3VOIP.cc @@ -280,11 +280,10 @@ int p3VOIP::sendVoipData(const RsPeerId& peer_id,const RsVOIPDataChunk& chunk) std::cerr << "Cannot allocate RsVOIPDataItem !" << std::endl; return false ; } - item->voip_data = malloc(chunk.size) ; + item->voip_data = rs_malloc(chunk.size) ; if(item->voip_data == NULL) { - std::cerr << "Cannot allocate RsVOIPDataItem.voip_data of size " << chunk.size << " !" << std::endl; delete item ; return false ; } @@ -432,7 +431,13 @@ bool p3VOIP::getIncomingData(const RsPeerId& peer_id,std::vectordata_size ; - chunk.data = malloc((*it2)->data_size) ; + chunk.data = rs_malloc((*it2)->data_size) ; + + if(chunk.data == NULL) + { + delete *it2 ; + continue ; + } uint32_t type_flags = (*it2)->flags & (RS_VOIP_FLAGS_AUDIO_DATA | RS_VOIP_FLAGS_VIDEO_DATA) ; if(type_flags == RS_VOIP_FLAGS_AUDIO_DATA) diff --git a/plugins/VOIP/services/rsVOIPItems.cc b/plugins/VOIP/services/rsVOIPItems.cc index afc76b505..cd3132dfd 100644 --- a/plugins/VOIP/services/rsVOIPItems.cc +++ b/plugins/VOIP/services/rsVOIPItems.cc @@ -447,7 +447,14 @@ RsVOIPDataItem::RsVOIPDataItem(void *data, uint32_t pktsize) ok &= getRawUInt32(data, rssize, &offset, &flags); ok &= getRawUInt32(data, rssize, &offset, &data_size); - voip_data = malloc(data_size) ; + if(data_size > rssize || rssize - data_size < offset) + throw std::runtime_error("Not enough space.") ; + + voip_data = rs_malloc(data_size) ; + + if(!voip_data) + throw std::runtime_error("Serialization error.") ; + memcpy(voip_data,&((uint8_t*)data)[offset],data_size) ; offset += data_size ; diff --git a/retroshare-gui/src/gui/Identity/IdDialog.cpp b/retroshare-gui/src/gui/Identity/IdDialog.cpp index 36bc69534..e7d22a475 100644 --- a/retroshare-gui/src/gui/Identity/IdDialog.cpp +++ b/retroshare-gui/src/gui/Identity/IdDialog.cpp @@ -73,7 +73,26 @@ #define RSID_FILTER_ALL 0xffff #define IMAGE_EDIT ":/images/edit_16.png" - +// quick solution for RSID_COL_VOTES sorting +class TreeWidgetItem : public QTreeWidgetItem { + public: + TreeWidgetItem(int type=Type): QTreeWidgetItem(type) {} + TreeWidgetItem(QTreeWidget *tree): QTreeWidgetItem(tree) {} + TreeWidgetItem(const QStringList& strings): QTreeWidgetItem (strings) {} + bool operator< (const QTreeWidgetItem& other ) const { + int column = treeWidget()->sortColumn(); + const QVariant v1 = data(column, Qt::DisplayRole); + const QVariant v2 = other.data(column, Qt::DisplayRole); + double value1 = v1.toDouble(); + double value2 = v2.toDouble(); + if (value1 != value2) { + return value1 < value2; + } + else { + return (v1.toString().compare (v2.toString(), Qt::CaseInsensitive) < 0); + } + } +}; /** Constructor */ IdDialog::IdDialog(QWidget *parent) : RsGxsUpdateBroadcastPage(rsIdentity, parent), @@ -368,7 +387,7 @@ bool IdDialog::fillIdListItem(const RsGxsIdGroup& data, QTreeWidgetItem *&item, return false; if (!item) - item = new QTreeWidgetItem(); + item = new TreeWidgetItem(); RsReputations::ReputationInfo info ; rsReputations->getReputationInfo(RsGxsId(data.mMeta.mGroupId),info) ; diff --git a/retroshare-gui/src/gui/common/RSTextBrowser.cpp b/retroshare-gui/src/gui/common/RSTextBrowser.cpp index 930c506ac..20c76703d 100644 --- a/retroshare-gui/src/gui/common/RSTextBrowser.cpp +++ b/retroshare-gui/src/gui/common/RSTextBrowser.cpp @@ -1,3 +1,5 @@ +#include + #include #include @@ -72,15 +74,25 @@ void RSTextBrowser::paintEvent(QPaintEvent *event) QVariant RSTextBrowser::loadResource(int type, const QUrl &name) { - if (mShowImages || type != QTextDocument::ImageResource || name.scheme().compare("data", Qt::CaseInsensitive) != 0) { - return QTextBrowser::loadResource(type, name); - } + // case 1: always trust the image if it comes from an internal resource + + if(name.scheme().compare("qrc",Qt::CaseInsensitive)==0 && type == QTextDocument::ImageResource) + return QTextBrowser::loadResource(type, name); + + // case 2: only display if the user allows it. Data resources can be bad (svg bombs) but we filter them out globally at the network layer. + // It would be good to add here a home-made resource loader that only loads images and not svg crap, just in case. + + if(name.scheme().compare("data",Qt::CaseInsensitive)==0 && mShowImages) + return QTextBrowser::loadResource(type, name); + + // case 3: otherwise, do not display + + std::cerr << "TEXTBROWSER: refusing load ressource request: type=" << type << " scheme=" << name.scheme().toStdString() << ", url=" << name.toString().toStdString() << std::endl; + + if (mImageBlockWidget) + mImageBlockWidget->show(); - if (mImageBlockWidget) { - mImageBlockWidget->show(); - } - - return QPixmap(":/trolltech/styles/commonstyle/images/file-16.png"); + return QPixmap(":/trolltech/styles/commonstyle/images/file-16.png"); } void RSTextBrowser::setImageBlockWidget(RSImageBlockWidget *widget) diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 6280f029c..fe2e1aad3 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -213,6 +213,8 @@ GxsForumThreadWidget::GxsForumThreadWidget(const RsGxsGroupId &forumId, QWidget ui->line_2->hide() ; ui->by_text_label->hide() ; ui->by_label->hide() ; + ui->postText->setImageBlockWidget(ui->imageBlockWidget); + ui->postText->resetImagesStatus(Settings->getForumLoadEmbeddedImages()) ; ui->subscribeToolButton->setToolTip(tr("

Subscribing to the forum will gather \ available posts from your subscribed friends, and make the \ @@ -590,6 +592,7 @@ void GxsForumThreadWidget::changedThread() if (mFillThread) { return; } + ui->postText->resetImagesStatus(Settings->getForumLoadEmbeddedImages()) ; insertMessage(); } diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.ui b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.ui index d7018afd7..f156d3a2c 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.ui +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.ui @@ -6,17 +6,14 @@ 0 0 - 622 - 412 + 851 + 721 Form - - - 0 - + @@ -117,6 +114,12 @@ QSizePolicy::MinimumExpanding + + + 0 + 0 + + @@ -446,21 +449,37 @@ - - - - 0 - 10 - - - - Qt::CustomContextMenu - + + + + + + + 0 + 0 + + + + + + + + + 0 + 10 + + + + Qt::CustomContextMenu + + + + - + :/images/document_save.png:/images/document_save.png @@ -502,6 +521,12 @@

gui/common/ElidedLabel.h
1 + + RSImageBlockWidget + QWidget +
gui/common/RSImageBlockWidget.h
+ 1 + diff --git a/retroshare-gui/src/gui/notifyqt.cpp b/retroshare-gui/src/gui/notifyqt.cpp index 7c19da271..883c05e28 100644 --- a/retroshare-gui/src/gui/notifyqt.cpp +++ b/retroshare-gui/src/gui/notifyqt.cpp @@ -151,11 +151,25 @@ class SignatureEventData { // We need a new memory chnk because there's no guarranty _sign nor _signlen are not in the stack - sign = (unsigned char *)malloc(_signlen) ; + sign = (unsigned char *)rs_malloc(_signlen) ; + + if(!sign) + { + signlen = NULL ; + signature_result = SELF_SIGNATURE_RESULT_FAILED ; + return ; + } + signlen = new unsigned int ; *signlen = _signlen ; signature_result = SELF_SIGNATURE_RESULT_PENDING ; - data = malloc(_len) ; + data = rs_malloc(_len) ; + + if(!data) + { + len = 0 ; + return ; + } len = _len ; memcpy(data,_data,len) ; }