From b12f9323dc441efcac2e4a2034f8c73366152ac3 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 10 Dec 2017 15:57:01 +0100 Subject: [PATCH 01/84] added some text to describe the protocol for GXS distant sync --- libretroshare/src/gxs/rsgxsnetservice.cc | 49 +++++++++++++++++++++++- 1 file changed, 47 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 88c7701d2..eba451bcd 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -166,8 +166,6 @@ // | | // (Only send if rand() < sendingProb()) +---comes from mClientMsgUpdateMap -----+ // -// -// // Suggestions // =========== // * handleRecvSyncGroup should use mit->second.mLastPost to limit the sending of already known data @@ -194,6 +192,53 @@ // Problem: without msg, we cannot know the grpId!! // // * mClientMsgUpdateMap[peerid][grpId] is only updated when new msgs are received. Up to date groups will keep asking for lists! +// +// Distant sync +// ============ +// +// Distant sync uses tunnels to sync subscribed GXS groups that are not supplied by friends. Peers can subscribe to a GXS group using a RS link +// which GXS uses to request updates through tunnels. +// * The whole exchange should be kept private and anonymous between the two distant peers, so we use the same trick than for FT: encrypt the data using the group ID. +// * The same node shouldn't be known as a common server for different GXS groups +// +// GXS net service: +// * talks to virtual peers, treated like normal peers +// * virtual peers only depend on the server ID, not on tunnel ID, and be kept constant accross time so that ClientGroupUpdateMap is kept consistent +// * does not use tunnels if friends can already supply the data (??) This causes issues with "islands". +// +// Tunnels: +// * a specific service named GxsSyncTunnelService handles the creation/management of sync tunnels: +// * tunnel data need to be encrypted. +// +// bool manageTunnels(const RsGxsGroupId&) ; // start managing tunnels for this group +// bool releaseTunnels(const RsGxsGroupId&) ; // stop managing tunnels for this group +// bool sendData(const unsigned char *data,uint32_t size,const RsPeerId& virtual_peer) ; // send data to this virtual peer +// bool getVirtualPeers(const RsGxsGroupId&, std::list& peers) ; // returns the virtual peers for this group +// +// Proposed protocol: +// * request tunnels based on H(GroupId) +// * encrypt tunnel data using chacha20+HMAC-SHA256 using AEAD( GroupId, 96bits IV, tunnel ID ) (similar to what FT does) +// * when tunnel is established, exchange virtual peer names: vpid = H( GroupID | Random bias ) +// * when vpid is known, notify the client (GXS net service) which can use the virtual peer to sync +// +// * only use a single tunnel per virtual peer ID +// +// Client ------------------ TR(H(GroupId)) --------------> Server +// +// Client <-------------------- T OK ---------------------- Server +// +// [Encrypted traffic using H(GroupId, 96bits IV, tunnel ID)] +// +// Client <--------- VPID = H( Random IV | GroupId ) ------ Server +// | | +// +--------------> Mark the virtual peer active <-----------+ +// +// Unsolved problems: +// * if we want to preserve anonymity, we cannot prevent GXS from duplicating the data from virtual/real peers that actually are the same peers. +// * ultimately we should only use tunnels to sync GXS. The mix between tunnels and real peers is not a problem but will cause unnecessary traffic. +// +// Notes: +// * given that GXS only talks to peers once every 2 mins, it's likely that keep-alive packets will be needed #include From a173f325a9198f70e8b9fad060ef928e7424792b Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Dec 2017 00:09:34 +0100 Subject: [PATCH 02/84] added .h for RsGxsNetTunnel service --- libretroshare/src/gxs/rsgxsnettunnel.cc | 0 libretroshare/src/gxs/rsgxsnettunnel.h | 148 ++++++++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 libretroshare/src/gxs/rsgxsnettunnel.cc create mode 100644 libretroshare/src/gxs/rsgxsnettunnel.h diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc new file mode 100644 index 000000000..e69de29bb diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h new file mode 100644 index 000000000..b831e5bbc --- /dev/null +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -0,0 +1,148 @@ +/* + * libretroshare/src/gxs: rsgxsnettunnel.h + * + * General Data service, interface for RetroShare. + * + * Copyright 2011-2011 by Robert Fernie, Evi-Parker Christopher + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com" + * + */ + +#include + +#include + +/*! + * \brief The RsGxsNetTunnelService class takes care of requesting tunnels to the turtle router, through which it is possible to sync + * a particular GXS group. For each group, a set of virtual peers, corresponding to active tunnels will be made available + * to RsGxsNetService. + * + * It is the responsibility of RsGxsNetService to activate/desactivate tunnels for each particular group, depending on wether the group + * is already available at friends or not. + */ + +// Proposed protocol: +// * request tunnels based on H(GroupId) +// * encrypt tunnel data using chacha20+HMAC-SHA256 using AEAD( GroupId, 96bits IV, tunnel ID ) (similar to what FT does) +// * when tunnel is established, exchange virtual peer names: vpid = H( GroupID | Random bias ) +// * when vpid is known, notify the client (GXS net service) which can use the virtual peer to sync +// +// * only use a single tunnel per virtual peer ID +// +// Client ------------------ TR(H(GroupId)) --------------> Server +// +// Client <-------------------- T OK ---------------------- Server +// +// Here, a turtle vpid is known +// +// [Encrypted traffic using H(GroupId, 96bits IV, tunnel ID)] +// +// Client <--------- VPID = H( Random IV | GroupId ) ------ Server +// | | +// +--------------> Mark the virtual peer active <-----------+ +// +// Here, a consistent virtual peer ID is known + +typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; + +struct RsGxsNetTunnelVirtualPeerInfo +{ + enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. + RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01, // tunnel has been established and we're waiting for virtual peer id + RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. + }; + + uint8_t vpid_status ; + RsGxsNetTunnelVirtualPeerId net_service_virtual_peer ; + uint8_t side ; // client/server +}; + +struct RsGxsNetTunnelInfo +{ + enum { RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status + RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED = 0x01, // waiting for turtle to send some virtual peers. + RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x02 // some virtual peers are available + }; + + uint8_t group_status ; + uint8_t encryption_master_key[16] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) + + std::map virtual_peers ; +}; + +class RsGxsNetTunnelService +{ +public: + RsGxsNetTunnelService() {} + + /*! + * \brief start managing tunnels for this group + * @param group_id group for which tunnels should be requested + */ + bool manageTunnels(const RsGxsGroupId&) ; + + /*! + * \brief Stop managing tunnels for this group + * @param group_id group for which tunnels should be released + */ + bool releaseTunnels(const RsGxsGroupId&) ; + + /*! + * sends data to this virtual peer ID + */ + bool sendData(const unsigned char *data,uint32_t size,const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; // send data to this virtual peer + + /*! + * \brief Get the list of active virtual peers for a given group. This implies that the tunnel is up and + * alive. + */ + bool getVirtualPeers(const RsGxsGroupId&, std::list& peers) ; // returns the virtual peers for this group + + /*! + * \brief dumps all information about monitored groups. + */ + void dump() const; + + // other methods are still missing. + // - derived from p3Config, to load/save data + // - method to respond to tunnel requests, probably using RsGxsNetService + // - method to encrypt/decrypt data and send/receive to/from turtle. + +private: + std::map mClientGroups ; // groups on the client side + std::map mServerGroups ; // groups on the server side + + std::map > mVirtualPeers ; + + /*! + * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to + * hide the real group id. + */ + + RsFileHash makeRequestHash(const RsGxsGroupId&) const ; + + /*! + * \brief makeVirtualPeerIdForGroup creates a virtual peer id that can be used and that will be constant accross time, whatever the + * tunnel ID and turtle virtual peer id. This allows RsGxsNetService to keep sync-ing the data consistently. + */ + + RsGxsNetTunnelVirtualPeerInfo makeVirtualPeerIdForGroup(const RsGxsGroupId&) const ; + + uint8_t mRandomBias[16] ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. +}; + From 76ec079b40b622e4c8266742efdebfc90f023865 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Dec 2017 22:14:58 +0100 Subject: [PATCH 03/84] added missing change in .pro --- libretroshare/src/libretroshare.pro | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 1521f5574..221fe5bd3 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -740,6 +740,7 @@ HEADERS += rsitems/rsnxsitems.h \ gxs/rsgxs.h \ gxs/rsdataservice.h \ gxs/rsgxsnetservice.h \ + gxs/rsgxsnettunnel.h \ retroshare/rsgxsflags.h \ retroshare/rsgxsifacetypes.h \ gxs/rsgenexchange.h \ @@ -765,6 +766,7 @@ SOURCES += rsitems/rsnxsitems.cc \ gxs/rsdataservice.cc \ gxs/rsgenexchange.cc \ gxs/rsgxsnetservice.cc \ + gxs/rsgxsnettunnel.cc \ gxs/rsgxsdata.cc \ rsitems/rsgxsitems.cc \ gxs/rsgxsdataaccess.cc \ From 076309133b5c0e5ddccce75d132a06f7c012d06d Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 15 Mar 2018 17:46:21 +0100 Subject: [PATCH 04/84] basic structure of GxsNetTunnelService --- libretroshare/src/gxs/rsgxsnettunnel.cc | 175 ++++++++++++++++++++++ libretroshare/src/gxs/rsgxsnettunnel.h | 85 +++++++---- libretroshare/src/gxstunnel/p3gxstunnel.h | 2 +- 3 files changed, 231 insertions(+), 31 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index e69de29bb..1e0e83cf7 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -0,0 +1,175 @@ +/* + * libretroshare/src/gxs: rsgxsnettunnel.cc + * + * General Data service, interface for RetroShare. + * + * Copyright 2018-2018 by Cyril Soler + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare.project@gmail.com" + * + */ + +#include "rsgxsnettunnel.h" + +#define DEBUG_RSGXSNETTUNNEL 1 + +#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } + +RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") {} + +bool RsGxsNetTunnelService::manage(const RsGxsGroupId& group_id) +{ + RsFileHash hash = calculateGroupHash(group_id) ; + + RsStackMutex stack(mGxsNetTunnelMtx); /********** STACK LOCKED MTX ******/ + + RsGxsNetTunnelGroupInfo& info(mClientGroups[group_id]) ; + + time_t now = time(NULL) ; + + if(info.group_status == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE) + return true; + + info.hash = hash ; + info.last_contact = now ; + info.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; + +#ifdef DEBUG_GXS_TUNNEL + std::cerr << "Starting distant chat to " << to_gxs_id << ", hash = " << hash << ", from " << from_gxs_id << std::endl; + std::cerr << "Asking turtle router to monitor tunnels for hash " << hash << std::endl; +#endif + + // Now ask the turtle router to manage a tunnel for that hash. + + mTurtle->monitorTunnels(hash,this,false) ; + + return true; +} + +bool RsGxsNetTunnelService::release(const RsGxsGroupId& group_id) +{ + RsStackMutex stack(mGxsNetTunnelMtx); /********** STACK LOCKED MTX ******/ + + // Here we need to clean the stuff that was created by this group id. + + auto it = mClientGroups.find(group_id) ; + + if(it == mClientGroups.end()) + { + std::cerr << "RsGxsNetTunnelService::release(): Weird. Cannot release client group " << group_id << " that is not known." << std::endl; + return false ; + } + + mClientGroups.erase(it) ; + return true ; +} + +bool RsGxsNetTunnelService::sendData(const unsigned char *data,uint32_t size,const RsGxsNetTunnelVirtualPeerId& virtual_peer) +{ + // The data is encrypted using chacha20+SHA256 and sent to the turtle router. + + NOT_IMPLEMENTED(); + return false ; +} + +bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId&, std::list& peers) +{ + // returns the virtual peers for this group + NOT_IMPLEMENTED(); + return false ; +} + +RsGxsNetTunnelVirtualPeerInfo RsGxsNetTunnelService::makeVirtualPeerIdForGroup(const RsGxsGroupId&) const +{ + NOT_IMPLEMENTED(); + return RsGxsNetTunnelVirtualPeerInfo(); +} +void RsGxsNetTunnelService::dump() const +{ + NOT_IMPLEMENTED(); +} + +//===========================================================================================================================================// +// Interaction with Turtle Router // +//===========================================================================================================================================// + +void RsGxsNetTunnelService::connectToTurtleRouter(p3turtle *tr) +{ + mTurtle = tr ; + mTurtle->registerTunnelService(this) ; +} + +bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) +{ + NOT_IMPLEMENTED(); + return false ; +} +void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) +{ + NOT_IMPLEMENTED(); +} +void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) +{ + NOT_IMPLEMENTED(); +} +void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) +{ + NOT_IMPLEMENTED(); +} + +RsFileHash RsGxsNetTunnelService::calculateGroupHash(const RsGxsGroupId&) const +{ + NOT_IMPLEMENTED(); + return RsFileHash() ; +} + +//===========================================================================================================================================// +// Service parts // +//===========================================================================================================================================// + +#ifdef TODO +void RsGxsNetTunnelService::handleIncomingItem(const RsGxsTunnelId& tunnel_id,RsGxsTunnelItem *item) +{ + if(item == NULL) + return ; + + // We have 3 things to do: + // + // 1 - if it's a data item, send an ACK + // 2 - if it's an ack item, mark the item as properly received, and remove it from the queue + // 3 - if it's a status item, act accordingly. + + switch(item->PacketSubType()) + { + + case RS_PKT_SUBTYPE_GXS_TUNNEL_DATA: handleRecvTunnelDataItem(tunnel_id,dynamic_cast(item)) ; + break ; + + case RS_PKT_SUBTYPE_GXS_TUNNEL_DATA_ACK: handleRecvTunnelDataAckItem(tunnel_id,dynamic_cast(item)) ; + break ; + + case RS_PKT_SUBTYPE_GXS_TUNNEL_STATUS: handleRecvStatusItem(tunnel_id,dynamic_cast(item)) ; + break ; + + default: + std::cerr << "(EE) impossible situation. DH items should be handled at the service level" << std::endl; + } + + delete item ; +} +#endif + diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index b831e5bbc..15c3b4f84 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -3,7 +3,7 @@ * * General Data service, interface for RetroShare. * - * Copyright 2011-2011 by Robert Fernie, Evi-Parker Christopher + * Copyright 2018-2018 by Cyril Soler * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Library General Public @@ -29,34 +29,40 @@ /*! * \brief The RsGxsNetTunnelService class takes care of requesting tunnels to the turtle router, through which it is possible to sync - * a particular GXS group. For each group, a set of virtual peers, corresponding to active tunnels will be made available - * to RsGxsNetService. + * a particular GXS group. For each group, a set of virtual peers, corresponding to active tunnels will be made available + * to RsGxsNetService. * - * It is the responsibility of RsGxsNetService to activate/desactivate tunnels for each particular group, depending on wether the group - * is already available at friends or not. + * It is the responsibility of RsGxsNetService to activate/desactivate tunnels for each particular group, depending on wether the group + * is already available at friends or not. */ -// Proposed protocol: +// Protocol: // * request tunnels based on H(GroupId) // * encrypt tunnel data using chacha20+HMAC-SHA256 using AEAD( GroupId, 96bits IV, tunnel ID ) (similar to what FT does) // * when tunnel is established, exchange virtual peer names: vpid = H( GroupID | Random bias ) // * when vpid is known, notify the client (GXS net service) which can use the virtual peer to sync // // * only use a single tunnel per virtual peer ID -// -// Client ------------------ TR(H(GroupId)) --------------> Server -// -// Client <-------------------- T OK ---------------------- Server -// -// Here, a turtle vpid is known -// -// [Encrypted traffic using H(GroupId, 96bits IV, tunnel ID)] -// -// Client <--------- VPID = H( Random IV | GroupId ) ------ Server -// | | -// +--------------> Mark the virtual peer active <-----------+ -// -// Here, a consistent virtual peer ID is known +// - +// Client ------------------ TR(H(GroupId)) --------------> Server | +// | Turtle +// Client <-------------------- T OK ---------------------- Server | +// - +// Here, a turtle vpid is known | [ addVirtualPeer() called by turtle ] +// - +// [Encrypted traffic using H(GroupId | Tunnel ID, 96bits IV)] | +// | +// Client <--------- VPID = H( Random IV | GroupId ) ------ Server | +// | | | +// +--------------> Mark the virtual peer active <-----------+ | Encrypted traffic decoded locally and sorted +// | +// Here, a consistent virtual peer ID is known | +// | +// Client <------------------- GXS Data ------------------> Server | +// - +// Notes: +// * tunnels are only used one-way. If a distant peers wants to sync the same group, he'll have to open his own tunnel, with a different ID. +// * each group will produced multiple tunnels, but each tunnel with have exactly one virtual peer ID typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; @@ -67,40 +73,47 @@ struct RsGxsNetTunnelVirtualPeerInfo RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. }; + RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN) { memset(encryption_master_key,0,16) ; } + uint8_t vpid_status ; RsGxsNetTunnelVirtualPeerId net_service_virtual_peer ; uint8_t side ; // client/server + uint8_t encryption_master_key[16] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) + time_t last_contact ; // last time some data was sent/recvd }; -struct RsGxsNetTunnelInfo +struct RsGxsNetTunnelGroupInfo { enum { RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED = 0x01, // waiting for turtle to send some virtual peers. RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x02 // some virtual peers are available }; - uint8_t group_status ; - uint8_t encryption_master_key[16] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) + RsGxsNetTunnelGroupInfo() : group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN),last_contact(0) {} + + uint8_t group_status ; + time_t last_contact ; + TurtleFileHash hash ; std::map virtual_peers ; }; -class RsGxsNetTunnelService +class RsGxsNetTunnelService: public RsTurtleClientService { public: - RsGxsNetTunnelService() {} + RsGxsNetTunnelService() ; /*! * \brief start managing tunnels for this group * @param group_id group for which tunnels should be requested */ - bool manageTunnels(const RsGxsGroupId&) ; + bool manage(const RsGxsGroupId& group_id) ; /*! * \brief Stop managing tunnels for this group * @param group_id group for which tunnels should be released */ - bool releaseTunnels(const RsGxsGroupId&) ; + bool release(const RsGxsGroupId&group_id) ; /*! * sends data to this virtual peer ID @@ -123,9 +136,19 @@ public: // - method to respond to tunnel requests, probably using RsGxsNetService // - method to encrypt/decrypt data and send/receive to/from turtle. + virtual void connectToTurtleRouter(p3turtle *tr) ; +protected: + // interaction with turtle router + + virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ; + virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; + void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ; + void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; + + p3turtle *mTurtle ; private: - std::map mClientGroups ; // groups on the client side - std::map mServerGroups ; // groups on the server side + std::map mClientGroups ; // groups on the client side + std::map mServerGroups ; // groups on the server side std::map > mVirtualPeers ; @@ -134,7 +157,7 @@ private: * hide the real group id. */ - RsFileHash makeRequestHash(const RsGxsGroupId&) const ; + RsFileHash calculateGroupHash(const RsGxsGroupId&) const ; /*! * \brief makeVirtualPeerIdForGroup creates a virtual peer id that can be used and that will be constant accross time, whatever the @@ -144,5 +167,7 @@ private: RsGxsNetTunnelVirtualPeerInfo makeVirtualPeerIdForGroup(const RsGxsGroupId&) const ; uint8_t mRandomBias[16] ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. + + RsMutex mGxsNetTunnelMtx; }; diff --git a/libretroshare/src/gxstunnel/p3gxstunnel.h b/libretroshare/src/gxstunnel/p3gxstunnel.h index cabe520d4..32020e619 100644 --- a/libretroshare/src/gxstunnel/p3gxstunnel.h +++ b/libretroshare/src/gxstunnel/p3gxstunnel.h @@ -95,7 +95,7 @@ // by a mix between our own GXS id and the GXS id we're talking to. That is what the TunnelVirtualPeer is. // // -// RequestTunnel(source_own_id,destination_id) - +// RequestTunnel(source_own_id,destination_id) - // | | // +---------------------------> p3Turtle::monitorTunnels( hash(destination_id) ) | // | | From 3407604a54e9704f6bf458c049a94475bd62715e Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 21 Mar 2018 22:09:40 +0100 Subject: [PATCH 05/84] added code to add/remove virtual peers in RsGxsNetTunnel --- libretroshare/src/gxs/rsgxsnettunnel.cc | 74 +++++++++++++++++++++---- libretroshare/src/gxs/rsgxsnettunnel.h | 11 ++-- 2 files changed, 70 insertions(+), 15 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 1e0e83cf7..2a960dd7a 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -23,6 +23,7 @@ * */ +#include "util/rsdir.h" #include "rsgxsnettunnel.h" #define DEBUG_RSGXSNETTUNNEL 1 @@ -31,6 +32,10 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") {} +//===========================================================================================================================================// +// Interface with rest of the software // +//===========================================================================================================================================// + bool RsGxsNetTunnelService::manage(const RsGxsGroupId& group_id) { RsFileHash hash = calculateGroupHash(group_id) ; @@ -48,6 +53,8 @@ bool RsGxsNetTunnelService::manage(const RsGxsGroupId& group_id) info.last_contact = now ; info.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; + mHandledHashes[hash] = group_id ; + #ifdef DEBUG_GXS_TUNNEL std::cerr << "Starting distant chat to " << to_gxs_id << ", hash = " << hash << ", from " << from_gxs_id << std::endl; std::cerr << "Asking turtle router to monitor tunnels for hash " << hash << std::endl; @@ -75,6 +82,10 @@ bool RsGxsNetTunnelService::release(const RsGxsGroupId& group_id) } mClientGroups.erase(it) ; + + RsFileHash hash = calculateGroupHash(group_id) ; + + mHandledHashes.erase(hash) ; return true ; } @@ -93,7 +104,7 @@ bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId&, std::listsecond) ; + + RsGxsNetTunnelGroupInfo& ginfo( mClientGroups[group_id] ) ; + ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE ; + + RsGxsNetTunnelVirtualPeerInfo& vpinfo( ginfo.virtual_peers[vpid] ) ; + + vpinfo.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK ; + vpinfo.net_service_virtual_peer.clear(); + vpinfo.side = dir ; + vpinfo.last_contact = time(NULL) ; + + generateEncryptionKey(group_id,vpid,vpinfo.encryption_key ); } -RsFileHash RsGxsNetTunnelService::calculateGroupHash(const RsGxsGroupId&) const +void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid) { - NOT_IMPLEMENTED(); - return RsFileHash() ; + auto it = mHandledHashes.find(hash) ; + + if(it == mHandledHashes.end()) + { + std::cerr << "RsGxsNetTunnelService::removeVirtualPeer(): error! hash " << hash << " is not handled. Cannot remove vpid " << vpid << std::endl; + return ; + } + + const RsGxsGroupId group_id(it->second) ; + + RsGxsNetTunnelGroupInfo& ginfo( mClientGroups[group_id] ) ; + + ginfo.virtual_peers.erase(vpid); + + if(ginfo.virtual_peers.empty()) + ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED ; +} + +RsFileHash RsGxsNetTunnelService::calculateGroupHash(const RsGxsGroupId& group_id) const +{ + return RsDirUtil::sha1sum(group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; } //===========================================================================================================================================// diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 15c3b4f84..03a676c07 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -77,7 +77,7 @@ struct RsGxsNetTunnelVirtualPeerInfo uint8_t vpid_status ; RsGxsNetTunnelVirtualPeerId net_service_virtual_peer ; - uint8_t side ; // client/server + uint8_t side ; // client/server uint8_t encryption_master_key[16] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) time_t last_contact ; // last time some data was sent/recvd }; @@ -150,21 +150,24 @@ private: std::map mClientGroups ; // groups on the client side std::map mServerGroups ; // groups on the server side - std::map > mVirtualPeers ; + std::map > mVirtualPeers ; // current virtual peers, with the (group,turtle vpid) they are for + std::map mHandledHashes ; // hashes asked to turtle /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to * hide the real group id. */ - RsFileHash calculateGroupHash(const RsGxsGroupId&) const ; + RsFileHash calculateGroupHash(const RsGxsGroupId&group_id) const ; /*! * \brief makeVirtualPeerIdForGroup creates a virtual peer id that can be used and that will be constant accross time, whatever the * tunnel ID and turtle virtual peer id. This allows RsGxsNetService to keep sync-ing the data consistently. */ - RsGxsNetTunnelVirtualPeerInfo makeVirtualPeerIdForGroup(const RsGxsGroupId&) const ; + RsGxsNetTunnelVirtualPeerInfo makeVirtualPeerIdForGroup(const RsGxsGroupId&group_id) const ; + + void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid) const ; uint8_t mRandomBias[16] ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. From 00a6bd5b73b0a7a76ad46e4f9535f4d69cd7e193 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 22 Mar 2018 14:41:50 +0100 Subject: [PATCH 06/84] started to move FT encryption into p3turtle --- libretroshare/src/gxs/rsgxsnettunnel.cc | 144 ++++++++++++++++++-- libretroshare/src/gxs/rsgxsnettunnel.h | 45 +++++-- libretroshare/src/turtle/p3turtle.cc | 172 ++++++++++++++++++++++++ libretroshare/src/turtle/p3turtle.h | 7 + 4 files changed, 342 insertions(+), 26 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 2a960dd7a..e9c389454 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -24,6 +24,7 @@ */ #include "util/rsdir.h" +#include "retroshare/rspeers.h" #include "rsgxsnettunnel.h" #define DEBUG_RSGXSNETTUNNEL 1 @@ -42,7 +43,7 @@ bool RsGxsNetTunnelService::manage(const RsGxsGroupId& group_id) RsStackMutex stack(mGxsNetTunnelMtx); /********** STACK LOCKED MTX ******/ - RsGxsNetTunnelGroupInfo& info(mClientGroups[group_id]) ; + RsGxsNetTunnelGroupInfo& info(mGroups[group_id]) ; time_t now = time(NULL) ; @@ -73,15 +74,15 @@ bool RsGxsNetTunnelService::release(const RsGxsGroupId& group_id) // Here we need to clean the stuff that was created by this group id. - auto it = mClientGroups.find(group_id) ; + auto it = mGroups.find(group_id) ; - if(it == mClientGroups.end()) + if(it == mGroups.end()) { std::cerr << "RsGxsNetTunnelService::release(): Weird. Cannot release client group " << group_id << " that is not known." << std::endl; return false ; } - mClientGroups.erase(it) ; + mGroups.erase(it) ; RsFileHash hash = calculateGroupHash(group_id) ; @@ -89,12 +90,66 @@ bool RsGxsNetTunnelService::release(const RsGxsGroupId& group_id) return true ; } -bool RsGxsNetTunnelService::sendData(const unsigned char *data,uint32_t size,const RsGxsNetTunnelVirtualPeerId& virtual_peer) +class ItemAutoDelete { - // The data is encrypted using chacha20+SHA256 and sent to the turtle router. +public: + ItemAutoDelete(RsItem *& item) : mItem(item) {} + ~ItemAutoDelete() { delete mItem; mItem=NULL ; } + RsItem *& mItem; +}; + +bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualPeerId& virtual_peer) +{ + // The item is serialized and encrypted using chacha20+SHA256, using the generic turtle encryption, and then sent to the turtle router. + + ItemAutoDelete iad(item) ; // This ensures the item is deleted whatsoever when leaving + + // 1 - find the virtual peer and the proper master key to encrypt with, and check that all the info is known + + auto it = mVirtualPeers.find(virtual_peer) ; + + if(it == mVirtualPeers.end()) + { + std::cerr << "(EE) RsGxsNetTunnelService::sendData(): cannot find virtual peer " << virtual_peer << ". Data is dropped." << std::endl; + return false ; + } + + auto it2 = mGroups.find(it->second.first); + + if(it2 == mGroups.end()) + { + std::cerr << "(EE) RsGxsNetTunnelService::sendData(): cannot find virtual peer " << virtual_peer << ". Data is dropped." << std::endl; + return false ; + } + + auto it3 = it2->second.virtual_peers.find(it->second.second); + + if(it3 == it2->second.virtual_peers.end()) + { + std::cerr << "(EE) RsGxsNetTunnelService::sendData(): cannot find turtle virtual peer " << it->second.second << ". Data is dropped." << std::endl; + return false ; + } + + if(it3->second.vpid_status != RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE) + { + std::cerr << "(EE) RsGxsNetTunnelService::sendData(): virtual peer " << it->second.second << " is not active. Data is dropped." << std::endl; + return false ; + } + + // 2 - encrypt and send the item. + + RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; NOT_IMPLEMENTED(); - return false ; + // if(!p3Turtle::encryptItem(item,it3->second.encryption_master_key,encrypted_turtle_item)) + // { + // std::cerr << "(EE) RsGxsNetTunnelService::sendData(): cannot encrypt. Something's wrong. Data is dropped." << std::endl; + // return false ; + // } + + mTurtle->sendTurtleData(it->second.second,encrypted_turtle_item) ; + + return true ; } bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId&, std::list& peers) @@ -104,14 +159,60 @@ bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId&, std::listgetOwnId() ; + + unsigned char mem[RsPeerId::SIZE_IN_BYTES + RsGxsGroupId::SIZE_IN_BYTES + RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE]; + + memcpy(mem ,ssl_id.toByteArray() ,RsPeerId::SIZE_IN_BYTES) ; + memcpy(mem+RsPeerId::SIZE_IN_BYTES ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; + memcpy(mem+RsPeerId::SIZE_IN_BYTES+RsGxsGroupId::SIZE_IN_BYTES,mRandomBias ,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; + + return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsPeerId::SIZE_IN_BYTES+RsGxsGroupId::SIZE_IN_BYTES+RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE).toByteArray()); } + void RsGxsNetTunnelService::dump() const { - NOT_IMPLEMENTED(); + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + static std::string group_status_str[3] = { + std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN ]"), + std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED]"), + std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE ]") + }; + + static std::string vpid_status_str[3] = { + std::string("[RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN ]"), + std::string("[RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK ]"), + std::string("[RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ]") + }; + + std::cerr << "GxsNetTunnelService dump: " << std::endl; + std::cerr << "Managed GXS groups: " << std::endl; + + for(auto it(mGroups.begin());it!=mGroups.end();++it) + { + std::cerr << " " << it->first << " hash: " << it->second.hash << " status: " << group_status_str[it->second.group_status] << "] Last contact: " << time(NULL) - it->second.last_contact << " secs ago" << std::endl; + + for(auto it2(it->second.virtual_peers.begin());it2!=it->second.virtual_peers.end();++it2) + std::cerr << " turtle:" << it2->first << " status: " << vpid_status_str[it2->second.vpid_status] << " s: " + << (int)it2->second.side << " last seen " << time(NULL)-it2->second.last_contact + << " ekey: " << RsUtil::BinToHex(it2->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE) + << " pending (" << it2->second.incoming_items.size() << "," << it2->second.outgoing_items.size() << ")" << std::endl; + } + + std::cerr << "Virtual peers: " << std::endl; + for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) + std::cerr << " GXS Peer:" << it->first << " group_id: " << it->second.first << " Turtle:" << it->second.second << std::endl; + + std::cerr << "Hashes: " << std::endl; + for(auto it(mHandledHashes.begin());it!=mHandledHashes.end();++it) + std::cerr << " hash: " << it->first << " GroupId: " << it->second << std::endl; } //===========================================================================================================================================// @@ -151,7 +252,7 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur const RsGxsGroupId group_id(it->second) ; - RsGxsNetTunnelGroupInfo& ginfo( mClientGroups[group_id] ) ; + RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE ; RsGxsNetTunnelVirtualPeerInfo& vpinfo( ginfo.virtual_peers[vpid] ) ; @@ -161,7 +262,7 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur vpinfo.side = dir ; vpinfo.last_contact = time(NULL) ; - generateEncryptionKey(group_id,vpid,vpinfo.encryption_key ); + generateEncryptionKey(group_id,vpid,vpinfo.encryption_master_key ); } void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid) @@ -176,7 +277,7 @@ void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const const RsGxsGroupId group_id(it->second) ; - RsGxsNetTunnelGroupInfo& ginfo( mClientGroups[group_id] ) ; + RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; ginfo.virtual_peers.erase(vpid); @@ -189,6 +290,21 @@ RsFileHash RsGxsNetTunnelService::calculateGroupHash(const RsGxsGroupId& group_i return RsDirUtil::sha1sum(group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; } +void RsGxsNetTunnelService::generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) const +{ + // The key is generated as H(group_id | vpid) + // Because group_id is not known it shouldn't be possible to recover the key by observing the traffic. + + assert(Sha256CheckSum::SIZE_IN_BYTES == 32) ; + + unsigned char mem[RsGxsGroupId::SIZE_IN_BYTES + TurtleVirtualPeerId::SIZE_IN_BYTES] ; + + memcpy(mem ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; + memcpy(mem+RsGxsGroupId::SIZE_IN_BYTES,vpid.toByteArray() ,TurtleVirtualPeerId::SIZE_IN_BYTES) ; + + memcpy( key, RsDirUtil::sha256sum(mem,RsGxsGroupId::SIZE_IN_BYTES+TurtleVirtualPeerId::SIZE_IN_BYTES).toByteArray(), RS_GXS_TUNNEL_CONST_EKEY_SIZE ) ; +} + //===========================================================================================================================================// // Service parts // //===========================================================================================================================================// diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 03a676c07..2f6cb0eae 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -75,11 +75,16 @@ struct RsGxsNetTunnelVirtualPeerInfo RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN) { memset(encryption_master_key,0,16) ; } - uint8_t vpid_status ; - RsGxsNetTunnelVirtualPeerId net_service_virtual_peer ; + uint8_t vpid_status ; // status of the peer uint8_t side ; // client/server uint8_t encryption_master_key[16] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) time_t last_contact ; // last time some data was sent/recvd + + RsGxsNetTunnelVirtualPeerId net_service_virtual_peer ; // anonymised peer that is used to communicate with client services + RsGxsGroupId group_id ; // group id + + std::list incoming_items ; + std::list outgoing_items ; }; struct RsGxsNetTunnelGroupInfo @@ -115,10 +120,24 @@ public: */ bool release(const RsGxsGroupId&group_id) ; + /*! - * sends data to this virtual peer ID + * \brief sendItem + * send data to this virtual peer, and takes memory ownership (deletes the item) + * \param item item to send + * \param virtual_peer destination virtual peer + * \return + * true if succeeded. */ - bool sendData(const unsigned char *data,uint32_t size,const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; // send data to this virtual peer + bool sendItem(RsItem *& item, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + + /*! + * \brief receivedItem + * returns the next received item from the given virtual peer. + * \param virtual_peer + * \return + */ + RsItem *receivedItem(const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! * \brief Get the list of active virtual peers for a given group. This implies that the tunnel is up and @@ -147,11 +166,13 @@ protected: p3turtle *mTurtle ; private: - std::map mClientGroups ; // groups on the client side - std::map mServerGroups ; // groups on the server side + static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 16 ; + static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE = 32 ; - std::map > mVirtualPeers ; // current virtual peers, with the (group,turtle vpid) they are for - std::map mHandledHashes ; // hashes asked to turtle + std::map mGroups ; // groups on the client and server side + + std::map > mVirtualPeers ; // current virtual peers, + std::map mHandledHashes ; // hashes asked to turtle /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to @@ -165,12 +186,12 @@ private: * tunnel ID and turtle virtual peer id. This allows RsGxsNetService to keep sync-ing the data consistently. */ - RsGxsNetTunnelVirtualPeerInfo makeVirtualPeerIdForGroup(const RsGxsGroupId&group_id) const ; + RsGxsNetTunnelVirtualPeerId makeServerVirtualPeerIdForGroup(const RsGxsGroupId&group_id) const ; - void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid) const ; + void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) const ; - uint8_t mRandomBias[16] ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. + uint8_t mRandomBias[RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE] ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. - RsMutex mGxsNetTunnelMtx; + mutable RsMutex mGxsNetTunnelMtx; }; diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 8217db2fa..f01f7390e 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -2080,6 +2080,178 @@ std::string p3turtle::getPeerNameForVirtualPeerId(const RsPeerId& virtual_peer_i return name; } +#ifdef TODO +static const uint32_t ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE = 12 ; +static const uint32_t ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE = 16 ; +static const uint32_t ENCRYPTED_FT_HEADER_SIZE = 4 ; +static const uint32_t ENCRYPTED_FT_EDATA_SIZE = 4 ; + +static const uint8_t ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_POLY1305 = 0x01 ; +static const uint8_t ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256 = 0x02 ; + +bool p3turtle::encryptItem(const RsItem *item,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item) +{ + uint8_t initialization_vector[ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE] ; + + RSRandom::random_bytes(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) ; + +#ifdef SERVER_DEBUG + FTSERVER_DEBUG() << "ftServer::Encrypting ft item." << std::endl; + FTSERVER_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) << std::endl; +#endif + + uint32_t item_serialized_size = RsGenericSerializer().size(clear_item) ; + uint32_t total_data_size = ENCRYPTED_FT_HEADER_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_EDATA_SIZE + item_serialized_size + ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE ; + +#ifdef SERVER_DEBUG + FTSERVER_DEBUG() << " clear part size : " << size(clear_item) << std::endl; + FTSERVER_DEBUG() << " total item size : " << total_data_size << std::endl; +#endif + + encrypted_item = new RsTurtleGenericDataItem ; + encrypted_item->data_bytes = rs_malloc( total_data_size ) ; + encrypted_item->data_size = total_data_size ; + + if(encrypted_item->data_bytes == NULL) + return false ; + + uint8_t *edata = (uint8_t*)encrypted_item->data_bytes ; + uint32_t edata_size = item_serialized_size; + uint32_t offset = 0; + + edata[0] = 0xae ; + edata[1] = 0xad ; + edata[2] = ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256 ; // means AEAD_chacha20_sha256 + edata[3] = 0x01 ; + + offset += ENCRYPTED_FT_HEADER_SIZE; + uint32_t aad_offset = offset ; + uint32_t aad_size = ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_EDATA_SIZE ; + + memcpy(&edata[offset], initialization_vector, ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) ; + offset += ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE ; + + edata[offset+0] = (edata_size >> 0) & 0xff ; + edata[offset+1] = (edata_size >> 8) & 0xff ; + edata[offset+2] = (edata_size >> 16) & 0xff ; + edata[offset+3] = (edata_size >> 24) & 0xff ; + + offset += ENCRYPTED_FT_EDATA_SIZE ; + + uint32_t ser_size = (uint32_t)((int)total_data_size - (int)offset); + + serialise(clear_item,&edata[offset], &ser_size); + +#ifdef SERVER_DEBUG + FTSERVER_DEBUG() << " clear item : " << RsUtil::BinToHex(&edata[offset],std::min(50,(int)total_data_size-(int)offset)) << "(...)" << std::endl; +#endif + + uint32_t clear_item_offset = offset ; + offset += edata_size ; + + uint32_t authentication_tag_offset = offset ; + assert(ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE + offset == total_data_size) ; + + uint8_t encryption_key[32] ; + deriveEncryptionKey(hash,encryption_key) ; + + if(edata[2] == ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_POLY1305) + librs::crypto::AEAD_chacha20_poly1305(encryption_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; + else if(edata[2] == ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256) + librs::crypto::AEAD_chacha20_sha256 (encryption_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; + else + return false ; + +#ifdef SERVER_DEBUG + FTSERVER_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; + FTSERVER_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE) << std::endl; + FTSERVER_DEBUG() << " final item : " << RsUtil::BinToHex(&edata[0],std::min(50u,total_data_size)) << "(...)" << std::endl; +#endif + + return true ; +} + +// Decrypts the given item using aead-chacha20-poly1305 +bool p3turtle::decryptItem(RsTurtleGenericDataItem *item, uint8_t* encryption_master_key, RsItem *& decrypted_item) +{ + uint8_t encryption_key[32] ; + deriveEncryptionKey(hash,encryption_key) ; + + uint8_t *edata = (uint8_t*)encrypted_item->data_bytes ; + uint32_t offset = 0; + + if(encrypted_item->data_size < ENCRYPTED_FT_HEADER_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_EDATA_SIZE) return false ; + + if(edata[0] != 0xae) return false ; + if(edata[1] != 0xad) return false ; + if(edata[2] != ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_POLY1305 && edata[2] != ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256) return false ; + if(edata[3] != 0x01) return false ; + + offset += ENCRYPTED_FT_HEADER_SIZE ; + uint32_t aad_offset = offset ; + uint32_t aad_size = ENCRYPTED_FT_EDATA_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE ; + + uint8_t *initialization_vector = &edata[offset] ; + +#ifdef SERVER_DEBUG + FTSERVER_DEBUG() << "ftServer::decrypting ft item." << std::endl; + FTSERVER_DEBUG() << " item data : " << RsUtil::BinToHex(edata,std::min(50u,encrypted_item->data_size)) << "(...)" << std::endl; + FTSERVER_DEBUG() << " hash : " << hash << std::endl; + FTSERVER_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; + FTSERVER_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) << std::endl; +#endif + + offset += ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE ; + + uint32_t edata_size = 0 ; + edata_size += ((uint32_t)edata[offset+0]) << 0 ; + edata_size += ((uint32_t)edata[offset+1]) << 8 ; + edata_size += ((uint32_t)edata[offset+2]) << 16 ; + edata_size += ((uint32_t)edata[offset+3]) << 24 ; + + if(edata_size + ENCRYPTED_FT_EDATA_SIZE + ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_HEADER_SIZE != encrypted_item->data_size) + { + FTSERVER_ERROR() << " ERROR: encrypted data size is " << edata_size << ", should be " << encrypted_item->data_size - (ENCRYPTED_FT_EDATA_SIZE + ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_HEADER_SIZE ) << std::endl; + return false ; + } + + offset += ENCRYPTED_FT_EDATA_SIZE ; + uint32_t clear_item_offset = offset ; + + uint32_t authentication_tag_offset = offset + edata_size ; +#ifdef SERVER_DEBUG + FTSERVER_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE) << std::endl; +#endif + + bool result ; + + if(edata[2] == ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_POLY1305) + result = librs::crypto::AEAD_chacha20_poly1305(encryption_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; + else if(edata[2] == ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256) + result = librs::crypto::AEAD_chacha20_sha256 (encryption_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; + else + return false ; + +#ifdef SERVER_DEBUG + FTSERVER_DEBUG() << " authen. result : " << result << std::endl; + FTSERVER_DEBUG() << " decrypted daya : " << RsUtil::BinToHex(&edata[clear_item_offset],std::min(50u,edata_size)) << "(...)" << std::endl; +#endif + + if(!result) + { + FTSERVER_ERROR() << "(EE) decryption/authentication went wrong." << std::endl; + return false ; + } + + decrypted_item = dynamic_cast(deserialise(&edata[clear_item_offset],&edata_size)) ; + + if(decrypted_item == NULL) + return false ; + + return true ; +} +#endif + void p3turtle::getInfo( std::vector >& hashes_info, std::vector >& tunnels_info, std::vector& search_reqs_info, diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index 0c230b88b..2f7512753 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -332,6 +332,13 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config /// Send a data request into the correct tunnel for the given file hash void sendTurtleData(const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem *item) ; +#ifdef TODO + /// Encrypts/decrypts an item, using a autenticated construction + chacha20, based on the given 32 bytes master key. + /// + static bool encryptItem(const RsItem *item, uint8_t* encryption_master_key, RsTurtleGenericDataItem *& encrypted_item) ; + static bool decryptItem(RsTurtleGenericDataItem *item,uint8_t *encryption_master_key,RsItem *& decrypted_item) ; +#endif + private: //--------------------------- Admin/Helper functions -------------------------// From 58aa2413b31b52e0d1106a4fd979226d1507ee71 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 23 Mar 2018 22:46:04 +0100 Subject: [PATCH 07/84] finished moving tunnel encryption into turtle --- libretroshare/src/ft/ftserver.cc | 38 ++++++++ libretroshare/src/turtle/p3turtle.cc | 131 ++++++++++++++------------- libretroshare/src/turtle/p3turtle.h | 7 +- 3 files changed, 109 insertions(+), 67 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index f2200d7db..47718caaa 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -1219,6 +1219,21 @@ static const uint8_t ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256 = 0x02 ; bool ftServer::encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHash& hash,RsTurtleGenericDataItem *& encrypted_item) { +#ifndef USE_NEW_METHOD + uint32_t item_serialized_size = size(clear_item) ; + + RsTemporaryMemory data(item_serialized_size) ; + + if(data == NULL) + return false ; + + serialise(clear_item, data, &item_serialized_size); + + uint8_t encryption_key[32] ; + deriveEncryptionKey(hash,encryption_key) ; + + return p3turtle::encryptData(data,item_serialized_size,encryption_key,encrypted_item) ; +#else uint8_t initialization_vector[ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE] ; RSRandom::random_bytes(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) ; @@ -1297,12 +1312,34 @@ bool ftServer::encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHas #endif return true ; +#endif } // Decrypts the given item using aead-chacha20-poly1305 bool ftServer::decryptItem(RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item) { +#ifndef USE_NEW_METHOD + unsigned char *data = NULL ; + uint32_t data_size = 0 ; + + uint8_t encryption_key[32] ; + deriveEncryptionKey(hash,encryption_key) ; + + if(!p3turtle::decryptItem(encrypted_item,encryption_key,data,data_size)) + { + FTSERVER_ERROR() << "Cannot decrypt data!" << std::endl; + + if(data) + free(data) ; + return false ; + } + decrypted_item = dynamic_cast(deserialise(data,&data_size)) ; + free(data); + + return (decrypted_item != NULL); + +#else uint8_t encryption_key[32] ; deriveEncryptionKey(hash,encryption_key) ; @@ -1378,6 +1415,7 @@ bool ftServer::decryptItem(RsTurtleGenericDataItem *encrypted_item,const RsFileH return false ; return true ; +#endif } bool ftServer::encryptHash(const RsFileHash& hash, RsFileHash& hash_of_hash) diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index f01f7390e..ca337f241 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -34,6 +34,7 @@ #endif #include "rsserver/p3face.h" +#include "crypto/chacha20.h" #include "pqi/authssl.h" #include "pqi/p3linkmgr.h" @@ -63,6 +64,9 @@ static std::map > TS_request_bounces void TS_dumpState() ; #endif +#define TURTLE_DEBUG() std::cerr << time(NULL) << " : TURTLE : " << __FUNCTION__ << " : " +#define TURTLE_ERROR() std::cerr << "(EE) TURTLE ERROR : " + // These number may be quite important. I setup them with sensible values, but // an in-depth test would be better to get an idea of what the ideal values // could ever be. @@ -2080,32 +2084,31 @@ std::string p3turtle::getPeerNameForVirtualPeerId(const RsPeerId& virtual_peer_i return name; } -#ifdef TODO -static const uint32_t ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE = 12 ; -static const uint32_t ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE = 16 ; -static const uint32_t ENCRYPTED_FT_HEADER_SIZE = 4 ; -static const uint32_t ENCRYPTED_FT_EDATA_SIZE = 4 ; +static const uint32_t ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE = 12 ; +static const uint32_t ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE = 16 ; +static const uint32_t ENCRYPTED_TURTLE_HEADER_SIZE = 4 ; +static const uint32_t ENCRYPTED_TURTLE_EDATA_SIZE = 4 ; -static const uint8_t ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_POLY1305 = 0x01 ; -static const uint8_t ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256 = 0x02 ; +static const uint8_t ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_POLY1305 = 0x01 ; +static const uint8_t ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256 = 0x02 ; -bool p3turtle::encryptItem(const RsItem *item,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item) +bool p3turtle::encryptData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item) { - uint8_t initialization_vector[ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE] ; + uint8_t initialization_vector[ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE] ; - RSRandom::random_bytes(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) ; + RSRandom::random_bytes(initialization_vector,ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) ; #ifdef SERVER_DEBUG - FTSERVER_DEBUG() << "ftServer::Encrypting ft item." << std::endl; - FTSERVER_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) << std::endl; + TURTLE_DEBUG() << "ftServer::Encrypting ft item." << std::endl; + TURTLE_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) << std::endl; #endif - uint32_t item_serialized_size = RsGenericSerializer().size(clear_item) ; - uint32_t total_data_size = ENCRYPTED_FT_HEADER_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_EDATA_SIZE + item_serialized_size + ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE ; + uint32_t item_serialized_size = clear_data_size;//RsGenericSerializer().size(clear_item) ; + uint32_t total_data_size = ENCRYPTED_TURTLE_HEADER_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_EDATA_SIZE + item_serialized_size + ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE ; #ifdef SERVER_DEBUG - FTSERVER_DEBUG() << " clear part size : " << size(clear_item) << std::endl; - FTSERVER_DEBUG() << " total item size : " << total_data_size << std::endl; + TURTLE_DEBUG() << " clear part size : " << size(clear_item) << std::endl; + TURTLE_DEBUG() << " total item size : " << total_data_size << std::endl; #endif encrypted_item = new RsTurtleGenericDataItem ; @@ -2121,87 +2124,85 @@ bool p3turtle::encryptItem(const RsItem *item,uint8_t *encryption_master_key,RsT edata[0] = 0xae ; edata[1] = 0xad ; - edata[2] = ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256 ; // means AEAD_chacha20_sha256 + edata[2] = ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256 ; // means AEAD_chacha20_sha256 edata[3] = 0x01 ; - offset += ENCRYPTED_FT_HEADER_SIZE; + offset += ENCRYPTED_TURTLE_HEADER_SIZE; uint32_t aad_offset = offset ; - uint32_t aad_size = ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_EDATA_SIZE ; + uint32_t aad_size = ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_EDATA_SIZE ; - memcpy(&edata[offset], initialization_vector, ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) ; - offset += ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE ; + memcpy(&edata[offset], initialization_vector, ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) ; + offset += ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE ; edata[offset+0] = (edata_size >> 0) & 0xff ; edata[offset+1] = (edata_size >> 8) & 0xff ; edata[offset+2] = (edata_size >> 16) & 0xff ; edata[offset+3] = (edata_size >> 24) & 0xff ; - offset += ENCRYPTED_FT_EDATA_SIZE ; + offset += ENCRYPTED_TURTLE_EDATA_SIZE ; - uint32_t ser_size = (uint32_t)((int)total_data_size - (int)offset); - - serialise(clear_item,&edata[offset], &ser_size); + memcpy(&edata[offset],clear_data,clear_data_size); #ifdef SERVER_DEBUG - FTSERVER_DEBUG() << " clear item : " << RsUtil::BinToHex(&edata[offset],std::min(50,(int)total_data_size-(int)offset)) << "(...)" << std::endl; + TURTLE_DEBUG() << " clear item : " << RsUtil::BinToHex(&edata[offset],std::min(50,(int)total_data_size-(int)offset)) << "(...)" << std::endl; #endif uint32_t clear_item_offset = offset ; offset += edata_size ; uint32_t authentication_tag_offset = offset ; - assert(ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE + offset == total_data_size) ; + assert(ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE + offset == total_data_size) ; - uint8_t encryption_key[32] ; - deriveEncryptionKey(hash,encryption_key) ; + //uint8_t encryption_key[32] ; + //deriveEncryptionKey(hash,encryption_key) ; - if(edata[2] == ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_POLY1305) - librs::crypto::AEAD_chacha20_poly1305(encryption_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; - else if(edata[2] == ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256) - librs::crypto::AEAD_chacha20_sha256 (encryption_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; + if(edata[2] == ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_POLY1305) + librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; + else if(edata[2] == ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256) + librs::crypto::AEAD_chacha20_sha256 (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; else return false ; #ifdef SERVER_DEBUG - FTSERVER_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; - FTSERVER_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE) << std::endl; - FTSERVER_DEBUG() << " final item : " << RsUtil::BinToHex(&edata[0],std::min(50u,total_data_size)) << "(...)" << std::endl; + TURTLE_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; + TURTLE_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE) << std::endl; + TURTLE_DEBUG() << " final item : " << RsUtil::BinToHex(&edata[0],std::min(50u,total_data_size)) << "(...)" << std::endl; #endif return true ; } // Decrypts the given item using aead-chacha20-poly1305 -bool p3turtle::decryptItem(RsTurtleGenericDataItem *item, uint8_t* encryption_master_key, RsItem *& decrypted_item) +bool p3turtle::decryptItem(const RsTurtleGenericDataItem* encrypted_item, uint8_t *encryption_master_key, unsigned char *& decrypted_data, uint32_t& decrypted_data_size) { - uint8_t encryption_key[32] ; - deriveEncryptionKey(hash,encryption_key) ; + //uint8_t encryption_key[32] ; + //deriveEncryptionKey(hash,encryption_key) ; uint8_t *edata = (uint8_t*)encrypted_item->data_bytes ; uint32_t offset = 0; - if(encrypted_item->data_size < ENCRYPTED_FT_HEADER_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_EDATA_SIZE) return false ; + if(encrypted_item->data_size < ENCRYPTED_TURTLE_HEADER_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_EDATA_SIZE) return false ; if(edata[0] != 0xae) return false ; if(edata[1] != 0xad) return false ; - if(edata[2] != ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_POLY1305 && edata[2] != ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256) return false ; + if(edata[2] != ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_POLY1305 && edata[2] != ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256) return false ; if(edata[3] != 0x01) return false ; - offset += ENCRYPTED_FT_HEADER_SIZE ; + offset += ENCRYPTED_TURTLE_HEADER_SIZE ; uint32_t aad_offset = offset ; - uint32_t aad_size = ENCRYPTED_FT_EDATA_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE ; + uint32_t aad_size = ENCRYPTED_TURTLE_EDATA_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE ; uint8_t *initialization_vector = &edata[offset] ; #ifdef SERVER_DEBUG - FTSERVER_DEBUG() << "ftServer::decrypting ft item." << std::endl; - FTSERVER_DEBUG() << " item data : " << RsUtil::BinToHex(edata,std::min(50u,encrypted_item->data_size)) << "(...)" << std::endl; - FTSERVER_DEBUG() << " hash : " << hash << std::endl; - FTSERVER_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; - FTSERVER_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) << std::endl; + TURTLE_DEBUG() << "ftServer::decrypting ft item." << std::endl; + TURTLE_DEBUG() << " item data : " << RsUtil::BinToHex(edata,std::min(50u,encrypted_item->data_size)) << "(...)" << std::endl; + TURTLE_DEBUG() << " hash : " << hash << std::endl; + TURTLE_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; + TURTLE_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) << std::endl; #endif - offset += ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE ; + offset += ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE ; uint32_t edata_size = 0 ; edata_size += ((uint32_t)edata[offset+0]) << 0 ; @@ -2209,48 +2210,52 @@ bool p3turtle::decryptItem(RsTurtleGenericDataItem *item, uint8_t* encryption_ma edata_size += ((uint32_t)edata[offset+2]) << 16 ; edata_size += ((uint32_t)edata[offset+3]) << 24 ; - if(edata_size + ENCRYPTED_FT_EDATA_SIZE + ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_HEADER_SIZE != encrypted_item->data_size) + if(edata_size + ENCRYPTED_TURTLE_EDATA_SIZE + ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_HEADER_SIZE != encrypted_item->data_size) { - FTSERVER_ERROR() << " ERROR: encrypted data size is " << edata_size << ", should be " << encrypted_item->data_size - (ENCRYPTED_FT_EDATA_SIZE + ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE + ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_FT_HEADER_SIZE ) << std::endl; + TURTLE_ERROR() << " ERROR: encrypted data size is " << edata_size << ", should be " << encrypted_item->data_size - (ENCRYPTED_TURTLE_EDATA_SIZE + ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_HEADER_SIZE ) << std::endl; return false ; } - offset += ENCRYPTED_FT_EDATA_SIZE ; + offset += ENCRYPTED_TURTLE_EDATA_SIZE ; uint32_t clear_item_offset = offset ; uint32_t authentication_tag_offset = offset + edata_size ; #ifdef SERVER_DEBUG - FTSERVER_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_FT_AUTHENTICATION_TAG_SIZE) << std::endl; + TURTLE_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE) << std::endl; #endif bool result ; - if(edata[2] == ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_POLY1305) - result = librs::crypto::AEAD_chacha20_poly1305(encryption_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; - else if(edata[2] == ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256) - result = librs::crypto::AEAD_chacha20_sha256 (encryption_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; + if(edata[2] == ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_POLY1305) + result = librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; + else if(edata[2] == ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256) + result = librs::crypto::AEAD_chacha20_sha256 (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; else return false ; #ifdef SERVER_DEBUG - FTSERVER_DEBUG() << " authen. result : " << result << std::endl; - FTSERVER_DEBUG() << " decrypted daya : " << RsUtil::BinToHex(&edata[clear_item_offset],std::min(50u,edata_size)) << "(...)" << std::endl; + TURTLE_DEBUG() << " authen. result : " << result << std::endl; + TURTLE_DEBUG() << " decrypted daya : " << RsUtil::BinToHex(&edata[clear_item_offset],std::min(50u,edata_size)) << "(...)" << std::endl; #endif if(!result) { - FTSERVER_ERROR() << "(EE) decryption/authentication went wrong." << std::endl; + TURTLE_ERROR() << "(EE) decryption/authentication went wrong." << std::endl; return false ; } - decrypted_item = dynamic_cast(deserialise(&edata[clear_item_offset],&edata_size)) ; + decrypted_data_size = edata_size ; + decrypted_data = (unsigned char*)rs_malloc(edata_size) ; - if(decrypted_item == NULL) + if(decrypted_data == NULL) + { + std::cerr << "Failed to allocate memory for decrypted data chunk of size " << edata_size << std::endl; return false ; + } + memcpy(decrypted_data,&edata[clear_item_offset],edata_size) ; return true ; } -#endif void p3turtle::getInfo( std::vector >& hashes_info, std::vector >& tunnels_info, diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index 2f7512753..b3c4debe4 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -332,12 +332,11 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config /// Send a data request into the correct tunnel for the given file hash void sendTurtleData(const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem *item) ; -#ifdef TODO /// Encrypts/decrypts an item, using a autenticated construction + chacha20, based on the given 32 bytes master key. + /// Input values are not touched (memory is not released). Memory ownership of outputs is left to the client. /// - static bool encryptItem(const RsItem *item, uint8_t* encryption_master_key, RsTurtleGenericDataItem *& encrypted_item) ; - static bool decryptItem(RsTurtleGenericDataItem *item,uint8_t *encryption_master_key,RsItem *& decrypted_item) ; -#endif + static bool encryptData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item); + static bool decryptItem(const RsTurtleGenericDataItem *item, uint8_t* encryption_master_key, unsigned char *& decrypted_data,uint32_t& decrypted_data_size); private: //--------------------------- Admin/Helper functions -------------------------// From 2255bda0070f01444b0a8a74ae677115e860da8c Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 24 Mar 2018 16:41:51 +0100 Subject: [PATCH 08/84] improved documentation of TurtleClientService class with doxygen type --- .../src/turtle/turtleclientservice.h | 92 ++++++++++++------- 1 file changed, 59 insertions(+), 33 deletions(-) diff --git a/libretroshare/src/turtle/turtleclientservice.h b/libretroshare/src/turtle/turtleclientservice.h index d9552c662..962e57cf0 100644 --- a/libretroshare/src/turtle/turtleclientservice.h +++ b/libretroshare/src/turtle/turtleclientservice.h @@ -42,51 +42,77 @@ class p3turtle ; class RsTurtleClientService { public: - // Handling of tunnel request for the given hash. Most of the time, it's a search in a predefined list. - // The output info_string is used by the turtle router to display info about tunnels it manages. It is - // not passed to the tunnel. - virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) { return false ; } + /*! + * \brief handleTunnelRequest + Handling of tunnel request for the given hash. To be derived by the service in order to tell the turtle router + whether the service handles this hash or not. Most of the time, it's a search in a predefined list. + + * \return true if the service + */ + virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) { return false ; } - // This method is called by the turtle router to send data that comes out of a turtle tunnel. - // The turtle router stays responsible for the memory management of data. Most of the time the - // data chunk is a serialized item to be de-serialized by the client service. - // - // Parameters: - // virtual_peer_id : name of the tunnel that sent the data - // data : memory chunk for the data - // size : size of data - // item->direction : direction of travel: - // RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a client - // RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a server - // - // Most of the time this parameter is not used by services, except when some info (such as chunk maps, chat items, etc) go - // both ways, and their nature cannot suffice to determine where they should be handled. - // - // By default (if not overloaded), the method will just free the data, as any subclass should do as well. - // Note: p3turtle stays owner of the item, so the client should not delete it! - // + + /*! + * \brief receiveTurtleData + * This method is called by the turtle router to send data that comes out of a turtle tunnel, and should + * be overloaded by the client service. + * The turtle router stays responsible for the memory management of data. Most of the time the + * data chunk is a serialized item to be de-serialized by the client service. + * + * Parameters: + * virtual_peer_id : name of the tunnel that sent the data + * data : memory chunk for the data + * size : size of data + * item->direction : direction of travel: + * RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a client + * RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a server + * + * Most of the time this parameter is not used by services, except when some info (such as chunk maps, chat items, etc) go + * both ways, and their nature cannot suffice to determine where they should be handled. + * + * By default (if not overloaded), the method will just free the data, as any subclass should do as well. + * Note: p3turtle stays owner of the item, so the client should not delete it! + */ virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/) { std::cerr << "!!!!!! Received Data from turtle router, but the client service is not handling it !!!!!!!!!!" << std::endl ; } - // Method for creating specific items of the client service. The - // method has a default behavior of not doing anything, since most client - // services might only use the generic item already provided by the turtle - // router: RsTurtleGenericDataItem - + /*! + * \brief serializer + * Method for creating specific items of the client service. The + * method has a default behavior of not doing anything, since most client + * services might only use the generic item already provided by the turtle + * router: RsTurtleGenericDataItem + * + * \return the client's serializer is returned + */ virtual RsServiceSerializer *serializer() { return NULL ; } - // These methods are called by the turtle router to add/remove virtual peers when tunnels are created/deleted - // + /*! + * \brief addVirtualPeer + * These methods are called by the turtle router to notify the client in order to add/remove virtual peers when tunnels are created/deleted + * These methods must be overloaded, because a service which does not care about tunel being openned or closed is not supposed to need tunnels. + * + * \param hash hash that the tunnel responds to + * \param virtual_peer_id virtual peer id provided by turtle to allow the client to send data into this tunnel. This peer is related to the tunnel itself + * rather than to its destination. As such, multiple peer ids may actually send data to the same computer because multiple tunnels + * arrive at the same location. + * \param dir dir indicates which side the cient will be talking to: CLIENT means that the client is the server. SERVER means that the client acts + * as a client (and therefore actually requested the tunnel). + */ virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) = 0 ; virtual void removeVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id) = 0 ; - // This function is mandatory. It should do two things: - // 1 - keep a pointer to the turtle router, so as to be able to send data (e.g. copy pt into a local variable) - // 2 - call pt->registerTunnelService(this), so that the TR knows that service and can send back information to it. - // + /*! + * \brief connectToTurtleRouter + * This function must be overloaded by the client. It should do two things: + * 1 - keep a pointer to the turtle router, so as to be able to send data (e.g. store pt into a local variable) + * 2 - call pt->registerTunnelService(this), so that the TR knows that service and can send back information to it. + * + * \param pt A pointer to the turtle router. + */ virtual void connectToTurtleRouter(p3turtle *pt) = 0 ; }; From 5566d90f328d383a9136d38d86b80e5b33974352 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 24 Mar 2018 16:42:16 +0100 Subject: [PATCH 09/84] finished tunnel handling and data send/recv in GxsNetTunnel --- libretroshare/src/gxs/rsgxsnettunnel.cc | 187 +++++++++++++++++++++--- libretroshare/src/gxs/rsgxsnettunnel.h | 14 +- 2 files changed, 181 insertions(+), 20 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index e9c389454..5e7fe557d 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -25,14 +25,77 @@ #include "util/rsdir.h" #include "retroshare/rspeers.h" +#include "serialiser/rstypeserializer.h" #include "rsgxsnettunnel.h" #define DEBUG_RSGXSNETTUNNEL 1 -#define NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } +#define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } +#define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL : " << __FUNCTION__ << " : " +#define GXS_NET_TUNNEL_ERROR() std::cerr << "(EE) GXS_NET_TUNNEL ERROR : " + RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") {} +//===========================================================================================================================================// +// Transport Items // +//===========================================================================================================================================// + +const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ; + +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; + +class RsGxsNetTunnelItem: public RsItem +{ +public: + explicit RsGxsNetTunnelItem(uint8_t item_subtype) : RsItem(RS_PKT_VERSION_SERVICE,RS_SERVICE_TYPE_GXS_NET_TUNNEL,item_subtype) + { + // no priority. All items are encapsulated into generic Turtle items anyway. + } + + virtual ~RsGxsNetTunnelItem() {} + virtual void clear() {} +}; + +class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem +{ +public: + RsGxsNetTunnelVirtualPeerItem() :RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER) {} + explicit RsGxsNetTunnelVirtualPeerItem(uint8_t subtype) :RsGxsNetTunnelItem(subtype) {} + + virtual ~RsGxsNetTunnelVirtualPeerItem() {} + + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,virtual_peer_id,"virtual_peer_id") ; + } + + RsPeerId virtual_peer_id ; +}; + +class RsGxsNetTunnelSerializer: public RsServiceSerializer +{ +public: + RsGxsNetTunnelSerializer() :RsServiceSerializer(RS_SERVICE_TYPE_GXS_NET_TUNNEL) {} + + virtual RsItem *create_item(uint16_t service,uint8_t item_subtype) const + { + if(service != RS_SERVICE_TYPE_GXS_NET_TUNNEL) + { + GXS_NET_TUNNEL_ERROR() << "received item with wrong service ID " << std::hex << service << std::dec << std::endl; + return NULL ; + } + + switch(item_subtype) + { + case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER: return new RsGxsNetTunnelVirtualPeerItem ; + default: + GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << item_subtype << std::dec << " is not handled!" << std::endl; + return NULL ; + } + } +}; + //===========================================================================================================================================// // Interface with rest of the software // //===========================================================================================================================================// @@ -57,8 +120,7 @@ bool RsGxsNetTunnelService::manage(const RsGxsGroupId& group_id) mHandledHashes[hash] = group_id ; #ifdef DEBUG_GXS_TUNNEL - std::cerr << "Starting distant chat to " << to_gxs_id << ", hash = " << hash << ", from " << from_gxs_id << std::endl; - std::cerr << "Asking turtle router to monitor tunnels for hash " << hash << std::endl; + GXS_NET_TUNNEL_DEBUG() << "Asking turtle router to monitor tunnels for hash " << hash << std::endl; #endif // Now ask the turtle router to manage a tunnel for that hash. @@ -78,7 +140,7 @@ bool RsGxsNetTunnelService::release(const RsGxsGroupId& group_id) if(it == mGroups.end()) { - std::cerr << "RsGxsNetTunnelService::release(): Weird. Cannot release client group " << group_id << " that is not known." << std::endl; + GXS_NET_TUNNEL_ERROR() << "RsGxsNetTunnelService::release(): Weird. Cannot release client group " << group_id << " that is not known." << std::endl; return false ; } @@ -110,7 +172,7 @@ bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualP if(it == mVirtualPeers.end()) { - std::cerr << "(EE) RsGxsNetTunnelService::sendData(): cannot find virtual peer " << virtual_peer << ". Data is dropped." << std::endl; + GXS_NET_TUNNEL_ERROR() << "cannot find virtual peer " << virtual_peer << ". Data is dropped." << std::endl; return false ; } @@ -118,7 +180,7 @@ bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualP if(it2 == mGroups.end()) { - std::cerr << "(EE) RsGxsNetTunnelService::sendData(): cannot find virtual peer " << virtual_peer << ". Data is dropped." << std::endl; + GXS_NET_TUNNEL_ERROR() << "cannot find virtual peer " << virtual_peer << ". Data is dropped." << std::endl; return false ; } @@ -126,13 +188,13 @@ bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualP if(it3 == it2->second.virtual_peers.end()) { - std::cerr << "(EE) RsGxsNetTunnelService::sendData(): cannot find turtle virtual peer " << it->second.second << ". Data is dropped." << std::endl; + std::cerr << "cannot find turtle virtual peer " << it->second.second << ". Data is dropped." << std::endl; return false ; } if(it3->second.vpid_status != RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE) { - std::cerr << "(EE) RsGxsNetTunnelService::sendData(): virtual peer " << it->second.second << " is not active. Data is dropped." << std::endl; + GXS_NET_TUNNEL_ERROR() << "virtual peer " << it->second.second << " is not active. Data is dropped." << std::endl; return false ; } @@ -140,12 +202,14 @@ bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualP RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; - NOT_IMPLEMENTED(); - // if(!p3Turtle::encryptItem(item,it3->second.encryption_master_key,encrypted_turtle_item)) - // { - // std::cerr << "(EE) RsGxsNetTunnelService::sendData(): cannot encrypt. Something's wrong. Data is dropped." << std::endl; - // return false ; - // } + uint32_t serialized_size = 0; // TODO + RsTemporaryMemory data(serialized_size) ; + + if(!p3turtle::encryptData(data,serialized_size,it3->second.encryption_master_key,encrypted_turtle_item)) + { + GXS_NET_TUNNEL_ERROR() << "cannot encrypt. Something's wrong. Data is dropped." << std::endl; + return false ; + } mTurtle->sendTurtleData(it->second.second,encrypted_turtle_item) ; @@ -155,7 +219,7 @@ bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualP bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId&, std::list& peers) { // returns the virtual peers for this group - NOT_IMPLEMENTED(); + GXS_NET_TUNNEL_NOT_IMPLEMENTED(); return false ; } @@ -227,7 +291,7 @@ void RsGxsNetTunnelService::connectToTurtleRouter(p3turtle *tr) bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) { - NOT_IMPLEMENTED(); + GXS_NET_TUNNEL_NOT_IMPLEMENTED(); // at this point we need to talk to the client services // There's 2 ways to do that: @@ -236,10 +300,70 @@ bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsP return true ; } + void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) { - NOT_IMPLEMENTED(); + GXS_NET_TUNNEL_NOT_IMPLEMENTED(); + + if(item->PacketSubType() != RS_TURTLE_SUBTYPE_GENERIC_DATA) + { + GXS_NET_TUNNEL_ERROR() << "item with type " << std::hex << item->PacketSubType() << std::dec << " received by GxsNetTunnel, but is not handled!" << std::endl; + return; + } + + // (cyril) this is a bit complicated. We should store pointers to the encryption keys in another structure and access it directly. + + auto it = mHandledHashes.find(hash) ; + + if(it == mHandledHashes.end()) + { + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for hash " << hash << " but this hash is unknown!" << std::endl; + return; + } + + RsGxsGroupId group_id = it->second; + + auto it2 = mGroups.find(group_id) ; + + if(it2 == mGroups.end()) + { + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for hash " << hash << " and group " << group_id << " but this group id is unknown!" << std::endl; + return; + } + + RsGxsNetTunnelGroupInfo& g_info(it2->second) ; + + g_info.last_contact = time(NULL) ; + + auto it3 = g_info.virtual_peers.find(virtual_peer_id) ; + + if(it3 == g_info.virtual_peers.end()) + { + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for hash " << hash << ", group " << group_id << " but the virtual peer id is missing!" << std::endl; + return; + } + + RsGxsNetTunnelVirtualPeerInfo& vp_info(it3->second) ; + + unsigned char *data = NULL ; + uint32_t data_size = 0 ; + + if(!p3turtle::decryptItem(static_cast(item),vp_info.encryption_master_key,data,data_size)) + { + GXS_NET_TUNNEL_ERROR() << "Cannot decrypt data!" << std::endl; + + if(data) + free(data) ; + + return ; + } + + RsGxsNetTunnelItem *decrypted_item = dynamic_cast(RsGxsNetTunnelSerializer().deserialise(data,&data_size)) ; + free(data); + + handleIncoming(decrypted_item); } + void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid,RsTurtleGenericTunnelItem::Direction dir) { auto it = mHandledHashes.find(hash) ; @@ -258,11 +382,25 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur RsGxsNetTunnelVirtualPeerInfo& vpinfo( ginfo.virtual_peers[vpid] ) ; vpinfo.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK ; - vpinfo.net_service_virtual_peer.clear(); vpinfo.side = dir ; vpinfo.last_contact = time(NULL) ; generateEncryptionKey(group_id,vpid,vpinfo.encryption_master_key ); + + // If we're a server, we need to send our own virtual peer id to the client + + if(dir == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) + { + vpinfo.net_service_virtual_peer = makeServerVirtualPeerIdForGroup(group_id); + + RsGxsNetTunnelVirtualPeerItem *pitem = new RsGxsNetTunnelVirtualPeerItem ; + + pitem->virtual_peer_id = vpinfo.net_service_virtual_peer ; + + vpinfo.outgoing_items.push_back(pitem) ; + } + else + vpinfo.net_service_virtual_peer.clear(); } void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid) @@ -309,6 +447,19 @@ void RsGxsNetTunnelService::generateEncryptionKey(const RsGxsGroupId& group_id,c // Service parts // //===========================================================================================================================================// +void RsGxsNetTunnelService::data_tick() +{ + GXS_NET_TUNNEL_DEBUG() << std::endl; + + // cleanup + + autowash(); +} + +void RsGxsNetTunnelService::autowash() +{ +} + #ifdef TODO void RsGxsNetTunnelService::handleIncomingItem(const RsGxsTunnelId& tunnel_id,RsGxsTunnelItem *item) { diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 2f6cb0eae..16f00de4b 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -66,6 +66,8 @@ typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; +class RsGxsNetTunnelItem ; + struct RsGxsNetTunnelVirtualPeerInfo { enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. @@ -77,7 +79,7 @@ struct RsGxsNetTunnelVirtualPeerInfo uint8_t vpid_status ; // status of the peer uint8_t side ; // client/server - uint8_t encryption_master_key[16] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) + uint8_t encryption_master_key[32] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) time_t last_contact ; // last time some data was sent/recvd RsGxsNetTunnelVirtualPeerId net_service_virtual_peer ; // anonymised peer that is used to communicate with client services @@ -103,7 +105,7 @@ struct RsGxsNetTunnelGroupInfo std::map virtual_peers ; }; -class RsGxsNetTunnelService: public RsTurtleClientService +class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread { public: RsGxsNetTunnelService() ; @@ -156,6 +158,11 @@ public: // - method to encrypt/decrypt data and send/receive to/from turtle. virtual void connectToTurtleRouter(p3turtle *tr) ; + + // Overloaded from RsTickingThread + + void data_tick() ; + protected: // interaction with turtle router @@ -166,6 +173,9 @@ protected: p3turtle *mTurtle ; private: + void autowash() ; + void handleIncoming(RsGxsNetTunnelItem *item) ; + static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 16 ; static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE = 32 ; From b488760d7d8c858c970712cbc37a8a46c0453f80 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 26 Mar 2018 23:19:29 +0200 Subject: [PATCH 10/84] fixed data incoming/outgoing in GxsNetTunnel --- libretroshare/src/gxs/rsgxsnettunnel.cc | 144 ++++++++++++++++++------ libretroshare/src/gxs/rsgxsnettunnel.h | 3 +- 2 files changed, 110 insertions(+), 37 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 5e7fe557d..36ac81f47 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -37,6 +37,19 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") {} +//===========================================================================================================================================// +// Internal structures // +//===========================================================================================================================================// + +RsGxsNetTunnelVirtualPeerInfo::~RsGxsNetTunnelVirtualPeerInfo() +{ + for(auto it(outgoing_items.begin());it!=outgoing_items.end();++it) + delete *it ; + + for(auto it(incoming_data.begin());it!=incoming_data.end();++it) + delete *it ; +} + //===========================================================================================================================================// // Transport Items // //===========================================================================================================================================// @@ -267,7 +280,7 @@ void RsGxsNetTunnelService::dump() const std::cerr << " turtle:" << it2->first << " status: " << vpid_status_str[it2->second.vpid_status] << " s: " << (int)it2->second.side << " last seen " << time(NULL)-it2->second.last_contact << " ekey: " << RsUtil::BinToHex(it2->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE) - << " pending (" << it2->second.incoming_items.size() << "," << it2->second.outgoing_items.size() << ")" << std::endl; + << " pending (" << it2->second.incoming_data.size() << "," << it2->second.outgoing_items.size() << ")" << std::endl; } std::cerr << "Virtual peers: " << std::endl; @@ -303,7 +316,9 @@ bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsP void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) { - GXS_NET_TUNNEL_NOT_IMPLEMENTED(); +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " received turtle data for vpid " << virtual_peer_id << " for hash " << hash << " in direction " << (int)direction << std::endl; +#endif if(item->PacketSubType() != RS_TURTLE_SUBTYPE_GENERIC_DATA) { @@ -358,10 +373,42 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co return ; } - RsGxsNetTunnelItem *decrypted_item = dynamic_cast(RsGxsNetTunnelSerializer().deserialise(data,&data_size)) ; - free(data); + // Now we might get 2 kinds of items: GxsNetTunnel items, to be handled here, and Gxs data items, to be handled by the client service - handleIncoming(decrypted_item); + RsItem *decrypted_item = RsGxsNetTunnelSerializer().deserialise(data,&data_size); + + RsGxsNetTunnelVirtualPeerItem *pid_item = dynamic_cast(decrypted_item) ; + + if(pid_item) + { + if(direction == RsTurtleGenericTunnelItem::DIRECTION_SERVER) + { +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " item is a virtual peer id item with vpid = "<< pid_item->virtual_peer_id << ". Setting virtual peer." << std::endl; +#endif + vp_info.net_service_virtual_peer = pid_item->virtual_peer_id; + vp_info.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; + } + else + GXS_NET_TUNNEL_ERROR() << "Cannot decrypt data!" << std::endl; + + free(data); + return ; + } + else + { +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " item is GXS data. Storing into incoming list." << std::endl; +#endif + // push the data into the service incoming data list + + RsTlvBinaryData *bind = new RsTlvBinaryData; + bind->tlvtype = 0; + bind->bin_len = data_size; + bind->bin_data = data; + + vp_info.incoming_data.push_back(bind) ; + } } void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid,RsTurtleGenericTunnelItem::Direction dir) @@ -374,6 +421,9 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur return ; } +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " adding virtual peer " << vpid << " for hash " << hash << " in direction " << dir << std::endl; +#endif const RsGxsGroupId group_id(it->second) ; RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; @@ -393,6 +443,9 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur { vpinfo.net_service_virtual_peer = makeServerVirtualPeerIdForGroup(group_id); +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " peer is server side: sending back virtual peer name " << vpinfo.net_service_virtual_peer << std::endl; +#endif RsGxsNetTunnelVirtualPeerItem *pitem = new RsGxsNetTunnelVirtualPeerItem ; pitem->virtual_peer_id = vpinfo.net_service_virtual_peer ; @@ -405,6 +458,9 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid) { +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " removing virtual peer " << vpid << " for hash " << hash << std::endl; +#endif auto it = mHandledHashes.find(hash) ; if(it == mHandledHashes.end()) @@ -420,7 +476,13 @@ void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const ginfo.virtual_peers.erase(vpid); if(ginfo.virtual_peers.empty()) + { ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED ; + +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " no more virtual peers for group " << group_id << ": setting status to TUNNELS_REQUESTED" << std::endl; +#endif + } } RsFileHash RsGxsNetTunnelService::calculateGroupHash(const RsGxsGroupId& group_id) const @@ -454,41 +516,51 @@ void RsGxsNetTunnelService::data_tick() // cleanup autowash(); + + static time_t last_dump = time(NULL); + time_t now = time(NULL); + + if(last_dump + 10 > now) + { + last_dump = now; + dump(); + } } void RsGxsNetTunnelService::autowash() { } -#ifdef TODO -void RsGxsNetTunnelService::handleIncomingItem(const RsGxsTunnelId& tunnel_id,RsGxsTunnelItem *item) -{ - if(item == NULL) - return ; - - // We have 3 things to do: - // - // 1 - if it's a data item, send an ACK - // 2 - if it's an ack item, mark the item as properly received, and remove it from the queue - // 3 - if it's a status item, act accordingly. - - switch(item->PacketSubType()) - { - - case RS_PKT_SUBTYPE_GXS_TUNNEL_DATA: handleRecvTunnelDataItem(tunnel_id,dynamic_cast(item)) ; - break ; - - case RS_PKT_SUBTYPE_GXS_TUNNEL_DATA_ACK: handleRecvTunnelDataAckItem(tunnel_id,dynamic_cast(item)) ; - break ; - - case RS_PKT_SUBTYPE_GXS_TUNNEL_STATUS: handleRecvStatusItem(tunnel_id,dynamic_cast(item)) ; - break ; - - default: - std::cerr << "(EE) impossible situation. DH items should be handled at the service level" << std::endl; - } - - delete item ; -} -#endif +// void RsGxsNetTunnelService::handleIncoming(const RsGxsTunnelId& tunnel_id,RsGxsTunnelItem *item) +// { +// #ifdef DEBUG_RSGXSNETTUNNEL +// GXS_NET_TUNNEL_DEBUG() << " received turtle data for vpid " << virtual_peer_id << " for hash " << hash << " in direction " << dir << std::endl; +// #endif +// if(item == NULL) +// return ; +// +// // We have 3 things to do: +// // +// // 1 - if it's a data item, send an ACK +// // 2 - if it's an ack item, mark the item as properly received, and remove it from the queue +// // 3 - if it's a status item, act accordingly. +// +// switch(item->PacketSubType()) +// { +// +// case RS_PKT_SUBTYPE_GXS_TUNNEL_DATA: handleRecvTunnelDataItem(tunnel_id,dynamic_cast(item)) ; +// break ; +// +// case RS_PKT_SUBTYPE_GXS_TUNNEL_DATA_ACK: handleRecvTunnelDataAckItem(tunnel_id,dynamic_cast(item)) ; +// break ; +// +// case RS_PKT_SUBTYPE_GXS_TUNNEL_STATUS: handleRecvStatusItem(tunnel_id,dynamic_cast(item)) ; +// break ; +// +// default: +// std::cerr << "(EE) impossible situation. DH items should be handled at the service level" << std::endl; +// } +// +// delete item ; +// } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 16f00de4b..798b49f1e 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -76,6 +76,7 @@ struct RsGxsNetTunnelVirtualPeerInfo }; RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN) { memset(encryption_master_key,0,16) ; } + ~RsGxsNetTunnelVirtualPeerInfo() ; uint8_t vpid_status ; // status of the peer uint8_t side ; // client/server @@ -85,7 +86,7 @@ struct RsGxsNetTunnelVirtualPeerInfo RsGxsNetTunnelVirtualPeerId net_service_virtual_peer ; // anonymised peer that is used to communicate with client services RsGxsGroupId group_id ; // group id - std::list incoming_items ; + std::list incoming_data ; std::list outgoing_items ; }; From 73b04f3109b8df3769c5c8199ea99b11c3213144 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 29 Mar 2018 10:54:58 +0200 Subject: [PATCH 11/84] finished implementation of GxsNetTunnel service --- libretroshare/src/gxs/rsgxsnetservice.cc | 4 +- libretroshare/src/gxs/rsgxsnetservice.h | 3 +- libretroshare/src/gxs/rsgxsnettunnel.cc | 180 ++++++++++++----------- libretroshare/src/gxs/rsgxsnettunnel.h | 69 ++++++--- 4 files changed, 148 insertions(+), 108 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 5fc4427ad..914b6ce4e 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -359,7 +359,7 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, const RsServiceInfo serviceInfo, RsGixsReputation* reputations, RsGcxs* circles, RsGixs *gixs, - PgpAuxUtils *pgpUtils, bool grpAutoSync, bool msgAutoSync, uint32_t default_store_period, uint32_t default_sync_period) + PgpAuxUtils *pgpUtils, bool grpAutoSync, bool msgAutoSync, bool distSync, uint32_t default_store_period, uint32_t default_sync_period) : p3ThreadedService(), p3Config(), mTransactionN(0), mObserver(nxsObs), mDataStore(gds), mServType(servType), mTransactionTimeOut(TRANSAC_TIMEOUT), @@ -368,7 +368,7 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, mLastCleanRejectedMessages(0), mSYNC_PERIOD(SYNC_PERIOD), mCircles(circles), mGixs(gixs), mReputations(reputations), mPgpUtils(pgpUtils), - mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync), + mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync),mAllowDistSync(distSync), mServiceInfo(serviceInfo), mDefaultMsgStorePeriod(default_store_period), mDefaultMsgSyncPeriod(default_sync_period) { diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 00ba07953..49d44dab2 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -90,7 +90,7 @@ public: const RsServiceInfo serviceInfo, RsGixsReputation* reputations = NULL, RsGcxs* circles = NULL, RsGixs *gixs=NULL, PgpAuxUtils *pgpUtils = NULL, - bool grpAutoSync = true, bool msgAutoSync = true, + bool grpAutoSync = true, bool msgAutoSync = true,bool distSync=false, uint32_t default_store_period = RS_GXS_DEFAULT_MSG_STORE_PERIOD, uint32_t default_sync_period = RS_GXS_DEFAULT_MSG_REQ_PERIOD); @@ -543,6 +543,7 @@ private: PgpAuxUtils *mPgpUtils; bool mGrpAutoSync; bool mAllowMsgSync; + bool mAllowDistSync; // need to be verfied std::vector mPendingResp; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 36ac81f47..6d763a092 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -57,6 +57,7 @@ RsGxsNetTunnelVirtualPeerInfo::~RsGxsNetTunnelVirtualPeerInfo() const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ; class RsGxsNetTunnelItem: public RsItem { @@ -74,7 +75,6 @@ class RsGxsNetTunnelVirtualPeerItem: public RsGxsNetTunnelItem { public: RsGxsNetTunnelVirtualPeerItem() :RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER) {} - explicit RsGxsNetTunnelVirtualPeerItem(uint8_t subtype) :RsGxsNetTunnelItem(subtype) {} virtual ~RsGxsNetTunnelVirtualPeerItem() {} @@ -86,6 +86,15 @@ public: RsPeerId virtual_peer_id ; }; +class RsGxsNetTunnelKeepAliveItem: public RsGxsNetTunnelItem +{ +public: + RsGxsNetTunnelKeepAliveItem() :RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE) {} + + virtual ~RsGxsNetTunnelKeepAliveItem() {} + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) {} +}; + class RsGxsNetTunnelSerializer: public RsServiceSerializer { public: @@ -102,6 +111,7 @@ public: switch(item_subtype) { case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER: return new RsGxsNetTunnelVirtualPeerItem ; + case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ; default: GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << item_subtype << std::dec << " is not handled!" << std::endl; return NULL ; @@ -113,71 +123,19 @@ public: // Interface with rest of the software // //===========================================================================================================================================// -bool RsGxsNetTunnelService::manage(const RsGxsGroupId& group_id) -{ - RsFileHash hash = calculateGroupHash(group_id) ; - - RsStackMutex stack(mGxsNetTunnelMtx); /********** STACK LOCKED MTX ******/ - - RsGxsNetTunnelGroupInfo& info(mGroups[group_id]) ; - - time_t now = time(NULL) ; - - if(info.group_status == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE) - return true; - - info.hash = hash ; - info.last_contact = now ; - info.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; - - mHandledHashes[hash] = group_id ; - -#ifdef DEBUG_GXS_TUNNEL - GXS_NET_TUNNEL_DEBUG() << "Asking turtle router to monitor tunnels for hash " << hash << std::endl; -#endif - - // Now ask the turtle router to manage a tunnel for that hash. - - mTurtle->monitorTunnels(hash,this,false) ; - - return true; -} - -bool RsGxsNetTunnelService::release(const RsGxsGroupId& group_id) -{ - RsStackMutex stack(mGxsNetTunnelMtx); /********** STACK LOCKED MTX ******/ - - // Here we need to clean the stuff that was created by this group id. - - auto it = mGroups.find(group_id) ; - - if(it == mGroups.end()) - { - GXS_NET_TUNNEL_ERROR() << "RsGxsNetTunnelService::release(): Weird. Cannot release client group " << group_id << " that is not known." << std::endl; - return false ; - } - - mGroups.erase(it) ; - - RsFileHash hash = calculateGroupHash(group_id) ; - - mHandledHashes.erase(hash) ; - return true ; -} - -class ItemAutoDelete +class DataAutoDelete { public: - ItemAutoDelete(RsItem *& item) : mItem(item) {} - ~ItemAutoDelete() { delete mItem; mItem=NULL ; } - RsItem *& mItem; + DataAutoDelete(unsigned char *& data) : mData(data) {} + ~DataAutoDelete() { free(mData); mData=NULL ; } + unsigned char *& mData; }; -bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualPeerId& virtual_peer) +bool RsGxsNetTunnelService::sendData(unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) { // The item is serialized and encrypted using chacha20+SHA256, using the generic turtle encryption, and then sent to the turtle router. - ItemAutoDelete iad(item) ; // This ensures the item is deleted whatsoever when leaving + DataAutoDelete iad(data) ; // This ensures the item is deleted whatsoever when leaving // 1 - find the virtual peer and the proper master key to encrypt with, and check that all the info is known @@ -215,10 +173,7 @@ bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualP RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; - uint32_t serialized_size = 0; // TODO - RsTemporaryMemory data(serialized_size) ; - - if(!p3turtle::encryptData(data,serialized_size,it3->second.encryption_master_key,encrypted_turtle_item)) + if(!p3turtle::encryptData(data,data_len,it3->second.encryption_master_key,encrypted_turtle_item)) { GXS_NET_TUNNEL_ERROR() << "cannot encrypt. Something's wrong. Data is dropped." << std::endl; return false ; @@ -229,11 +184,63 @@ bool RsGxsNetTunnelService::sendItem(RsItem *& item,const RsGxsNetTunnelVirtualP return true ; } -bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId&, std::list& peers) +bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId& group_id, std::list& peers) { - // returns the virtual peers for this group - GXS_NET_TUNNEL_NOT_IMPLEMENTED(); - return false ; + // This function has two effects: + // - return the virtual peers for this group + // - passively set the group as "managed", so that it we answer tunnel requests. + + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + // update the hash entry if needed + + RsFileHash hash = calculateGroupHash(group_id); + mHandledHashes[hash] = group_id ; + + // Create the group entry, if needed, with passive mode. + + RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; + + ginfo.hash = hash ; + + for(auto it2 = ginfo.virtual_peers.begin();it2 != ginfo.virtual_peers.end();++it2) + if(it2->second.vpid_status == RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE) + peers.push_back(it2->second.net_service_virtual_peer) ; + +#ifdef DEBUG_GXS_TUNNEL + GXS_NET_TUNNEL_DEBUG() << "returning " << peers.size() << " peers." << std::endl; +#endif + + return true ; +} + +bool RsGxsNetTunnelService::requestPeers(const RsGxsGroupId& group_id) +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + // Now ask the turtle router to manage a tunnel for that hash. + + RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; + + ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE; + + // we dont set the group policy here. It will only be set if no peers, or too few peers are available. + + return true; +} + +bool RsGxsNetTunnelService::releasePeers(const RsGxsGroupId& group_id) +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + // Ask turtle router to stop requesting tunnels for that hash. + + RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; + + ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE; + mTurtle->stopMonitoringTunnels(ginfo.hash) ; + + return true; } RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::makeServerVirtualPeerIdForGroup(const RsGxsGroupId& group_id) const @@ -304,14 +311,9 @@ void RsGxsNetTunnelService::connectToTurtleRouter(p3turtle *tr) bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) { - GXS_NET_TUNNEL_NOT_IMPLEMENTED(); + // We simply check for wether a managed group has a hash that corresponds to the given hash. - // at this point we need to talk to the client services - // There's 2 ways to do that: - // 1 - client services "register" and we ask them one by one. - // 2 - client service derives from RsGxsNetTunnelService and the client is interrogated using an overloaded virtual method - - return true ; + return mHandledHashes.find(hash) != mHandledHashes.end(); } void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) @@ -437,23 +439,27 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur generateEncryptionKey(group_id,vpid,vpinfo.encryption_master_key ); - // If we're a server, we need to send our own virtual peer id to the client + // We need to send our own virtual peer id to the other end of the tunnel - if(dir == RsTurtleGenericTunnelItem::DIRECTION_CLIENT) - { - vpinfo.net_service_virtual_peer = makeServerVirtualPeerIdForGroup(group_id); + vpinfo.net_service_virtual_peer = makeServerVirtualPeerIdForGroup(group_id); #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " peer is server side: sending back virtual peer name " << vpinfo.net_service_virtual_peer << std::endl; + GXS_NET_TUNNEL_DEBUG() << " sending back virtual peer name " << vpinfo.net_service_virtual_peer << " to end of tunnel" << std::endl; #endif - RsGxsNetTunnelVirtualPeerItem *pitem = new RsGxsNetTunnelVirtualPeerItem ; + RsGxsNetTunnelVirtualPeerItem pitem ; + pitem.virtual_peer_id = vpinfo.net_service_virtual_peer ; - pitem->virtual_peer_id = vpinfo.net_service_virtual_peer ; + RsTemporaryMemory tmpmem( RsGxsNetTunnelSerializer().size(&pitem) ) ; + uint32_t len = tmpmem.size(); - vpinfo.outgoing_items.push_back(pitem) ; - } + RsGxsNetTunnelSerializer().serialise(&pitem,tmpmem,&len); + + RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; + + if(p3turtle::encryptData(tmpmem,len,vpinfo.encryption_master_key,encrypted_turtle_item)) + mTurtle->sendTurtleData(vpid,encrypted_turtle_item) ; else - vpinfo.net_service_virtual_peer.clear(); + GXS_NET_TUNNEL_ERROR() << "cannot encrypt. Something's wrong. Data is dropped." << std::endl; } void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid) @@ -531,6 +537,12 @@ void RsGxsNetTunnelService::autowash() { } +// info.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; +// +// mTurtle->monitorTunnels(hash,this,false) ; +// info.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; + + // void RsGxsNetTunnelService::handleIncoming(const RsGxsTunnelId& tunnel_id,RsGxsTunnelItem *item) // { // #ifdef DEBUG_RSGXSNETTUNNEL diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 798b49f1e..901aba61d 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -34,6 +34,10 @@ * * It is the responsibility of RsGxsNetService to activate/desactivate tunnels for each particular group, depending on wether the group * is already available at friends or not. + * + * Tunnel management is done at the level of groups rather than services, because we would like to keep the possibility to not + * request tunnels for some groups which do not need it, and only request tunnels for specific groups that cannot be provided + * by direct connections. */ // Protocol: @@ -61,8 +65,23 @@ // Client <------------------- GXS Data ------------------> Server | // - // Notes: -// * tunnels are only used one-way. If a distant peers wants to sync the same group, he'll have to open his own tunnel, with a different ID. -// * each group will produced multiple tunnels, but each tunnel with have exactly one virtual peer ID +// * tunnels are established symetrically. If a distant peers wants to sync the same group, they'll have to open a single tunnel, with a different ID. +// Groups therefore have two states: +// - managed : the group can be used to answer tunnel requests. If server tunnels are established, the group will be synced with these peers +// - tunneled: the group will actively request tunnels. If tunnels are established both ways, the same virtual peers will be used so the tunnels are "merged". +// * In practice, that means one of the two tunnels will not be used and therefore die. +// * If a tunneled group already has enough virtual peers, it will not request for tunnels itself. +// +// Group policy | Request tunnels | SyncWithPeers | Item receipt +// --------------------+-------------------+-----------------------+---------------- +// Passive | no | If peers present | If peers present +// Active | yes, if no peers | If peers present | If peers present +// | | | +// +// * when a service has the DistSync flag set, groups to sync are communicated passively to the GxsNetTunnel service when requesting distant peers. +// However, a call should be made to set a particular group policy to "ACTIVE" for group that do not have peers and need some. +// +// * services also need to retrieve GXS data items that come out of tunnels. These will be available as (data,len) type, since they are not de-serialized. typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; @@ -75,7 +94,7 @@ struct RsGxsNetTunnelVirtualPeerInfo RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. }; - RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN) { memset(encryption_master_key,0,16) ; } + RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN) { memset(encryption_master_key,0,32) ; } ~RsGxsNetTunnelVirtualPeerInfo() ; uint8_t vpid_status ; // status of the peer @@ -92,14 +111,23 @@ struct RsGxsNetTunnelVirtualPeerInfo struct RsGxsNetTunnelGroupInfo { - enum { RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status - RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED = 0x01, // waiting for turtle to send some virtual peers. - RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x02 // some virtual peers are available + enum GroupStatus { + RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status + RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE = 0x01, // no virtual peers requested, just waiting + RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED = 0x02, // virtual peers requested, and waiting for turtle to answer + RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x03 // some virtual peers are available. Data can be read/written }; - RsGxsNetTunnelGroupInfo() : group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN),last_contact(0) {} + enum GroupPolicy { + RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN = 0x00, // nothing has been set + RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE = 0x01, // group is available for server side tunnels, but does not explicitely request tunnels + RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group explicitely request tunnels, if none available + }; - uint8_t group_status ; + RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {} + + GroupPolicy group_policy ; + GroupStatus group_status ; time_t last_contact ; TurtleFileHash hash ; @@ -112,27 +140,32 @@ public: RsGxsNetTunnelService() ; /*! - * \brief start managing tunnels for this group - * @param group_id group for which tunnels should be requested + * \brief Manage tunnels for this group + * @param group_id group for which tunnels should be released */ - bool manage(const RsGxsGroupId& group_id) ; + bool requestPeers(const RsGxsGroupId&group_id) ; /*! * \brief Stop managing tunnels for this group * @param group_id group for which tunnels should be released */ - bool release(const RsGxsGroupId&group_id) ; - + bool releasePeers(const RsGxsGroupId&group_id) ; /*! - * \brief sendItem + * \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and + * alive. This function also "registers" the group which allows to handle tunnel requests in the server side. + */ + bool getVirtualPeers(const RsGxsGroupId& group_id, std::list& peers) ; // returns the virtual peers for this group + + /*! + * \brief sendData * send data to this virtual peer, and takes memory ownership (deletes the item) * \param item item to send * \param virtual_peer destination virtual peer * \return * true if succeeded. */ - bool sendItem(RsItem *& item, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + bool sendData(unsigned char *& data, uint32_t data_len, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! * \brief receivedItem @@ -142,12 +175,6 @@ public: */ RsItem *receivedItem(const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; - /*! - * \brief Get the list of active virtual peers for a given group. This implies that the tunnel is up and - * alive. - */ - bool getVirtualPeers(const RsGxsGroupId&, std::list& peers) ; // returns the virtual peers for this group - /*! * \brief dumps all information about monitored groups. */ From f0f69b8dd95d411aab8a7e7b892f3d36d5fe8c90 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 29 Mar 2018 16:26:36 +0200 Subject: [PATCH 12/84] shared virtual peers across services --- libretroshare/src/gxs/rsgxsnetservice.cc | 14 ++ libretroshare/src/gxs/rsgxsnettunnel.cc | 273 +++++++++++------------ libretroshare/src/gxs/rsgxsnettunnel.h | 37 ++- 3 files changed, 170 insertions(+), 154 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 914b6ce4e..7962f9bf3 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -565,6 +565,20 @@ void RsGxsNetService::syncWithPeers() std::set peers; mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers); + +#ifdef TODO + if(mAllowDistSync) + { + // Grab all online virtual peers of distant tunnels for the current service. + + std::list vpids ; + mGxsNetTunnel->getVirtualPeers(mServType,vpids); + + for(auto it(vpids.begin());it!=vpids.end();++it) + peers.push_back(RsPeerId(*it)) ; + } +#endif + if (peers.empty()) { // nothing to do return; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 6d763a092..d5628eed3 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -43,11 +43,9 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") RsGxsNetTunnelVirtualPeerInfo::~RsGxsNetTunnelVirtualPeerInfo() { - for(auto it(outgoing_items.begin());it!=outgoing_items.end();++it) - delete *it ; - - for(auto it(incoming_data.begin());it!=incoming_data.end();++it) - delete *it ; + for(auto it(providing_set.begin());it!=providing_set.end();++it) + for(auto it2(it->second.incoming_data.begin());it2!=it->second.incoming_data.end();++it2) + delete *it2 ; } //===========================================================================================================================================// @@ -131,8 +129,21 @@ public: unsigned char *& mData; }; +RsGxsNetTunnelService::~RsGxsNetTunnelService() +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + // This is needed because we need to clear these structure in a mutex protected environment + // Also this calls the destructor of the objects which calls the freeing of memory e.g. allocated in the incoming data list. + + mGroups.clear(); + mHandledHashes.clear(); + mVirtualPeers.clear(); +} + bool RsGxsNetTunnelService::sendData(unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) { + RS_STACK_MUTEX(mGxsNetTunnelMtx); // The item is serialized and encrypted using chacha20+SHA256, using the generic turtle encryption, and then sent to the turtle router. DataAutoDelete iad(data) ; // This ensures the item is deleted whatsoever when leaving @@ -147,25 +158,9 @@ bool RsGxsNetTunnelService::sendData(unsigned char *& data,uint32_t data_len,con return false ; } - auto it2 = mGroups.find(it->second.first); - - if(it2 == mGroups.end()) + if(it->second.vpid_status != RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE) { - GXS_NET_TUNNEL_ERROR() << "cannot find virtual peer " << virtual_peer << ". Data is dropped." << std::endl; - return false ; - } - - auto it3 = it2->second.virtual_peers.find(it->second.second); - - if(it3 == it2->second.virtual_peers.end()) - { - std::cerr << "cannot find turtle virtual peer " << it->second.second << ". Data is dropped." << std::endl; - return false ; - } - - if(it3->second.vpid_status != RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE) - { - GXS_NET_TUNNEL_ERROR() << "virtual peer " << it->second.second << " is not active. Data is dropped." << std::endl; + GXS_NET_TUNNEL_ERROR() << "virtual peer " << virtual_peer << " is not active. Data is dropped." << std::endl; return false ; } @@ -173,18 +168,18 @@ bool RsGxsNetTunnelService::sendData(unsigned char *& data,uint32_t data_len,con RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; - if(!p3turtle::encryptData(data,data_len,it3->second.encryption_master_key,encrypted_turtle_item)) + if(!p3turtle::encryptData(data,data_len,it->second.encryption_master_key,encrypted_turtle_item)) { GXS_NET_TUNNEL_ERROR() << "cannot encrypt. Something's wrong. Data is dropped." << std::endl; return false ; } - mTurtle->sendTurtleData(it->second.second,encrypted_turtle_item) ; + mTurtle->sendTurtleData(it->second.turtle_virtual_peer_id,encrypted_turtle_item) ; return true ; } -bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId& group_id, std::list& peers) +bool RsGxsNetTunnelService::getVirtualPeers(uint16_t service_id, std::list& peers) { // This function has two effects: // - return the virtual peers for this group @@ -194,21 +189,12 @@ bool RsGxsNetTunnelService::getVirtualPeers(const RsGxsGroupId& group_id, std::l // update the hash entry if needed - RsFileHash hash = calculateGroupHash(group_id); - mHandledHashes[hash] = group_id ; - - // Create the group entry, if needed, with passive mode. - - RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; - - ginfo.hash = hash ; - - for(auto it2 = ginfo.virtual_peers.begin();it2 != ginfo.virtual_peers.end();++it2) - if(it2->second.vpid_status == RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE) - peers.push_back(it2->second.net_service_virtual_peer) ; + for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) + if(it->second.providing_set.find(service_id) != it->second.providing_set.end()) + peers.push_back(it->first) ; #ifdef DEBUG_GXS_TUNNEL - GXS_NET_TUNNEL_DEBUG() << "returning " << peers.size() << " peers." << std::endl; + GXS_NET_TUNNEL_DEBUG() << " service " << std::hex << service_id << std::dec << " returning " << peers.size() << " peers." << std::endl; #endif return true ; @@ -243,19 +229,19 @@ bool RsGxsNetTunnelService::releasePeers(const RsGxsGroupId& group_id) return true; } -RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::makeServerVirtualPeerIdForGroup(const RsGxsGroupId& group_id) const +RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId() const { assert(RsPeerId::SIZE_IN_BYTES <= Sha1CheckSum::SIZE_IN_BYTES) ; - // We compute sha1( SSL_id | group_id | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId + // We compute sha1( SSL_id | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId RsPeerId ssl_id = rsPeers->getOwnId() ; - unsigned char mem[RsPeerId::SIZE_IN_BYTES + RsGxsGroupId::SIZE_IN_BYTES + RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE]; + unsigned char mem[RsPeerId::SIZE_IN_BYTES /*+ RsGxsGroupId::SIZE_IN_BYTES */ + RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE]; memcpy(mem ,ssl_id.toByteArray() ,RsPeerId::SIZE_IN_BYTES) ; - memcpy(mem+RsPeerId::SIZE_IN_BYTES ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; - memcpy(mem+RsPeerId::SIZE_IN_BYTES+RsGxsGroupId::SIZE_IN_BYTES,mRandomBias ,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; + //memcpy(mem+RsPeerId::SIZE_IN_BYTES ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; + memcpy(mem+RsPeerId::SIZE_IN_BYTES /*+RsGxsGroupId::SIZE_IN_BYTES*/,mRandomBias ,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsPeerId::SIZE_IN_BYTES+RsGxsGroupId::SIZE_IN_BYTES+RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE).toByteArray()); } @@ -264,12 +250,18 @@ void RsGxsNetTunnelService::dump() const { RS_STACK_MUTEX(mGxsNetTunnelMtx); - static std::string group_status_str[3] = { + static std::string group_status_str[4] = { std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN ]"), + std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE ]"), std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED]"), std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE ]") }; + static std::string group_policy_str[3] = { + std::string("[RS_GXS_NET_TUNNEL_POLICY_UNKNOWN]"), + std::string("[RS_GXS_NET_TUNNEL_POLICY_PASSIVE]"), + std::string("[RS_GXS_NET_TUNNEL_POLICY_ACTIVE ]"), + }; static std::string vpid_status_str[3] = { std::string("[RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN ]"), std::string("[RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK ]"), @@ -281,18 +273,23 @@ void RsGxsNetTunnelService::dump() const for(auto it(mGroups.begin());it!=mGroups.end();++it) { - std::cerr << " " << it->first << " hash: " << it->second.hash << " status: " << group_status_str[it->second.group_status] << "] Last contact: " << time(NULL) - it->second.last_contact << " secs ago" << std::endl; - + std::cerr << " " << it->first << " hash: " << it->second.hash << " policy: " << group_policy_str[it->second.group_policy] << " status: " << group_status_str[it->second.group_status] << "] Last contact: " << time(NULL) - it->second.last_contact << " secs ago" << std::endl; + std::cerr << " virtual peers:" << std::endl; for(auto it2(it->second.virtual_peers.begin());it2!=it->second.virtual_peers.end();++it2) - std::cerr << " turtle:" << it2->first << " status: " << vpid_status_str[it2->second.vpid_status] << " s: " - << (int)it2->second.side << " last seen " << time(NULL)-it2->second.last_contact - << " ekey: " << RsUtil::BinToHex(it2->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE) - << " pending (" << it2->second.incoming_data.size() << "," << it2->second.outgoing_items.size() << ")" << std::endl; + std::cerr << " " << *it2 << std::endl; } std::cerr << "Virtual peers: " << std::endl; for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) - std::cerr << " GXS Peer:" << it->first << " group_id: " << it->second.first << " Turtle:" << it->second.second << std::endl; + { + std::cerr << " GXS Peer:" << it->first << " Turtle:" << it->second.turtle_virtual_peer_id + << " status: " << vpid_status_str[it->second.vpid_status] << " s: " + << (int)it->second.side << " last seen " << time(NULL)-it->second.last_contact + << " ekey: " << RsUtil::BinToHex(it->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE) ; + + for(auto it2(it->second.providing_set.begin());it2!=it->second.providing_set.end();++it2) + std::cerr << " service " << std::hex << it2->first << std::dec << " " << it2->second.provided_groups.size() << " groups, " << it2->second.incoming_data.size() << " data" << std::endl; + } std::cerr << "Hashes: " << std::endl; for(auto it(mHandledHashes.begin());it!=mHandledHashes.end();++it) @@ -311,15 +308,18 @@ void RsGxsNetTunnelService::connectToTurtleRouter(p3turtle *tr) bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) { + RS_STACK_MUTEX(mGxsNetTunnelMtx); // We simply check for wether a managed group has a hash that corresponds to the given hash. return mHandledHashes.find(hash) != mHandledHashes.end(); } -void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) +void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& turtle_virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) { + RS_STACK_MUTEX(mGxsNetTunnelMtx); + #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " received turtle data for vpid " << virtual_peer_id << " for hash " << hash << " in direction " << (int)direction << std::endl; + GXS_NET_TUNNEL_DEBUG() << " received turtle data for vpid " << turtle_virtual_peer_id << " for hash " << hash << " in direction " << (int)direction << std::endl; #endif if(item->PacketSubType() != RS_TURTLE_SUBTYPE_GENERIC_DATA) @@ -327,45 +327,29 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co GXS_NET_TUNNEL_ERROR() << "item with type " << std::hex << item->PacketSubType() << std::dec << " received by GxsNetTunnel, but is not handled!" << std::endl; return; } - - // (cyril) this is a bit complicated. We should store pointers to the encryption keys in another structure and access it directly. + // find the group id auto it = mHandledHashes.find(hash) ; if(it == mHandledHashes.end()) { - GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for hash " << hash << " but this hash is unknown!" << std::endl; + GXS_NET_TUNNEL_ERROR() << "Cannot find hash " << hash << " to be handled by GxsNetTunnel" << std::endl; return; } - RsGxsGroupId group_id = it->second; - auto it2 = mGroups.find(group_id) ; - - if(it2 == mGroups.end()) - { - GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for hash " << hash << " and group " << group_id << " but this group id is unknown!" << std::endl; - return; - } - - RsGxsNetTunnelGroupInfo& g_info(it2->second) ; - - g_info.last_contact = time(NULL) ; - - auto it3 = g_info.virtual_peers.find(virtual_peer_id) ; - - if(it3 == g_info.virtual_peers.end()) - { - GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for hash " << hash << ", group " << group_id << " but the virtual peer id is missing!" << std::endl; - return; - } - - RsGxsNetTunnelVirtualPeerInfo& vp_info(it3->second) ; + // Now check if we got an item to advertise a virtual peer unsigned char *data = NULL ; uint32_t data_size = 0 ; - if(!p3turtle::decryptItem(static_cast(item),vp_info.encryption_master_key,data,data_size)) + // generate the decryption key based on virtual peer id and group id + + uint8_t encryption_master_key[RS_GXS_TUNNEL_CONST_EKEY_SIZE] ; + + generateEncryptionKey(group_id,turtle_virtual_peer_id,encryption_master_key); + + if(!p3turtle::decryptItem(static_cast(item),encryption_master_key,data,data_size)) { GXS_NET_TUNNEL_ERROR() << "Cannot decrypt data!" << std::endl; @@ -375,28 +359,45 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co return ; } - // Now we might get 2 kinds of items: GxsNetTunnel items, to be handled here, and Gxs data items, to be handled by the client service - RsItem *decrypted_item = RsGxsNetTunnelSerializer().deserialise(data,&data_size); - RsGxsNetTunnelVirtualPeerItem *pid_item = dynamic_cast(decrypted_item) ; if(pid_item) { - if(direction == RsTurtleGenericTunnelItem::DIRECTION_SERVER) - { #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " item is a virtual peer id item with vpid = "<< pid_item->virtual_peer_id << ". Setting virtual peer." << std::endl; + GXS_NET_TUNNEL_DEBUG() << " item is a virtual peer id item with vpid = "<< pid_item->virtual_peer_id << ". Setting virtual peer." << std::endl; +#endif +#ifdef TODO + vp_info.net_service_virtual_peer = pid_item->virtual_peer_id; + vp_info.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; #endif - vp_info.net_service_virtual_peer = pid_item->virtual_peer_id; - vp_info.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; - } - else - GXS_NET_TUNNEL_ERROR() << "Cannot decrypt data!" << std::endl; free(data); return ; } + +#ifdef TODO + auto it = mTurtle2GxsPeer.find(turtle_virtual_peer_id) ; + + if(it == mTurtle2GxsPeer.end()) + { + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for vpid " << turtle_virtual_peer_id << " but this vpid is unknown!" << std::endl; + return; + } + + RsGxsNetTunnelVirtualPeerId gxs_vpid = it->second ; + + auto it2 = mVirtualPeers.find(gxs_vpid) ; + + if(it2 == mVirtualPeers.end()) + { + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for GXS vpid " << gxs_vpid " but the virtual peer id is missing!" << std::endl; + return; + } + + RsGxsNetTunnelVirtualPeerInfo& vp_info(it2->second) ; + + else { #ifdef DEBUG_RSGXSNETTUNNEL @@ -411,10 +412,13 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co vp_info.incoming_data.push_back(bind) ; } +#endif } void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid,RsTurtleGenericTunnelItem::Direction dir) { + RS_STACK_MUTEX(mGxsNetTunnelMtx); + auto it = mHandledHashes.find(hash) ; if(it == mHandledHashes.end()) @@ -431,23 +435,19 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE ; - RsGxsNetTunnelVirtualPeerInfo& vpinfo( ginfo.virtual_peers[vpid] ) ; + uint8_t encryption_master_key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]; - vpinfo.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK ; - vpinfo.side = dir ; - vpinfo.last_contact = time(NULL) ; - - generateEncryptionKey(group_id,vpid,vpinfo.encryption_master_key ); + generateEncryptionKey(group_id,vpid,encryption_master_key ); // We need to send our own virtual peer id to the other end of the tunnel - vpinfo.net_service_virtual_peer = makeServerVirtualPeerIdForGroup(group_id); + RsGxsNetTunnelVirtualPeerId net_service_virtual_peer = locked_makeVirtualPeerId(); #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " sending back virtual peer name " << vpinfo.net_service_virtual_peer << " to end of tunnel" << std::endl; + GXS_NET_TUNNEL_DEBUG() << " sending back virtual peer name " << net_service_virtual_peer << " to end of tunnel" << std::endl; #endif RsGxsNetTunnelVirtualPeerItem pitem ; - pitem.virtual_peer_id = vpinfo.net_service_virtual_peer ; + pitem.virtual_peer_id = net_service_virtual_peer ; RsTemporaryMemory tmpmem( RsGxsNetTunnelSerializer().size(&pitem) ) ; uint32_t len = tmpmem.size(); @@ -456,7 +456,7 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; - if(p3turtle::encryptData(tmpmem,len,vpinfo.encryption_master_key,encrypted_turtle_item)) + if(p3turtle::encryptData(tmpmem,len,encryption_master_key,encrypted_turtle_item)) mTurtle->sendTurtleData(vpid,encrypted_turtle_item) ; else GXS_NET_TUNNEL_ERROR() << "cannot encrypt. Something's wrong. Data is dropped." << std::endl; @@ -464,6 +464,8 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid) { + RS_STACK_MUTEX(mGxsNetTunnelMtx); + #ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << " removing virtual peer " << vpid << " for hash " << hash << std::endl; #endif @@ -496,7 +498,7 @@ RsFileHash RsGxsNetTunnelService::calculateGroupHash(const RsGxsGroupId& group_i return RsDirUtil::sha1sum(group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; } -void RsGxsNetTunnelService::generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) const +void RsGxsNetTunnelService::generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) { // The key is generated as H(group_id | vpid) // Because group_id is not known it shouldn't be possible to recover the key by observing the traffic. @@ -519,12 +521,19 @@ void RsGxsNetTunnelService::data_tick() { GXS_NET_TUNNEL_DEBUG() << std::endl; + time_t now = time(NULL); + // cleanup - autowash(); + static time_t last_autowash = time(NULL); + + if(last_autowash + 5 > now) + { + autowash(); + last_autowash = now; + } static time_t last_dump = time(NULL); - time_t now = time(NULL); if(last_dump + 10 > now) { @@ -535,44 +544,24 @@ void RsGxsNetTunnelService::data_tick() void RsGxsNetTunnelService::autowash() { + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + for(auto it(mGroups.begin());it!=mGroups.end();++it) + { + // check if the group is in active or passive mode, in which case make sure that turtle manages tunnels or not. + // if active, then we check wether they are tunnels are not. If not, we ask turtle to monitor. + + RsGxsNetTunnelGroupInfo& ginfo(it->second) ; + + if(ginfo.group_policy == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE && ginfo.virtual_peers.empty()) + { + mTurtle->monitorTunnels(ginfo.hash,this,false) ; + ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; + } + + if(ginfo.group_policy == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE) + mTurtle->stopMonitoringTunnels(ginfo.hash); + } } -// info.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; -// -// mTurtle->monitorTunnels(hash,this,false) ; -// info.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; - - -// void RsGxsNetTunnelService::handleIncoming(const RsGxsTunnelId& tunnel_id,RsGxsTunnelItem *item) -// { -// #ifdef DEBUG_RSGXSNETTUNNEL -// GXS_NET_TUNNEL_DEBUG() << " received turtle data for vpid " << virtual_peer_id << " for hash " << hash << " in direction " << dir << std::endl; -// #endif -// if(item == NULL) -// return ; -// -// // We have 3 things to do: -// // -// // 1 - if it's a data item, send an ACK -// // 2 - if it's an ack item, mark the item as properly received, and remove it from the queue -// // 3 - if it's a status item, act accordingly. -// -// switch(item->PacketSubType()) -// { -// -// case RS_PKT_SUBTYPE_GXS_TUNNEL_DATA: handleRecvTunnelDataItem(tunnel_id,dynamic_cast(item)) ; -// break ; -// -// case RS_PKT_SUBTYPE_GXS_TUNNEL_DATA_ACK: handleRecvTunnelDataAckItem(tunnel_id,dynamic_cast(item)) ; -// break ; -// -// case RS_PKT_SUBTYPE_GXS_TUNNEL_STATUS: handleRecvStatusItem(tunnel_id,dynamic_cast(item)) ; -// break ; -// -// default: -// std::cerr << "(EE) impossible situation. DH items should be handled at the service level" << std::endl; -// } -// -// delete item ; -// } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 901aba61d..a8b9e701b 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -82,11 +82,22 @@ // However, a call should be made to set a particular group policy to "ACTIVE" for group that do not have peers and need some. // // * services also need to retrieve GXS data items that come out of tunnels. These will be available as (data,len) type, since they are not de-serialized. +// +// * GxsNetService stores data information (such as update maps) per peerId, so it makes sense to use the same PeerId for all groups of a given service +// Therefore, virtual peers are stored separately from groups, because each one can sync multiple groups. +// +// * virtual peers are also shared among services. This reduces the required amount of tunnels and tunnel requests to send. typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; class RsGxsNetTunnelItem ; +struct RsGxsNetTunnelVirtualPeerProvidingSet +{ + std::set provided_groups ; + std::list incoming_data ; +}; + struct RsGxsNetTunnelVirtualPeerInfo { enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. @@ -102,11 +113,10 @@ struct RsGxsNetTunnelVirtualPeerInfo uint8_t encryption_master_key[32] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) time_t last_contact ; // last time some data was sent/recvd - RsGxsNetTunnelVirtualPeerId net_service_virtual_peer ; // anonymised peer that is used to communicate with client services - RsGxsGroupId group_id ; // group id + TurtleVirtualPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. + RsGxsGroupId group_id ; // group id - std::list incoming_data ; - std::list outgoing_items ; + std::map providing_set; // partial list of groups provided by this virtual peer id, based on tunnel results, for each service }; struct RsGxsNetTunnelGroupInfo @@ -124,20 +134,22 @@ struct RsGxsNetTunnelGroupInfo RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group explicitely request tunnels, if none available }; - RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {} + RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0),service_id(0) {} GroupPolicy group_policy ; GroupStatus group_status ; time_t last_contact ; TurtleFileHash hash ; + uint16_t service_id ; - std::map virtual_peers ; + std::set virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. }; class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread { public: RsGxsNetTunnelService() ; + virtual ~RsGxsNetTunnelService() ; /*! * \brief Manage tunnels for this group @@ -155,7 +167,7 @@ public: * \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and * alive. This function also "registers" the group which allows to handle tunnel requests in the server side. */ - bool getVirtualPeers(const RsGxsGroupId& group_id, std::list& peers) ; // returns the virtual peers for this group + bool getVirtualPeers(uint16_t service_id, std::list& peers) ; // returns the virtual peers for this group /*! * \brief sendData @@ -204,13 +216,14 @@ private: void autowash() ; void handleIncoming(RsGxsNetTunnelItem *item) ; - static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 16 ; + static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 20 ; static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE = 32 ; std::map mGroups ; // groups on the client and server side - std::map > mVirtualPeers ; // current virtual peers, - std::map mHandledHashes ; // hashes asked to turtle + std::map mVirtualPeers ; // current virtual peers, which group they provide, and how to talk to them through turtle + std::map mHandledHashes ; // hashes asked to turtle. Used to answer tunnel requests + std::map mTurtle2GxsPeer ; // convertion table to find GXS peer id from turtle /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to @@ -224,9 +237,9 @@ private: * tunnel ID and turtle virtual peer id. This allows RsGxsNetService to keep sync-ing the data consistently. */ - RsGxsNetTunnelVirtualPeerId makeServerVirtualPeerIdForGroup(const RsGxsGroupId&group_id) const ; + RsGxsNetTunnelVirtualPeerId locked_makeVirtualPeerId() const ; - void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) const ; + static void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) ; uint8_t mRandomBias[RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE] ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. From aa59694d881cdf7681ab8aa8379836a5f0317a4a Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 1 Apr 2018 22:04:16 +0200 Subject: [PATCH 13/84] added automatic detection for need to dist-sync groups --- libretroshare/src/gxs/rsgxsnetservice.cc | 118 +++++++++++++++++------ libretroshare/src/gxs/rsgxsnetservice.h | 4 + libretroshare/src/gxs/rsgxsnettunnel.cc | 63 +++++++----- libretroshare/src/gxs/rsgxsnettunnel.h | 18 ++-- libretroshare/src/rsserver/rsinit.cc | 8 +- 5 files changed, 148 insertions(+), 63 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 7962f9bf3..2a17b54d9 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -269,6 +269,7 @@ NXS_NET_DEBUG_5 summary of transactions (useful to just know what comes in/out) NXS_NET_DEBUG_6 group sync statistics (e.g. number of posts at nighbour nodes, etc) NXS_NET_DEBUG_7 encryption/decryption of transactions + NXS_NET_DEBUG_8 gxs distant sync ***/ //#define NXS_NET_DEBUG_0 1 @@ -279,6 +280,7 @@ //#define NXS_NET_DEBUG_5 1 //#define NXS_NET_DEBUG_6 1 //#define NXS_NET_DEBUG_7 1 +#define NXS_NET_DEBUG_8 1 //#define NXS_FRAG @@ -312,11 +314,12 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING = 0x05 ; // Debug system to allow to print only for some IDs (group, Peer, etc) #if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2) || defined(NXS_NET_DEBUG_3) \ - || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6) || defined(NXS_NET_DEBUG_7) + || defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6) || defined(NXS_NET_DEBUG_7) \ + || defined(NXS_NET_DEBUG_8) static const RsPeerId peer_to_print = RsPeerId(std::string("")) ; static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("")) ; // use this to allow to this group id only, or "" for all IDs -static const uint32_t service_to_print = RS_SERVICE_TYPE_GXS_TRANS ; // use this to allow to this service id only, or 0 for all services +static const uint32_t service_to_print = RS_SERVICE_GXS_TYPE_CHANNELS ; // use this to allow to this service id only, or 0 for all services // warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums) class nullstream: public std::ostream {}; @@ -448,6 +451,7 @@ int RsGxsNetService::tick() { syncWithPeers(); syncGrpStatistics(); + checkDistantSyncState(); mSyncTs = now; } @@ -566,7 +570,6 @@ void RsGxsNetService::syncWithPeers() std::set peers; mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers); -#ifdef TODO if(mAllowDistSync) { // Grab all online virtual peers of distant tunnels for the current service. @@ -575,9 +578,8 @@ void RsGxsNetService::syncWithPeers() mGxsNetTunnel->getVirtualPeers(mServType,vpids); for(auto it(vpids.begin());it!=vpids.end();++it) - peers.push_back(RsPeerId(*it)) ; + peers.insert(RsPeerId(*it)) ; } -#endif if (peers.empty()) { // nothing to do @@ -735,6 +737,62 @@ void RsGxsNetService::syncWithPeers() #endif } +void RsGxsNetService::checkDistantSyncState() +{ + if(!mAllowDistSync) + return ; + + RsGxsGrpMetaTemporaryMap grpMeta; + mDataStore->retrieveGxsGrpMetaData(grpMeta); + + // Go through group statistics and groups without information are re-requested to random peers selected + // among the ones who provided the group info. + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___<< "Checking distant sync for all groups." << std::endl; +#endif + // get the list of online peers + + std::set online_peers; + mNetMgr->getOnlineList(mServiceInfo.mServiceType , online_peers); + + uint16_t service_id = ((mServiceInfo.mServiceType >> 8)& 0xffff); + + RS_STACK_MUTEX(mNxsMutex) ; + + for(auto it(grpMeta.begin());it!=grpMeta.end();++it) + if(it->second->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) // we only consider subscribed groups here. + { +#warning (cyril) We might need to also remove peers for recently unsubscribed groups + const RsGxsGroupId& grpId(it->first); + const RsGxsGrpConfig& rec = locked_getGrpConfig(grpId) ; + +#ifdef NXS_NET_DEBUG_6 + GXSNETDEBUG__G(it->first) << " group " << grpId; +#endif + bool at_least_one_friend_is_supplier = false ; + + for(auto it2(rec.suppliers.ids.begin());it2!=rec.suppliers.ids.end() && !at_least_one_friend_is_supplier;++it2) + if(online_peers.find(*it2) != online_peers.end()) // check that the peer is online + at_least_one_friend_is_supplier = true ; + + if(at_least_one_friend_is_supplier) + { + mGxsNetTunnel->releasePeers(service_id,grpId); +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___<< " Group " << grpId << ": suppliers among friends. Releasing peers." << std::endl; +#endif + } + else + { + mGxsNetTunnel->requestPeers(service_id,grpId); +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___<< " Group " << grpId << ": no suppliers among friends. Requesting peers." << std::endl; +#endif + } + } +} + void RsGxsNetService::syncGrpStatistics() { RS_STACK_MUTEX(mNxsMutex) ; @@ -763,44 +821,44 @@ void RsGxsNetService::syncGrpStatistics() #endif if(rec.statistics_update_TS + GROUP_STATS_UPDATE_DELAY < now && rec.suppliers.ids.size() > 0) - { + { #ifdef NXS_NET_DEBUG_6 - GXSNETDEBUG__G(it->first) << " needs update. Randomly asking to some friends" << std::endl; + GXSNETDEBUG__G(it->first) << " needs update. Randomly asking to some friends" << std::endl; #endif - // randomly select GROUP_STATS_UPDATE_NB_PEERS friends among the suppliers of this group + // randomly select GROUP_STATS_UPDATE_NB_PEERS friends among the suppliers of this group - uint32_t n = RSRandom::random_u32() % rec.suppliers.ids.size() ; + uint32_t n = RSRandom::random_u32() % rec.suppliers.ids.size() ; - std::set::const_iterator rit = rec.suppliers.ids.begin(); - for(uint32_t i=0;i::const_iterator rit = rec.suppliers.ids.begin(); + for(uint32_t i=0;ifirst) << " asking friend " << peer_id << " for an update of stats for group " << it->first << std::endl; + GXSNETDEBUG_PG(peer_id,it->first) << " asking friend " << peer_id << " for an update of stats for group " << it->first << std::endl; #endif - RsNxsSyncGrpStatsItem *grs = new RsNxsSyncGrpStatsItem(mServType) ; + RsNxsSyncGrpStatsItem *grs = new RsNxsSyncGrpStatsItem(mServType) ; - grs->request_type = RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_REQUEST ; + grs->request_type = RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_REQUEST ; - grs->grpId = it->first ; - grs->PeerId(peer_id) ; + grs->grpId = it->first ; + grs->PeerId(peer_id) ; - sendItem(grs) ; + sendItem(grs) ; + } } - } - } + } #ifdef NXS_NET_DEBUG_6 else GXSNETDEBUG__G(it->first) << " up to date." << std::endl; diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 49d44dab2..8142f2e74 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -35,6 +35,7 @@ #include "pqi/p3linkmgr.h" #include "rsitems/rsnxsitems.h" #include "rsitems/rsgxsupdateitems.h" +#include "rsgxsnettunnel.h" #include "rsgxsnetutils.h" #include "pqi/p3cfgmgr.h" #include "rsgixs.h" @@ -394,6 +395,7 @@ private: void locked_pushGrpRespFromList(std::list& respList, const RsPeerId& peer, const uint32_t& transN); void locked_pushMsgRespFromList(std::list& itemL, const RsPeerId& sslId, const RsGxsGroupId &grp_id, const uint32_t& transN); + void checkDistantSyncState(); void syncWithPeers(); void syncGrpStatistics(); void addGroupItemToList(NxsTransaction*& tr, @@ -582,6 +584,8 @@ private: uint32_t mDefaultMsgStorePeriod ; uint32_t mDefaultMsgSyncPeriod ; + + RsGxsNetTunnelService *mGxsNetTunnel; }; #endif // RSGXSNETSERVICE_H diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index d5628eed3..c502f2c6d 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -193,14 +193,14 @@ bool RsGxsNetTunnelService::getVirtualPeers(uint16_t service_id, std::listsecond.providing_set.find(service_id) != it->second.providing_set.end()) peers.push_back(it->first) ; -#ifdef DEBUG_GXS_TUNNEL +#ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << " service " << std::hex << service_id << std::dec << " returning " << peers.size() << " peers." << std::endl; #endif return true ; } -bool RsGxsNetTunnelService::requestPeers(const RsGxsGroupId& group_id) +bool RsGxsNetTunnelService::requestPeers(uint16_t service_id,const RsGxsGroupId& group_id) { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -211,11 +211,14 @@ bool RsGxsNetTunnelService::requestPeers(const RsGxsGroupId& group_id) ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE; // we dont set the group policy here. It will only be set if no peers, or too few peers are available. +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " service " << std::hex << service_id << std::dec << " requesting peers for group " << group_id << std::endl; +#endif return true; } -bool RsGxsNetTunnelService::releasePeers(const RsGxsGroupId& group_id) +bool RsGxsNetTunnelService::releasePeers(uint16_t service_id, const RsGxsGroupId& group_id) { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -226,6 +229,9 @@ bool RsGxsNetTunnelService::releasePeers(const RsGxsGroupId& group_id) ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE; mTurtle->stopMonitoringTunnels(ginfo.hash) ; +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " service " << std::hex << service_id << std::dec << " releasing peers for group " << group_id << std::endl; +#endif return true; } @@ -329,14 +335,14 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co } // find the group id - auto it = mHandledHashes.find(hash) ; + auto it4 = mHandledHashes.find(hash) ; - if(it == mHandledHashes.end()) + if(it4 == mHandledHashes.end()) { GXS_NET_TUNNEL_ERROR() << "Cannot find hash " << hash << " to be handled by GxsNetTunnel" << std::endl; return; } - RsGxsGroupId group_id = it->second; + RsGxsGroupId group_id = it4->second; // Now check if we got an item to advertise a virtual peer @@ -367,21 +373,33 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co #ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << " item is a virtual peer id item with vpid = "<< pid_item->virtual_peer_id << ". Setting virtual peer." << std::endl; #endif -#ifdef TODO - vp_info.net_service_virtual_peer = pid_item->virtual_peer_id; - vp_info.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; -#endif + // we receive a virtual peer id, so we need to update the local information for this peer id + + mTurtle2GxsPeer[turtle_virtual_peer_id] = pid_item->virtual_peer_id ; + + RsGxsNetTunnelVirtualPeerInfo& vp_info(mVirtualPeers[pid_item->virtual_peer_id]) ; + + vp_info.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; // status of the peer + vp_info.side = direction; // client/server + vp_info.last_contact = time(NULL); // last time some data was sent/recvd + + memcpy(vp_info.encryption_master_key,encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE); + + vp_info.turtle_virtual_peer_id = turtle_virtual_peer_id; // turtle peer to use when sending data to this vpid. free(data); return ; } + delete decrypted_item ; + + // item is a generic data item for the client. Let's store the data in the appropriate incoming data queue. -#ifdef TODO auto it = mTurtle2GxsPeer.find(turtle_virtual_peer_id) ; if(it == mTurtle2GxsPeer.end()) { GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for vpid " << turtle_virtual_peer_id << " but this vpid is unknown!" << std::endl; + free(data); return; } @@ -391,28 +409,27 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co if(it2 == mVirtualPeers.end()) { - GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for GXS vpid " << gxs_vpid " but the virtual peer id is missing!" << std::endl; + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for GXS vpid " << gxs_vpid << " but the virtual peer id is missing!" << std::endl; + free(data); return; } RsGxsNetTunnelVirtualPeerInfo& vp_info(it2->second) ; + uint16_t service_id = getRsItemService(getRsItemId(data)) ; - else - { #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " item is GXS data. Storing into incoming list." << std::endl; + GXS_NET_TUNNEL_DEBUG() << "item contains generic data for service " << std::hex << service_id << std::dec << " for VPID " << gxs_vpid << ". Storing in incoming list" << std::endl; #endif - // push the data into the service incoming data list - RsTlvBinaryData *bind = new RsTlvBinaryData; - bind->tlvtype = 0; - bind->bin_len = data_size; - bind->bin_data = data; + // push the data into the service incoming data list - vp_info.incoming_data.push_back(bind) ; - } -#endif + RsTlvBinaryData *bind = new RsTlvBinaryData; + bind->tlvtype = 0; + bind->bin_len = data_size; + bind->bin_data = data; + + vp_info.providing_set[service_id].incoming_data.push_back(bind) ; } void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid,RsTurtleGenericTunnelItem::Direction dir) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index a8b9e701b..451f5fb9c 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -87,6 +87,13 @@ // Therefore, virtual peers are stored separately from groups, because each one can sync multiple groups. // // * virtual peers are also shared among services. This reduces the required amount of tunnels and tunnel requests to send. +// +// +// How do we know that a group needs distant sync? +// * look into GrpConfigMap for suppliers. Suppliers is cleared at load. +// * last_update_TS in GrpConfigMap is randomised so it cannot be used +// * we need a way to know that there's no suppliers for good reasons (not that we just started) +// * typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; @@ -105,16 +112,15 @@ struct RsGxsNetTunnelVirtualPeerInfo RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. }; - RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN) { memset(encryption_master_key,0,32) ; } + RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; } ~RsGxsNetTunnelVirtualPeerInfo() ; uint8_t vpid_status ; // status of the peer - uint8_t side ; // client/server - uint8_t encryption_master_key[32] ; // key from which the encryption key is derived for each virtual peer (using H(master_key | random IV)) time_t last_contact ; // last time some data was sent/recvd + uint8_t side ; // client/server + uint8_t encryption_master_key[32]; TurtleVirtualPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. - RsGxsGroupId group_id ; // group id std::map providing_set; // partial list of groups provided by this virtual peer id, based on tunnel results, for each service }; @@ -155,13 +161,13 @@ public: * \brief Manage tunnels for this group * @param group_id group for which tunnels should be released */ - bool requestPeers(const RsGxsGroupId&group_id) ; + bool requestPeers(uint16_t service_id, const RsGxsGroupId&group_id) ; /*! * \brief Stop managing tunnels for this group * @param group_id group for which tunnels should be released */ - bool releasePeers(const RsGxsGroupId&group_id) ; + bool releasePeers(uint16_t service_id,const RsGxsGroupId&group_id) ; /*! * \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 0d9300202..17314626e 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1386,10 +1386,10 @@ int RsServer::StartupRetroShare() // create GXS photo service RsGxsNetService* gxschannels_ns = new RsGxsNetService( - RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr, - mGxsChannels, mGxsChannels->getServiceInfo(), - mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils); + RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr, + mGxsChannels, mGxsChannels->getServiceInfo(), + mReputations, mGxsCircles,mGxsIdService, + pgpAuxUtils,true,true,true); mGxsChannels->setNetworkExchangeService(gxschannels_ns) ; From 8fe3eb711d6cae4b6be875dc5deef7e0cc9d8b7a Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 2 Apr 2018 17:07:32 +0200 Subject: [PATCH 14/84] fixed a few bugs in GXS dist sync tunneling --- libretroshare/src/gxs/rsgxsnetservice.cc | 5 ++- libretroshare/src/gxs/rsgxsnetservice.h | 6 +-- libretroshare/src/gxs/rsgxsnettunnel.cc | 51 ++++++++++++++++-------- libretroshare/src/gxs/rsgxsnettunnel.h | 3 ++ libretroshare/src/rsserver/rsinit.cc | 15 ++++--- 5 files changed, 53 insertions(+), 27 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 2a17b54d9..e86d325de 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -362,7 +362,8 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, const RsServiceInfo serviceInfo, RsGixsReputation* reputations, RsGcxs* circles, RsGixs *gixs, - PgpAuxUtils *pgpUtils, bool grpAutoSync, bool msgAutoSync, bool distSync, uint32_t default_store_period, uint32_t default_sync_period) + PgpAuxUtils *pgpUtils, RsGxsNetTunnelService *mGxsNT, + bool grpAutoSync, bool msgAutoSync, bool distSync, uint32_t default_store_period, uint32_t default_sync_period) : p3ThreadedService(), p3Config(), mTransactionN(0), mObserver(nxsObs), mDataStore(gds), mServType(servType), mTransactionTimeOut(TRANSAC_TIMEOUT), @@ -370,7 +371,7 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, mSyncTs(0), mLastKeyPublishTs(0), mLastCleanRejectedMessages(0), mSYNC_PERIOD(SYNC_PERIOD), mCircles(circles), mGixs(gixs), - mReputations(reputations), mPgpUtils(pgpUtils), + mReputations(reputations), mPgpUtils(pgpUtils),mGxsNetTunnel(mGxsNT), mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync),mAllowDistSync(distSync), mServiceInfo(serviceInfo), mDefaultMsgStorePeriod(default_store_period), mDefaultMsgSyncPeriod(default_sync_period) diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 8142f2e74..e798f8717 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -90,7 +90,7 @@ public: RsNxsObserver *nxsObs, // used to be = NULL. const RsServiceInfo serviceInfo, RsGixsReputation* reputations = NULL, RsGcxs* circles = NULL, RsGixs *gixs=NULL, - PgpAuxUtils *pgpUtils = NULL, + PgpAuxUtils *pgpUtils = NULL, RsGxsNetTunnelService *mGxsNT = NULL, bool grpAutoSync = true, bool msgAutoSync = true,bool distSync=false, uint32_t default_store_period = RS_GXS_DEFAULT_MSG_STORE_PERIOD, uint32_t default_sync_period = RS_GXS_DEFAULT_MSG_REQ_PERIOD); @@ -543,6 +543,8 @@ private: RsGixs *mGixs; RsGixsReputation* mReputations; PgpAuxUtils *mPgpUtils; + RsGxsNetTunnelService *mGxsNetTunnel; + bool mGrpAutoSync; bool mAllowMsgSync; bool mAllowDistSync; @@ -584,8 +586,6 @@ private: uint32_t mDefaultMsgStorePeriod ; uint32_t mDefaultMsgSyncPeriod ; - - RsGxsNetTunnelService *mGxsNetTunnel; }; #endif // RSGXSNETSERVICE_H diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index c502f2c6d..3d0f25a15 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -24,6 +24,7 @@ */ #include "util/rsdir.h" +#include "util/rstime.h" #include "retroshare/rspeers.h" #include "serialiser/rstypeserializer.h" #include "rsgxsnettunnel.h" @@ -209,6 +210,9 @@ bool RsGxsNetTunnelService::requestPeers(uint16_t service_id,const RsGxsGroupId& RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE; + ginfo.hash = calculateGroupHash(group_id) ; + + mHandledHashes[ginfo.hash] = group_id ; // we dont set the group policy here. It will only be set if no peers, or too few peers are available. #ifdef DEBUG_RSGXSNETTUNNEL @@ -227,6 +231,10 @@ bool RsGxsNetTunnelService::releasePeers(uint16_t service_id, const RsGxsGroupId RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE; + ginfo.hash = calculateGroupHash(group_id) ; + + mHandledHashes[ginfo.hash] = group_id ; // yes, we do not remove, because we're supposed to answer tunnel requests from other peers. + mTurtle->stopMonitoringTunnels(ginfo.hash) ; #ifdef DEBUG_RSGXSNETTUNNEL @@ -257,21 +265,21 @@ void RsGxsNetTunnelService::dump() const RS_STACK_MUTEX(mGxsNetTunnelMtx); static std::string group_status_str[4] = { - std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN ]"), - std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE ]"), - std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED]"), - std::string("[RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE ]") + std::string("[UNKNOWN ]"), + std::string("[IDLE ]"), + std::string("[TUNNELS_REQUESTED]"), + std::string("[VPIDS_AVAILABLE ]") }; static std::string group_policy_str[3] = { - std::string("[RS_GXS_NET_TUNNEL_POLICY_UNKNOWN]"), - std::string("[RS_GXS_NET_TUNNEL_POLICY_PASSIVE]"), - std::string("[RS_GXS_NET_TUNNEL_POLICY_ACTIVE ]"), + std::string("[UNKNOWN]"), + std::string("[PASSIVE]"), + std::string("[ACTIVE ]"), }; static std::string vpid_status_str[3] = { - std::string("[RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN ]"), - std::string("[RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK ]"), - std::string("[RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ]") + std::string("[UNKNOWN ]"), + std::string("[TUNNEL_OK]"), + std::string("[ACTIVE ]") }; std::cerr << "GxsNetTunnelService dump: " << std::endl; @@ -279,7 +287,7 @@ void RsGxsNetTunnelService::dump() const for(auto it(mGroups.begin());it!=mGroups.end();++it) { - std::cerr << " " << it->first << " hash: " << it->second.hash << " policy: " << group_policy_str[it->second.group_policy] << " status: " << group_status_str[it->second.group_status] << "] Last contact: " << time(NULL) - it->second.last_contact << " secs ago" << std::endl; + std::cerr << " " << it->first << " hash: " << it->second.hash << " policy: " << group_policy_str[it->second.group_policy] << " status: " << group_status_str[it->second.group_status] << " Last contact: " << time(NULL) - it->second.last_contact << " secs ago" << std::endl; std::cerr << " virtual peers:" << std::endl; for(auto it2(it->second.virtual_peers.begin());it2!=it->second.virtual_peers.end();++it2) std::cerr << " " << *it2 << std::endl; @@ -291,7 +299,7 @@ void RsGxsNetTunnelService::dump() const std::cerr << " GXS Peer:" << it->first << " Turtle:" << it->second.turtle_virtual_peer_id << " status: " << vpid_status_str[it->second.vpid_status] << " s: " << (int)it->second.side << " last seen " << time(NULL)-it->second.last_contact - << " ekey: " << RsUtil::BinToHex(it->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE) ; + << " ekey: " << RsUtil::BinToHex(it->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE) << std::endl; for(auto it2(it->second.providing_set.begin());it2!=it->second.providing_set.end();++it2) std::cerr << " service " << std::hex << it2->first << std::dec << " " << it2->second.provided_groups.size() << " groups, " << it2->second.incoming_data.size() << " data" << std::endl; @@ -474,7 +482,7 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; if(p3turtle::encryptData(tmpmem,len,encryption_master_key,encrypted_turtle_item)) - mTurtle->sendTurtleData(vpid,encrypted_turtle_item) ; + mPendingTurtleItems.push_back(std::make_pair(vpid,encrypted_turtle_item)) ; // we cannot send directly because of turtle mutex locked before calling addVirtualPeer. else GXS_NET_TUNNEL_ERROR() << "cannot encrypt. Something's wrong. Data is dropped." << std::endl; } @@ -536,7 +544,15 @@ void RsGxsNetTunnelService::generateEncryptionKey(const RsGxsGroupId& group_id,c void RsGxsNetTunnelService::data_tick() { - GXS_NET_TUNNEL_DEBUG() << std::endl; + while(!mPendingTurtleItems.empty()) + { + auto& it(mPendingTurtleItems.front()); + + mTurtle->sendTurtleData(it.first,it.second) ; + mPendingTurtleItems.pop_front(); + } + + rstime::rs_usleep(1*1000*1000); // 1 sec time_t now = time(NULL); @@ -544,7 +560,7 @@ void RsGxsNetTunnelService::data_tick() static time_t last_autowash = time(NULL); - if(last_autowash + 5 > now) + if(last_autowash + 5 < now) { autowash(); last_autowash = now; @@ -552,7 +568,7 @@ void RsGxsNetTunnelService::data_tick() static time_t last_dump = time(NULL); - if(last_dump + 10 > now) + if(last_dump + 10 < now) { last_dump = now; dump(); @@ -577,7 +593,10 @@ void RsGxsNetTunnelService::autowash() } if(ginfo.group_policy == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE) + { mTurtle->stopMonitoringTunnels(ginfo.hash); + ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE; + } } } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 451f5fb9c..3d6910d14 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -221,6 +221,7 @@ protected: private: void autowash() ; void handleIncoming(RsGxsNetTunnelItem *item) ; + void flush_pending_items(); static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 20 ; static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE = 32 ; @@ -231,6 +232,8 @@ private: std::map mHandledHashes ; // hashes asked to turtle. Used to answer tunnel requests std::map mTurtle2GxsPeer ; // convertion table to find GXS peer id from turtle + std::list > mPendingTurtleItems ; // items that need to be sent off-turtle Mutex. + /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to * hide the real group id. diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 17314626e..c51abf01e 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1300,7 +1300,7 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_GXSID, gxsid_ds, nxsMgr, mGxsIdService, mGxsIdService->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils, + pgpAuxUtils,NULL, false,false); // don't synchronise group automatic (need explicit group request) // don't sync messages at all. @@ -1319,9 +1319,7 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_GXSCIRCLE, gxscircles_ds, nxsMgr, mGxsCircles, mGxsCircles->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils, - true, // synchronise group automatic - true); // sync messages automatic, since they contain subscription requests. + pgpAuxUtils); mGxsCircles->setNetworkExchangeService(gxscircles_ns) ; @@ -1379,6 +1377,8 @@ int RsServer::StartupRetroShare() /**** Channel GXS service ****/ + RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ; + RsGeneralDataService* gxschannels_ds = new RsDataService(currGxsDir + "/", "gxschannels_db", RS_SERVICE_GXS_TYPE_CHANNELS, NULL, rsInitConfig->gxs_passwd); @@ -1389,7 +1389,7 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr, mGxsChannels, mGxsChannels->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils,true,true,true); + pgpAuxUtils,mGxsNetTunnel,true,true,true); mGxsChannels->setNetworkExchangeService(gxschannels_ns) ; @@ -1444,7 +1444,7 @@ int RsServer::StartupRetroShare() RsGxsNetService* gxstrans_ns = new RsGxsNetService( RS_SERVICE_TYPE_GXS_TRANS, gxstrans_ds, nxsMgr, mGxsTrans, mGxsTrans->getServiceInfo(), mReputations, mGxsCircles, - mGxsIdService, pgpAuxUtils,true,true,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD); + mGxsIdService, pgpAuxUtils,NULL,true,true,false,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD); mGxsTrans->setNetworkExchangeService(gxstrans_ns); pqih->addService(gxstrans_ns, true); @@ -1486,6 +1486,7 @@ int RsServer::StartupRetroShare() // connect components to turtle router. + mGxsNetTunnel->connectToTurtleRouter(tr) ; ftserver->connectToTurtleRouter(tr) ; ftserver->connectToFileDatabase(fdb) ; chatSrv->connectToGxsTunnelService(mGxsTunnels) ; @@ -1824,6 +1825,8 @@ int RsServer::StartupRetroShare() //rsWire = mWire; /*** start up GXS core runner ***/ + startServiceThread(mGxsNetTunnel, "gxs net tunnel"); + startServiceThread(mGxsIdService, "gxs id"); startServiceThread(mGxsCircles, "gxs circle"); startServiceThread(mPosted, "gxs posted"); From 7d561bccebb56989842eb9bb923d939ea807c418 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 4 Apr 2018 21:41:21 +0200 Subject: [PATCH 15/84] added distant data access in GxsNetService --- libretroshare/src/gxs/rsgxsnetservice.cc | 80 +++++++++++++++++------- libretroshare/src/gxs/rsgxsnetservice.h | 3 + libretroshare/src/gxs/rsgxsnettunnel.cc | 56 ++++++++++++----- libretroshare/src/gxs/rsgxsnettunnel.h | 24 +++++-- 4 files changed, 120 insertions(+), 43 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index e86d325de..cd68ea6bf 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -336,7 +336,6 @@ static std::string nice_time_stamp(time_t now,time_t TS) } } - static std::ostream& gxsnetdebug(const RsPeerId& peer_id,const RsGxsGroupId& grp_id,uint32_t service_type) { static nullstream null ; @@ -611,7 +610,7 @@ void RsGxsNetService::syncWithPeers() #ifdef NXS_NET_DEBUG_5 GXSNETDEBUG_P_(*sit) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global group TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) to himself" << std::endl; #endif - sendItem(grp); + generic_sendItem(grp); } if(!mAllowMsgSync) @@ -727,7 +726,7 @@ void RsGxsNetService::syncWithPeers() #ifdef NXS_NET_DEBUG_7 GXSNETDEBUG_PG(*sit,grpId) << " Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself - in clear " << std::endl; #endif - sendItem(msg); + generic_sendItem(msg); #ifdef NXS_NET_DEBUG_5 GXSNETDEBUG_PG(*sit,grpId) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself" << std::endl; @@ -738,6 +737,28 @@ void RsGxsNetService::syncWithPeers() #endif } +void RsGxsNetService::generic_sendItem(RsNxsItem *si) +{ + // check if the item is to be sent to a distant peer or not + + if(mAllowDistSync && mGxsNetTunnel->isDistantPeer( static_cast(si->PeerId()))) + { + RsNxsSerialiser ser(mServType); + + uint32_t size = ser.size(si); + unsigned char *mem = (unsigned char *)rs_malloc(size) ; + + if(!mem) + return ; + + ser.serialise(si,mem,&size) ; + + mGxsNetTunnel->sendData(mem,size,static_cast(si->PeerId())); + } + else + sendItem(si) ; +} + void RsGxsNetService::checkDistantSyncState() { if(!mAllowDistSync) @@ -757,8 +778,6 @@ void RsGxsNetService::checkDistantSyncState() std::set online_peers; mNetMgr->getOnlineList(mServiceInfo.mServiceType , online_peers); - uint16_t service_id = ((mServiceInfo.mServiceType >> 8)& 0xffff); - RS_STACK_MUTEX(mNxsMutex) ; for(auto it(grpMeta.begin());it!=grpMeta.end();++it) @@ -779,14 +798,14 @@ void RsGxsNetService::checkDistantSyncState() if(at_least_one_friend_is_supplier) { - mGxsNetTunnel->releasePeers(service_id,grpId); + mGxsNetTunnel->releasePeers(mServType,grpId); #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___<< " Group " << grpId << ": suppliers among friends. Releasing peers." << std::endl; #endif } else { - mGxsNetTunnel->requestPeers(service_id,grpId); + mGxsNetTunnel->requestPeers(mServType,grpId); #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___<< " Group " << grpId << ": no suppliers among friends. Requesting peers." << std::endl; #endif @@ -856,7 +875,7 @@ void RsGxsNetService::syncGrpStatistics() grs->grpId = it->first ; grs->PeerId(peer_id) ; - sendItem(grs) ; + generic_sendItem(grs) ; } } } @@ -937,7 +956,7 @@ void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStatsItem *grs) GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " sending back statistics item with " << vec.size() << " elements." << std::endl; #endif - sendItem(grs_resp) ; + generic_sendItem(grs_resp) ; } else if(grs->request_type == RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_RESPONSE) { @@ -1638,11 +1657,30 @@ RsSerialiser *RsGxsNetService::setupSerialiser() return rss; } +RsItem *RsGxsNetService::generic_recvItem() +{ + { + RsItem *item ; + + if(NULL != (item=recvItem())) + return item ; + } + + unsigned char *data = NULL ; + uint32_t size = 0 ; + RsGxsNetTunnelVirtualPeerId virtual_peer_id ; + + if(mGxsNetTunnel->receiveData(mServType,data,size,virtual_peer_id)) + return dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; + + return NULL ; +} + void RsGxsNetService::recvNxsItemQueue() { RsItem *item ; - while(NULL != (item=recvItem())) + while(NULL != (item=generic_recvItem())) { #ifdef NXS_NET_DEBUG_1 GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetService Item:" << (void*)item << " type=" << std::hex << item->PacketId() << std::dec << std::endl ; @@ -2243,7 +2281,7 @@ void RsGxsNetService::processTransactions() lit_end = tr->mItems.end(); for(; lit != lit_end; ++lit){ - sendItem(*lit); + generic_sendItem(*lit); } tr->mItems.clear(); // clear so they don't get deleted in trans cleaning @@ -2352,7 +2390,7 @@ void RsGxsNetService::processTransactions() trans->transactFlag = RsNxsTransacItem::FLAG_END_SUCCESS; trans->transactionNumber = transN; trans->PeerId(tr->mTransaction->PeerId()); - sendItem(trans); + generic_sendItem(trans); // move to completed transactions @@ -2395,7 +2433,7 @@ void RsGxsNetService::processTransactions() (tr->mTransaction->transactFlag & RsNxsTransacItem::FLAG_TYPE_MASK); trans->transactionNumber = transN; trans->PeerId(tr->mTransaction->PeerId()); - sendItem(trans); + generic_sendItem(trans); tr->mFlag = NxsTransaction::FLAG_STATE_RECEIVING; } @@ -2772,7 +2810,7 @@ void RsGxsNetService::locked_pushMsgTransactionFromList(std::list& r newTrans->mTransaction->PeerId(mOwnId); if (locked_addTransaction(newTrans)) - sendItem(transac); + generic_sendItem(transac); else { delete newTrans; @@ -3068,7 +3106,7 @@ void RsGxsNetService::locked_pushGrpTransactionFromList( std::list& newTrans->mTransaction->PeerId(mOwnId); if (locked_addTransaction(newTrans)) - sendItem(transac); + generic_sendItem(transac); else { delete newTrans; @@ -3272,8 +3310,8 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) ntr->PeerId(tr->mTransaction->PeerId()); if(locked_addTransaction(newTr)) - sendItem(ntr); - else + generic_sendItem(ntr); + else { delete ntr ; delete newTr; @@ -3567,7 +3605,7 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr) ntr->PeerId(tr->mTransaction->PeerId()); if(locked_addTransaction(newTr)) - sendItem(ntr); + generic_sendItem(ntr); else { delete ntr ; @@ -3886,7 +3924,7 @@ void RsGxsNetService::locked_pushGrpRespFromList(std::list& respList << peer << " with " << respList.size() << " groups " << std::endl; #endif if(locked_addTransaction(tr)) - sendItem(trItem); + generic_sendItem(trItem); else { delete tr ; @@ -4411,7 +4449,7 @@ void RsGxsNetService::locked_pushMsgRespFromList(std::list& itemL, c #endif // signal peer to prepare for transaction if(locked_addTransaction(tr)) - sendItem(trItem); + generic_sendItem(trItem); else { delete tr ; @@ -4798,7 +4836,7 @@ void RsGxsNetService::sharePublishKeysPending() publishKeyItem->private_key = publishKey ; publishKeyItem->PeerId(*it); - sendItem(publishKeyItem); + generic_sendItem(publishKeyItem); #ifdef NXS_NET_DEBUG_3 GXSNETDEBUG_PG(*it,grpMeta->mGroupId) << " sent key item to " << *it << std::endl; #endif diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index e798f8717..aac6c474a 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -495,6 +495,9 @@ private: void cleanRejectedMessages(); void processObserverNotifications(); + void generic_sendItem(RsNxsItem *si); + RsItem *generic_recvItem(); + private: static void locked_checkDelay(uint32_t& time_in_secs); diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 3d0f25a15..65ae72111 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -38,17 +38,6 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") {} -//===========================================================================================================================================// -// Internal structures // -//===========================================================================================================================================// - -RsGxsNetTunnelVirtualPeerInfo::~RsGxsNetTunnelVirtualPeerInfo() -{ - for(auto it(providing_set.begin());it!=providing_set.end();++it) - for(auto it2(it->second.incoming_data.begin());it2!=it->second.incoming_data.end();++it2) - delete *it2 ; -} - //===========================================================================================================================================// // Transport Items // //===========================================================================================================================================// @@ -140,6 +129,40 @@ RsGxsNetTunnelService::~RsGxsNetTunnelService() mGroups.clear(); mHandledHashes.clear(); mVirtualPeers.clear(); + mIncomingData.clear(); +} + +bool RsGxsNetTunnelService::isDistantPeer(const RsGxsNetTunnelVirtualPeerId& virtual_peer) +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + return mVirtualPeers.find(virtual_peer) != mVirtualPeers.end(); +} + +bool RsGxsNetTunnelService::receiveData(uint16_t service_id,unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + std::list >& lst(mIncomingData[service_id]) ; + + if(lst.empty()) + { + data = NULL; + data_len = 0; + return false ; + } + + data = (unsigned char*)lst.front().second->bin_data ; + data_len = lst.front().second->bin_len ; + virtual_peer = lst.front().first; + + lst.front().second->bin_data = NULL ; // avoids deletion + lst.front().second->bin_len = 0 ; // avoids deletion + + delete lst.front().second; + lst.pop_front(); + + return true; } bool RsGxsNetTunnelService::sendData(unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) @@ -297,12 +320,12 @@ void RsGxsNetTunnelService::dump() const for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) { std::cerr << " GXS Peer:" << it->first << " Turtle:" << it->second.turtle_virtual_peer_id - << " status: " << vpid_status_str[it->second.vpid_status] << " s: " + << " status: " << vpid_status_str[it->second.vpid_status] << " direction: " << (int)it->second.side << " last seen " << time(NULL)-it->second.last_contact - << " ekey: " << RsUtil::BinToHex(it->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE) << std::endl; + << " ekey: " << RsUtil::BinToHex(it->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE,10) << std::endl; for(auto it2(it->second.providing_set.begin());it2!=it->second.providing_set.end();++it2) - std::cerr << " service " << std::hex << it2->first << std::dec << " " << it2->second.provided_groups.size() << " groups, " << it2->second.incoming_data.size() << " data" << std::endl; + std::cerr << " service " << std::hex << it2->first << std::dec << " " << it2->second.provided_groups.size() << " groups" << std::endl; } std::cerr << "Hashes: " << std::endl; @@ -422,8 +445,6 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co return; } - RsGxsNetTunnelVirtualPeerInfo& vp_info(it2->second) ; - uint16_t service_id = getRsItemService(getRsItemId(data)) ; #ifdef DEBUG_RSGXSNETTUNNEL @@ -437,7 +458,7 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co bind->bin_len = data_size; bind->bin_data = data; - vp_info.providing_set[service_id].incoming_data.push_back(bind) ; + mIncomingData[service_id].push_back(std::make_pair(gxs_vpid,bind)) ; } void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid,RsTurtleGenericTunnelItem::Direction dir) @@ -459,6 +480,7 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE ; + ginfo.virtual_peers.insert(vpid); uint8_t encryption_master_key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 3d6910d14..7543689a2 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -113,7 +113,7 @@ struct RsGxsNetTunnelVirtualPeerInfo }; RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; } - ~RsGxsNetTunnelVirtualPeerInfo() ; + virtual ~RsGxsNetTunnelVirtualPeerInfo(){} uint8_t vpid_status ; // status of the peer time_t last_contact ; // last time some data was sent/recvd @@ -186,12 +186,24 @@ public: bool sendData(unsigned char *& data, uint32_t data_len, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! - * \brief receivedItem - * returns the next received item from the given virtual peer. - * \param virtual_peer + * \brief receiveData + * returns the next piece of data received fro the given service, and the virtual GXS peer that sended it. + * \param service_id service that provide the data + * \param data memory check containing the data. Memory ownership belongs to the client. + * \param data_len length of memory chunk + * \param virtual_peer peer who sent the data * \return + * true if something is returned. If not, data is set to NULL, data_len to 0. */ - RsItem *receivedItem(const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + bool receiveData(uint16_t service_id,unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + + /*! + * \brief isDistantPeer + * returns wether the peer is in the list of available distant peers or not + * \return true if the peer is a distant GXS peer. + */ + + bool isDistantPeer(const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! * \brief dumps all information about monitored groups. @@ -234,6 +246,8 @@ private: std::list > mPendingTurtleItems ; // items that need to be sent off-turtle Mutex. + std::map > > mIncomingData; // list of incoming data items, per service. + /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to * hide the real group id. From 5775203b69bd11992ae0d4ff4dfbf5fb45cf243d Mon Sep 17 00:00:00 2001 From: RetroPooh Date: Thu, 5 Apr 2018 22:22:54 +0300 Subject: [PATCH 16/84] bring back open local file on link click --- retroshare-gui/src/gui/RetroShareLink.cpp | 184 +++++++++++----------- 1 file changed, 96 insertions(+), 88 deletions(-) diff --git a/retroshare-gui/src/gui/RetroShareLink.cpp b/retroshare-gui/src/gui/RetroShareLink.cpp index 5b45ff584..ad149c15a 100644 --- a/retroshare-gui/src/gui/RetroShareLink.cpp +++ b/retroshare-gui/src/gui/RetroShareLink.cpp @@ -1388,11 +1388,104 @@ static void processList(const QStringList &list, const QString &textSingular, co case TYPE_FILE: { - col.merge_in(link.name(),link.size(),RsFileHash(link.hash().toStdString())) ; - fileLinkFound = true; + FileInfo fi1; + if(links.size()==1 && rsFiles->alreadyHaveFile(RsFileHash(link.hash().toStdString()), fi1)) + { + /* fallthrough */ + } + else + { + col.merge_in(link.name(),link.size(),RsFileHash(link.hash().toStdString())) ; + fileLinkFound = true; + break; + } + } + //break; + case TYPE_EXTRAFILE: + { +#ifdef DEBUG_RSLINK + std::cerr << " RetroShareLink::process FileRequest : fileName : " << link.name().toUtf8().constData() << ". fileHash : " << link.hash().toStdString() << ". fileSize : " << link.size() << std::endl; +#endif + + needNotifySuccess = true; + std::list srcIds; + + // Add the link built-in source. This is needed for EXTRA files, where the source is specified in the link. + + if(link.type() == TYPE_EXTRAFILE) + { +#ifdef DEBUG_RSLINK + std::cerr << " RetroShareLink::process Adding built-in source " << link.SSLId().toStdString() << std::endl; +#endif + srcIds.push_back(RsPeerId(link.SSLId().toStdString())) ; + } + + // Get a list of available direct sources, in case the file is browsable only. + // + FileInfo finfo ; + rsFiles->FileDetails(RsFileHash(link.hash().toStdString()), RS_FILE_HINTS_REMOTE, finfo) ; + + for(std::vector::const_iterator it(finfo.peers.begin());it!=finfo.peers.end();++it) + { +#ifdef DEBUG_RSLINK + std::cerr << " adding peerid " << (*it).peerId << std::endl ; +#endif + srcIds.push_back((*it).peerId) ; + } + + QString cleanname = link.name() ; + static const QString bad_chars_str = "/\\\"*:?<>|" ; + + for(int i=0;ialreadyHaveFile(RsFileHash(link.hash().toStdString()), fi)) { + /* make path for downloaded file */ + std::string path; + path = fi.path;//Shared files has path with filename included + + //Seems that all FileInfo get .path==filepath+filename + //if (fi.downloadStatus == FT_STATE_COMPLETE) + // path = fi.path + "/" + fi.fname; + + QFileInfo qinfo; + qinfo.setFile(QString::fromUtf8(path.c_str())); + if (qinfo.exists() && qinfo.isFile() && !dontOpenNextFile) { + QString question = ""; + question += QObject::tr("Warning: Retroshare is about to ask your system to open this file. "); + question += QObject::tr("Before you do so, please make sure that this file does not contain malicious executable code."); + question += "

" + cleanname + ""; + + QMessageBox mb(QObject::tr("Confirmation"), question, QMessageBox::Warning, QMessageBox::Yes, QMessageBox::No, links.size()>1 ? QMessageBox::NoToAll : 0, 0); + int ret = mb.exec(); + if(ret == QMessageBox::Yes) { + ++countFileOpened; + bFileOpened = true; + /* open file with a suitable application */ + if (!RsUrlHandler::openUrl(QUrl::fromLocalFile(qinfo.absoluteFilePath()))) { + std::cerr << "RetroShareLink::process(): can't open file " << path << std::endl; + } + } else if (ret == QMessageBox::NoToAll) { + dontOpenNextFile = true; + } + } + } + + if (rsFiles->FileRequest(cleanname.toUtf8().constData(), RsFileHash(link.hash().toStdString()), link.size(), "", RS_FILE_REQ_ANONYMOUS_ROUTING, srcIds)) { + fileAdded.append(link.name()); + } else { + if (!bFileOpened) fileExist.append(link.name()); + } } break; - + case TYPE_PERSON: { #ifdef DEBUG_RSLINK @@ -1547,91 +1640,6 @@ static void processList(const QStringList &list, const QString &textSingular, co } break ; - case TYPE_EXTRAFILE: - { -#ifdef DEBUG_RSLINK - std::cerr << " RetroShareLink::process FileRequest : fileName : " << link.name().toUtf8().constData() << ". fileHash : " << link.hash().toStdString() << ". fileSize : " << link.size() << std::endl; -#endif - - needNotifySuccess = true; - std::list srcIds; - - // Add the link built-in source. This is needed for EXTRA files, where the source is specified in the link. - - if(link.type() == TYPE_EXTRAFILE) - { -#ifdef DEBUG_RSLINK - std::cerr << " RetroShareLink::process Adding built-in source " << link.SSLId().toStdString() << std::endl; -#endif - srcIds.push_back(RsPeerId(link.SSLId().toStdString())) ; - } - - // Get a list of available direct sources, in case the file is browsable only. - // - FileInfo finfo ; - rsFiles->FileDetails(RsFileHash(link.hash().toStdString()), RS_FILE_HINTS_REMOTE, finfo) ; - - for(std::vector::const_iterator it(finfo.peers.begin());it!=finfo.peers.end();++it) - { -#ifdef DEBUG_RSLINK - std::cerr << " adding peerid " << (*it).peerId << std::endl ; -#endif - srcIds.push_back((*it).peerId) ; - } - - QString cleanname = link.name() ; - static const QString bad_chars_str = "/\\\"*:?<>|" ; - - for(int i=0;ialreadyHaveFile(RsFileHash(link.hash().toStdString()), fi)) { - /* make path for downloaded file */ - std::string path; - path = fi.path;//Shared files has path with filename included - - //Seems that all FileInfo get .path==filepath+filename - //if (fi.downloadStatus == FT_STATE_COMPLETE) - // path = fi.path + "/" + fi.fname; - - QFileInfo qinfo; - qinfo.setFile(QString::fromUtf8(path.c_str())); - if (qinfo.exists() && qinfo.isFile() && !dontOpenNextFile) { - QString question = ""; - question += QObject::tr("Warning: Retroshare is about to ask your system to open this file. "); - question += QObject::tr("Before you do so, please make sure that this file does not contain malicious executable code."); - question += "

" + cleanname + ""; - - QMessageBox mb(QObject::tr("Confirmation"), question, QMessageBox::Warning, QMessageBox::Yes, QMessageBox::No, links.size()>1 ? QMessageBox::NoToAll : 0, 0); - int ret = mb.exec(); - if(ret == QMessageBox::Yes) { - ++countFileOpened; - bFileOpened = true; - /* open file with a suitable application */ - if (!RsUrlHandler::openUrl(QUrl::fromLocalFile(qinfo.absoluteFilePath()))) { - std::cerr << "RetroShareLink::process(): can't open file " << path << std::endl; - } - } else if (ret == QMessageBox::NoToAll) { - dontOpenNextFile = true; - } - } - } - - if (rsFiles->FileRequest(cleanname.toUtf8().constData(), RsFileHash(link.hash().toStdString()), link.size(), "", RS_FILE_REQ_ANONYMOUS_ROUTING, srcIds)) { - fileAdded.append(link.name()); - } else { - if (!bFileOpened) fileExist.append(link.name()); - } - } - break; - //TYPE_PRIVATE_CHAT case TYPE_PUBLIC_MSG: From 154f089dd4b1325d72af91a4e61ff6852a7bd46b Mon Sep 17 00:00:00 2001 From: RetroPooh Date: Fri, 6 Apr 2018 12:50:10 +0300 Subject: [PATCH 17/84] fixes --- retroshare-gui/src/gui/RetroShareLink.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/retroshare-gui/src/gui/RetroShareLink.cpp b/retroshare-gui/src/gui/RetroShareLink.cpp index ad149c15a..a8c20c8af 100644 --- a/retroshare-gui/src/gui/RetroShareLink.cpp +++ b/retroshare-gui/src/gui/RetroShareLink.cpp @@ -1389,7 +1389,8 @@ static void processList(const QStringList &list, const QString &textSingular, co case TYPE_FILE: { FileInfo fi1; - if(links.size()==1 && rsFiles->alreadyHaveFile(RsFileHash(link.hash().toStdString()), fi1)) + if(links.size()==1 && rsFiles->alreadyHaveFile(RsFileHash(link.hash().toStdString()), fi1) + && !link.name().endsWith(RsCollection::ExtensionString)) { /* fallthrough */ } @@ -1475,14 +1476,14 @@ static void processList(const QStringList &list, const QString &textSingular, co } else if (ret == QMessageBox::NoToAll) { dontOpenNextFile = true; } + needNotifySuccess = false; } } if (rsFiles->FileRequest(cleanname.toUtf8().constData(), RsFileHash(link.hash().toStdString()), link.size(), "", RS_FILE_REQ_ANONYMOUS_ROUTING, srcIds)) { fileAdded.append(link.name()); } else { - if (!bFileOpened) fileExist.append(link.name()); - } + if (!bFileOpened && links.size()>1) fileExist.append(link.name());} } break; @@ -1780,6 +1781,8 @@ static void processList(const QStringList &list, const QString &textSingular, co QString result; + + if (flag & (RSLINK_PROCESS_NOTIFY_SUCCESS | RSLINK_PROCESS_NOTIFY_ERROR)) { result += (links.count() == 1 ? QObject::tr("%1 of %2 RetroShare link processed.") : QObject::tr("%1 of %2 RetroShare links processed.")).arg(countProcessed).arg(links.count()) + "

"; } From dceeab0e4df9cd3eb31ab45bc403649286843b6a Mon Sep 17 00:00:00 2001 From: RetroPooh Date: Fri, 6 Apr 2018 13:02:30 +0300 Subject: [PATCH 18/84] cleanup --- retroshare-gui/src/gui/RetroShareLink.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/retroshare-gui/src/gui/RetroShareLink.cpp b/retroshare-gui/src/gui/RetroShareLink.cpp index a8c20c8af..5a0f7c7ce 100644 --- a/retroshare-gui/src/gui/RetroShareLink.cpp +++ b/retroshare-gui/src/gui/RetroShareLink.cpp @@ -1781,8 +1781,6 @@ static void processList(const QStringList &list, const QString &textSingular, co QString result; - - if (flag & (RSLINK_PROCESS_NOTIFY_SUCCESS | RSLINK_PROCESS_NOTIFY_ERROR)) { result += (links.count() == 1 ? QObject::tr("%1 of %2 RetroShare link processed.") : QObject::tr("%1 of %2 RetroShare links processed.")).arg(countProcessed).arg(links.count()) + "

"; } From a14b1f60a2b62f693d1db75356fa20cc7acd9431 Mon Sep 17 00:00:00 2001 From: RetroPooh Date: Fri, 6 Apr 2018 14:01:52 +0300 Subject: [PATCH 19/84] tooltips improv --- retroshare-gui/src/gui/RetroShareLink.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/retroshare-gui/src/gui/RetroShareLink.cpp b/retroshare-gui/src/gui/RetroShareLink.cpp index 5a0f7c7ce..ed9484f08 100644 --- a/retroshare-gui/src/gui/RetroShareLink.cpp +++ b/retroshare-gui/src/gui/RetroShareLink.cpp @@ -860,17 +860,17 @@ QString RetroShareLink::title() const break; case TYPE_FILE: - return QString("%1 (%2)").arg(hash()).arg(misc::friendlyUnit(size())); + return QString("Size: %2 hash: %1").arg(hash()).arg(misc::friendlyUnit(size())); case TYPE_PERSON: return PeerDefs::rsidFromId(RsPgpId(hash().toStdString())); case TYPE_FORUM: - /* fallthrough */ + return QString("Forum id: %1").arg(hash()); case TYPE_CHANNEL: - /* fallthrough */ + return QString("Channel id: %1").arg(hash()); case TYPE_SEARCH: - break; + return QString("Search files"); case TYPE_MESSAGE: return PeerDefs::rsidFromId(RsPeerId(hash().toStdString())); From 2b9139bf85e26792f85a653bab12f82aeedcf728 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 6 Apr 2018 15:26:54 +0200 Subject: [PATCH 20/84] improved GXS dist sync item handling --- libretroshare/src/gxs/rsgxsnetservice.cc | 11 ++- libretroshare/src/gxs/rsgxsnettunnel.cc | 89 +++++++++++++++--------- libretroshare/src/gxs/rsgxsnettunnel.h | 1 - 3 files changed, 65 insertions(+), 36 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index cd68ea6bf..8ddb992fe 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -1670,8 +1670,15 @@ RsItem *RsGxsNetService::generic_recvItem() uint32_t size = 0 ; RsGxsNetTunnelVirtualPeerId virtual_peer_id ; - if(mGxsNetTunnel->receiveData(mServType,data,size,virtual_peer_id)) - return dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; + if(mAllowDistSync && mGxsNetTunnel->receiveData(mServType,data,size,virtual_peer_id)) + { + RsItem *item = dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; + item->PeerId(virtual_peer_id) ; + + free(data) ; + + return item ; + } return NULL ; } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 65ae72111..068414099 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -129,6 +129,11 @@ RsGxsNetTunnelService::~RsGxsNetTunnelService() mGroups.clear(); mHandledHashes.clear(); mVirtualPeers.clear(); + + for(auto it(mIncomingData.begin());it!=mIncomingData.end();++it) + for(auto it2((*it).second.begin());it2!=(*it).second.end();++it2) + delete (*it2).second; + mIncomingData.clear(); } @@ -234,6 +239,7 @@ bool RsGxsNetTunnelService::requestPeers(uint16_t service_id,const RsGxsGroupId& ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE; ginfo.hash = calculateGroupHash(group_id) ; + ginfo.service_id = service_id; mHandledHashes[ginfo.hash] = group_id ; @@ -255,6 +261,7 @@ bool RsGxsNetTunnelService::releasePeers(uint16_t service_id, const RsGxsGroupId ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE; ginfo.hash = calculateGroupHash(group_id) ; + ginfo.service_id = service_id; mHandledHashes[ginfo.hash] = group_id ; // yes, we do not remove, because we're supposed to answer tunnel requests from other peers. @@ -331,6 +338,11 @@ void RsGxsNetTunnelService::dump() const std::cerr << "Hashes: " << std::endl; for(auto it(mHandledHashes.begin());it!=mHandledHashes.end();++it) std::cerr << " hash: " << it->first << " GroupId: " << it->second << std::endl; + + std::cerr << "Incoming data: " << std::endl; + for(auto it(mIncomingData.begin());it!=mIncomingData.end();++it) + for(auto it2(it->second.begin());it2!=it->second.end();++it2) + std::cerr << " service " << std::hex << it->first << std::dec << " peer id " << it2->first << " " << (void*)it2->second << std::endl; } //===========================================================================================================================================// @@ -396,13 +408,23 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co return ; } - RsItem *decrypted_item = RsGxsNetTunnelSerializer().deserialise(data,&data_size); - RsGxsNetTunnelVirtualPeerItem *pid_item = dynamic_cast(decrypted_item) ; - - if(pid_item) + if(getRsItemService(getRsItemId(data)) == RS_SERVICE_TYPE_GXS_NET_TUNNEL) { + RsItem *decrypted_item = RsGxsNetTunnelSerializer().deserialise(data,&data_size); + RsGxsNetTunnelVirtualPeerItem *pid_item = dynamic_cast(decrypted_item) ; + + if(!pid_item) + { + delete decrypted_item ; + return ; + } + + uint16_t service_id = mGroups[group_id].service_id ; + #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " item is a virtual peer id item with vpid = "<< pid_item->virtual_peer_id << ". Setting virtual peer." << std::endl; + GXS_NET_TUNNEL_DEBUG() << " item is a virtual peer id item with vpid = "<< pid_item->virtual_peer_id + << " for group " << group_id << " in service " << std::hex << service_id << std::dec + << ". Setting virtual peer." << std::endl; #endif // we receive a virtual peer id, so we need to update the local information for this peer id @@ -413,52 +435,53 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co vp_info.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; // status of the peer vp_info.side = direction; // client/server vp_info.last_contact = time(NULL); // last time some data was sent/recvd + vp_info.providing_set[service_id].provided_groups.insert(group_id); memcpy(vp_info.encryption_master_key,encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE); vp_info.turtle_virtual_peer_id = turtle_virtual_peer_id; // turtle peer to use when sending data to this vpid. free(data); - return ; } - delete decrypted_item ; - - // item is a generic data item for the client. Let's store the data in the appropriate incoming data queue. - - auto it = mTurtle2GxsPeer.find(turtle_virtual_peer_id) ; - - if(it == mTurtle2GxsPeer.end()) + else { - GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for vpid " << turtle_virtual_peer_id << " but this vpid is unknown!" << std::endl; - free(data); - return; - } + // item is a generic data item for the client. Let's store the data in the appropriate incoming data queue. - RsGxsNetTunnelVirtualPeerId gxs_vpid = it->second ; + auto it = mTurtle2GxsPeer.find(turtle_virtual_peer_id) ; - auto it2 = mVirtualPeers.find(gxs_vpid) ; + if(it == mTurtle2GxsPeer.end()) + { + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for vpid " << turtle_virtual_peer_id << " but this vpid is unknown!" << std::endl; + free(data); + return; + } - if(it2 == mVirtualPeers.end()) - { - GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for GXS vpid " << gxs_vpid << " but the virtual peer id is missing!" << std::endl; - free(data); - return; - } + RsGxsNetTunnelVirtualPeerId gxs_vpid = it->second ; - uint16_t service_id = getRsItemService(getRsItemId(data)) ; + auto it2 = mVirtualPeers.find(gxs_vpid) ; + + if(it2 == mVirtualPeers.end()) + { + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for GXS vpid " << gxs_vpid << " but the virtual peer id is missing!" << std::endl; + free(data); + return; + } + + uint16_t service_id = getRsItemService(getRsItemId(data)) ; #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << "item contains generic data for service " << std::hex << service_id << std::dec << " for VPID " << gxs_vpid << ". Storing in incoming list" << std::endl; + GXS_NET_TUNNEL_DEBUG() << "item contains generic data for service " << std::hex << service_id << std::dec << " for VPID " << gxs_vpid << ". Storing in incoming list" << std::endl; #endif - // push the data into the service incoming data list + // push the data into the service incoming data list - RsTlvBinaryData *bind = new RsTlvBinaryData; - bind->tlvtype = 0; - bind->bin_len = data_size; - bind->bin_data = data; + RsTlvBinaryData *bind = new RsTlvBinaryData; + bind->tlvtype = 0; + bind->bin_len = data_size; + bind->bin_data = data; - mIncomingData[service_id].push_back(std::make_pair(gxs_vpid,bind)) ; + mIncomingData[service_id].push_back(std::make_pair(gxs_vpid,bind)) ; + } } void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid,RsTurtleGenericTunnelItem::Direction dir) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 7543689a2..4341f9ce6 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -102,7 +102,6 @@ class RsGxsNetTunnelItem ; struct RsGxsNetTunnelVirtualPeerProvidingSet { std::set provided_groups ; - std::list incoming_data ; }; struct RsGxsNetTunnelVirtualPeerInfo From c0570ffef6a3582ec6285fbf39534250a94d879c Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 7 Apr 2018 00:56:07 +0200 Subject: [PATCH 21/84] fixed costly polling in RsGenExchange --- libretroshare/src/gxs/rsgenexchange.cc | 30 +++++++++++++++----------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 14a2471e1..11137f8b0 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -64,7 +64,7 @@ static const uint32_t INDEX_AUTHEN_ADMIN = 0x00000040; // admin key #define GXS_MASK "GXS_MASK_HACK" -//#define GEN_EXCH_DEBUG 1 +#define GEN_EXCH_DEBUG 1 static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes static const uint32_t INTEGRITY_CHECK_PERIOD = 60*31; // 31 minutes @@ -2860,8 +2860,10 @@ void RsGenExchange::processRecvdMessages() time_t now = time(NULL); + if(mMsgPendingValidate.empty()) + return ; #ifdef GEN_EXCH_DEBUG - if(!mMsgPendingValidate.empty()) + else std::cerr << "processing received messages" << std::endl; #endif // 1 - First, make sure items metadata is deserialised, clean old failed items, and collect the groups Ids we have to check @@ -2904,9 +2906,11 @@ void RsGenExchange::processRecvdMessages() } } - // 2 - Retrieve the metadata for the associated groups. + // 2 - Retrieve the metadata for the associated groups. The test is here to avoid the default behavior to + // retrieve all groups when the list is empty - mDataStore->retrieveGxsGrpMetaData(grpMetas); + if(!grpMetas.empty()) + mDataStore->retrieveGxsGrpMetaData(grpMetas); GxsMsgReq msgIds; RsNxsMsgDataTemporaryList msgs_to_store; @@ -2934,7 +2938,7 @@ void RsGenExchange::processRecvdMessages() // } #ifdef GEN_EXCH_DEBUG - std::cerr << " deserialised info: grp id=" << meta->mGroupId << ", msg id=" << meta->mMsgId ; + std::cerr << " deserialised info: grp id=" << msg->grpId << ", msg id=" << msg->msgId ; #endif std::map::iterator mit = grpMetas.find(msg->grpId); @@ -2977,8 +2981,8 @@ void RsGenExchange::processRecvdMessages() msg->metaData->recvTS = time(NULL); #ifdef GEN_EXCH_DEBUG - std::cerr << " new status flags: " << meta->mMsgStatus << std::endl; - std::cerr << " computed hash: " << meta->mHash << std::endl; + std::cerr << " new status flags: " << msg->metaData->mMsgStatus << std::endl; + std::cerr << " computed hash: " << msg->metaData->mHash << std::endl; std::cerr << "Message received. Identity=" << msg->metaData->mAuthorId << ", from peer " << msg->PeerId() << std::endl; #endif @@ -3062,9 +3066,6 @@ void RsGenExchange::processRecvdGroups() GxsPendingItem& gpsi = vit->second; RsNxsGrp* grp = gpsi.mItem; -#ifdef GEN_EXCH_DEBUG - std::cerr << " processing validation for group " << meta->mGroupId << ", original attempt time: " << time(NULL) - gpsi.mFirstTryTS << " seconds ago" << std::endl; -#endif if(grp->metaData == NULL) { RsGxsGrpMetaData* meta = new RsGxsGrpMetaData(); @@ -3074,6 +3075,9 @@ void RsGenExchange::processRecvdGroups() else delete meta ; } +#ifdef GEN_EXCH_DEBUG + std::cerr << " processing validation for group " << grp->metaData->mGroupId << ", original attempt time: " << time(NULL) - gpsi.mFirstTryTS << " seconds ago" << std::endl; +#endif // early deletion of group from the pending list if it's malformed, not accepted, or has been tried unsuccessfully for too long @@ -3102,7 +3106,7 @@ void RsGenExchange::processRecvdGroups() if(!grp->metaData->mAuthorId.isNull()) { #ifdef GEN_EXCH_DEBUG - std::cerr << "Group routage info: Identity=" << meta->mAuthorId << " from " << grp->PeerId() << std::endl; + std::cerr << "Group routage info: Identity=" << grp->metaData->mAuthorId << " from " << grp->PeerId() << std::endl; #endif mRoutingClues[grp->metaData->mAuthorId].insert(grp->PeerId()) ; } @@ -3349,7 +3353,7 @@ void RsGenExchange::removeDeleteExistingMessages( std::list& msgs, Gx const RsGxsMessageId::std_vector& msgIds = msgIdReq[(*cit2)->metaData->mGroupId]; #ifdef GEN_EXCH_DEBUG - std::cerr << " grpid=" << cit2->second->mGroupId << ", msgid=" << cit2->second->mMsgId ; + std::cerr << " grpid=" << (*cit2)->grpId << ", msgid=" << (*cit2)->msgId ; #endif // Avoid storing messages that are already in the database, as well as messages that are too old (or generally do not pass the database storage test) @@ -3369,7 +3373,7 @@ void RsGenExchange::removeDeleteExistingMessages( std::list& msgs, Gx } } #ifdef GEN_EXCH_DEBUG - std::cerr << " discarding " << cit2->second->mMsgId << std::endl; + std::cerr << " discarding " << (*cit2)->msgId << std::endl; #endif delete *cit2; From 393ff75c90c4034f43513cd87d3b696e4691f5f3 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 7 Apr 2018 00:56:44 +0200 Subject: [PATCH 22/84] fixed polling strategy in RsGxsNetService causing some delay in distant post syncing --- libretroshare/src/gxs/rsgxsnetservice.cc | 28 ++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 8ddb992fe..da9da520f 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -272,8 +272,8 @@ NXS_NET_DEBUG_8 gxs distant sync ***/ -//#define NXS_NET_DEBUG_0 1 -//#define NXS_NET_DEBUG_1 1 +#define NXS_NET_DEBUG_0 1 +#define NXS_NET_DEBUG_1 1 //#define NXS_NET_DEBUG_2 1 //#define NXS_NET_DEBUG_3 1 //#define NXS_NET_DEBUG_4 1 @@ -318,7 +318,7 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING = 0x05 ; || defined(NXS_NET_DEBUG_8) static const RsPeerId peer_to_print = RsPeerId(std::string("")) ; -static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("")) ; // use this to allow to this group id only, or "" for all IDs +static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("ff8d59ef38cad0429f34cc21749dda71")) ; // use this to allow to this group id only, or "" for all IDs static const uint32_t service_to_print = RS_SERVICE_GXS_TYPE_CHANNELS ; // use this to allow to this service id only, or 0 for all services // warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums) @@ -427,8 +427,7 @@ int RsGxsNetService::tick() { // always check for new items arriving // from peers - if(receivedItems()) - recvNxsItemQueue(); + recvNxsItemQueue(); bool should_notify = false; @@ -751,6 +750,10 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) if(!mem) return ; +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG_P_(si->PeerId()) << "Sending RsGxsNetTunnelService Item:" << (void*)si << " of type: " << std::hex << si->PacketId() << std::dec + << " transaction " << si->transactionNumber << " to virtual peer " << si->PeerId() << std::endl ; +#endif ser.serialise(si,mem,&size) ; mGxsNetTunnel->sendData(mem,size,static_cast(si->PeerId())); @@ -1670,13 +1673,20 @@ RsItem *RsGxsNetService::generic_recvItem() uint32_t size = 0 ; RsGxsNetTunnelVirtualPeerId virtual_peer_id ; - if(mAllowDistSync && mGxsNetTunnel->receiveData(mServType,data,size,virtual_peer_id)) + while(mAllowDistSync && mGxsNetTunnel->receiveData(mServType,data,size,virtual_peer_id)) { - RsItem *item = dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; + RsNxsItem *item = dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; item->PeerId(virtual_peer_id) ; free(data) ; + if(!item) + continue ; + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetTunnelService Item:" << (void*)item << " of type: " << std::hex << item->PacketId() << std::dec + << " transaction " << item->transactionNumber << " from virtual peer " << item->PeerId() << std::endl ; +#endif return item ; } @@ -1992,7 +2002,7 @@ void RsGxsNetService::debugDump() GXSNETDEBUG___<< "RsGxsNetService::debugDump():" << std::endl; - RsGxsMetaDataTemporaryMap grpMetas; + RsGxsGrpMetaTemporaryMap grpMetas; if(!group_id_to_print.isNull()) grpMetas[group_id_to_print] = NULL ; @@ -2005,7 +2015,7 @@ void RsGxsNetService::debugDump() for(ServerMsgMap::const_iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it) { - RsGxsMetaDataTemporaryMap::const_iterator it2 = grpMetas.find(it->first) ; + RsGxsGrpMetaTemporaryMap::const_iterator it2 = grpMetas.find(it->first) ; RsGxsGrpMetaData *grpMeta = (it2 != grpMetas.end())? it2->second : NULL; std::string subscribe_string = (grpMeta==NULL)?"Unknown" : ((grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED)?" Subscribed":" NOT Subscribed") ; From 66df281f25f925df47dfbecd35e4ac0ce098c223 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 11 Apr 2018 23:14:10 +0200 Subject: [PATCH 23/84] added asymmetry in GXS tunnel management to reduce the number of tunnels --- libretroshare/src/gxs/rsgxsnetservice.cc | 3 + libretroshare/src/gxs/rsgxsnettunnel.cc | 118 ++++++++++++++++++++--- libretroshare/src/gxs/rsgxsnettunnel.h | 7 +- 3 files changed, 113 insertions(+), 15 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index da9da520f..c430449d2 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -799,6 +799,9 @@ void RsGxsNetService::checkDistantSyncState() if(online_peers.find(*it2) != online_peers.end()) // check that the peer is online at_least_one_friend_is_supplier = true ; + // That strategy is likely to create islands of friends connected to each other. There's no real way + // to decide what to do here, except maybe checking the last message TS remotely vs. locally. + if(at_least_one_friend_is_supplier) { mGxsNetTunnel->releasePeers(mServType,grpId); diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 068414099..4656b1292 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -36,7 +36,11 @@ #define GXS_NET_TUNNEL_ERROR() std::cerr << "(EE) GXS_NET_TUNNEL ERROR : " -RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") {} +RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") +{ +#warning this is for testing only. In the final version this needs to be initialized with some random content. + memset(mRandomBias,0,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; +} //===========================================================================================================================================// // Transport Items // @@ -287,7 +291,7 @@ RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId() co //memcpy(mem+RsPeerId::SIZE_IN_BYTES ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; memcpy(mem+RsPeerId::SIZE_IN_BYTES /*+RsGxsGroupId::SIZE_IN_BYTES*/,mRandomBias ,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; - return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsPeerId::SIZE_IN_BYTES+RsGxsGroupId::SIZE_IN_BYTES+RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE).toByteArray()); + return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsPeerId::SIZE_IN_BYTES+/*RsGxsGroupId::SIZE_IN_BYTES+*/ RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE).toByteArray()); } void RsGxsNetTunnelService::dump() const @@ -318,7 +322,9 @@ void RsGxsNetTunnelService::dump() const for(auto it(mGroups.begin());it!=mGroups.end();++it) { std::cerr << " " << it->first << " hash: " << it->second.hash << " policy: " << group_policy_str[it->second.group_policy] << " status: " << group_status_str[it->second.group_status] << " Last contact: " << time(NULL) - it->second.last_contact << " secs ago" << std::endl; - std::cerr << " virtual peers:" << std::endl; + + if(!it->second.virtual_peers.empty()) + std::cerr << " virtual peers:" << std::endl; for(auto it2(it->second.virtual_peers.begin());it2!=it->second.virtual_peers.end();++it2) std::cerr << " " << *it2 << std::endl; } @@ -334,6 +340,9 @@ void RsGxsNetTunnelService::dump() const for(auto it2(it->second.providing_set.begin());it2!=it->second.providing_set.end();++it2) std::cerr << " service " << std::hex << it2->first << std::dec << " " << it2->second.provided_groups.size() << " groups" << std::endl; } + std::cerr << "Virtual peer turtle => GXS conversion table: " << std::endl; + for(auto it(mTurtle2GxsPeer.begin());it!=mTurtle2GxsPeer.end();++it) + std::cerr << " " << it->first << " => " << it->second << std::endl; std::cerr << "Hashes: " << std::endl; for(auto it(mHandledHashes.begin());it!=mHandledHashes.end();++it) @@ -413,7 +422,7 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co RsItem *decrypted_item = RsGxsNetTunnelSerializer().deserialise(data,&data_size); RsGxsNetTunnelVirtualPeerItem *pid_item = dynamic_cast(decrypted_item) ; - if(!pid_item) + if(!pid_item) // this handles the case of a KeepAlive packet. { delete decrypted_item ; return ; @@ -555,7 +564,7 @@ void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const if(ginfo.virtual_peers.empty()) { - ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED ; + ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE ; #ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << " no more virtual peers for group " << group_id << ": setting status to TUNNELS_REQUESTED" << std::endl; @@ -617,30 +626,115 @@ void RsGxsNetTunnelService::data_tick() { last_dump = now; dump(); + + sendKeepAlivePackets() ; } } +void RsGxsNetTunnelService::sendKeepAlivePackets() +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + RsGxsNetTunnelVirtualPeerId own_gxs_vpid = locked_makeVirtualPeerId() ; + +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " sending keep-alive packets. Own GXS peer ID is " << own_gxs_vpid << std::endl; +#endif + + // We send KA packets for each GXS virtual peer. The advantage is that unused tunnels will automatically die which eliminates duplicate tunnels + // automatically. We only send from the client side. + + for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) + if(own_gxs_vpid < it->first) + { +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " sending to virtual peer " << it->first << " through tunnel " << it->second.turtle_virtual_peer_id << std::endl; +#endif + RsGxsNetTunnelKeepAliveItem pitem ; + RsTemporaryMemory tmpmem( RsGxsNetTunnelSerializer().size(&pitem) ) ; + uint32_t len = tmpmem.size(); + + RsGxsNetTunnelSerializer().serialise(&pitem,tmpmem,&len); + + RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; + + if(p3turtle::encryptData(tmpmem,len,it->second.encryption_master_key,encrypted_turtle_item)) + mPendingTurtleItems.push_back(std::make_pair(it->second.turtle_virtual_peer_id,encrypted_turtle_item)) ; + } +#ifdef DEBUG_RSGXSNETTUNNEL + else + GXS_NET_TUNNEL_DEBUG() << " ignoring virtual peer " << it->first << std::endl; +#endif +} + void RsGxsNetTunnelService::autowash() { RS_STACK_MUTEX(mGxsNetTunnelMtx); +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " performing per-group consistency test." << std::endl; +#endif + RsGxsNetTunnelVirtualPeerId own_gxs_vpid = locked_makeVirtualPeerId() ; + for(auto it(mGroups.begin());it!=mGroups.end();++it) { + RsGxsNetTunnelGroupInfo& ginfo(it->second) ; + bool should_monitor_tunnels = false ; + +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " group " << it->first << ": " ; +#endif + // check whether the group already has GXS virtual peers with suppliers. If so, we can set the GRP policy as passive. + + if(ginfo.group_policy == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE) + { + bool found = false ; + + // check wether one virtual peer provided by GXS has ID > own ID. In this case we leave the priority to it. + + for(auto it2(ginfo.virtual_peers.begin());!found && it2!=ginfo.virtual_peers.end();++it2) + { + auto it3 = mTurtle2GxsPeer.find(*it2) ; + + if( it3 != mTurtle2GxsPeer.end() && it3->second < own_gxs_vpid) + found = true ; + } + + if(found) + { +#ifdef DEBUG_RSGXSNETTUNNEL + std::cerr << " active, with client-side peers : "; +#endif + should_monitor_tunnels = true ; + } + else + { + should_monitor_tunnels = false ; +#ifdef DEBUG_RSGXSNETTUNNEL + std::cerr << " active, and no client-side peers available : " ; +#endif + } + } +#ifdef DEBUG_RSGXSNETTUNNEL + else + std::cerr << " passive : "; +#endif + // check if the group is in active or passive mode, in which case make sure that turtle manages tunnels or not. // if active, then we check wether they are tunnels are not. If not, we ask turtle to monitor. - RsGxsNetTunnelGroupInfo& ginfo(it->second) ; - - if(ginfo.group_policy == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE && ginfo.virtual_peers.empty()) + if(should_monitor_tunnels) { +#ifdef DEBUG_RSGXSNETTUNNEL + std::cerr << " requesting tunnels" << std::endl; +#endif mTurtle->monitorTunnels(ginfo.hash,this,false) ; - ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED; } - - if(ginfo.group_policy == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE) + else { +#ifdef DEBUG_RSGXSNETTUNNEL + std::cerr << " dropping tunnels" << std::endl; +#endif mTurtle->stopMonitoringTunnels(ginfo.hash); - ginfo.group_status = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE; } } } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 4341f9ce6..7d5f6bd78 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -129,14 +129,14 @@ struct RsGxsNetTunnelGroupInfo enum GroupStatus { RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE = 0x01, // no virtual peers requested, just waiting - RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED = 0x02, // virtual peers requested, and waiting for turtle to answer +// RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED = 0x02, // virtual peers requested, and waiting for turtle to answer RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x03 // some virtual peers are available. Data can be read/written }; enum GroupPolicy { RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN = 0x00, // nothing has been set RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE = 0x01, // group is available for server side tunnels, but does not explicitely request tunnels - RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group explicitely request tunnels, if none available + RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group will explicitely request tunnels, if none available }; RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0),service_id(0) {} @@ -147,7 +147,7 @@ struct RsGxsNetTunnelGroupInfo TurtleFileHash hash ; uint16_t service_id ; - std::set virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. + std::set virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. }; class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread @@ -231,6 +231,7 @@ protected: p3turtle *mTurtle ; private: void autowash() ; + void sendKeepAlivePackets() ; void handleIncoming(RsGxsNetTunnelItem *item) ; void flush_pending_items(); From 3c9af3d2e7e46fd42d34671a53327a096e25539a Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 14 Apr 2018 11:48:55 +0200 Subject: [PATCH 24/84] fixed bug in tunnel monitoring code --- libretroshare/src/gxs/rsgxsnettunnel.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 4656b1292..4de597285 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -704,11 +704,11 @@ void RsGxsNetTunnelService::autowash() #ifdef DEBUG_RSGXSNETTUNNEL std::cerr << " active, with client-side peers : "; #endif - should_monitor_tunnels = true ; + should_monitor_tunnels = false ; } else { - should_monitor_tunnels = false ; + should_monitor_tunnels = true ; #ifdef DEBUG_RSGXSNETTUNNEL std::cerr << " active, and no client-side peers available : " ; #endif @@ -719,8 +719,9 @@ void RsGxsNetTunnelService::autowash() std::cerr << " passive : "; #endif - // check if the group is in active or passive mode, in which case make sure that turtle manages tunnels or not. - // if active, then we check wether they are tunnels are not. If not, we ask turtle to monitor. + // We should also check whether the group is supplied using another tunnel. If so, no need to request tunnels. + // Otherwise the engine will keep requesting tunnels for all groups. +#warning CODE MISSING HERE if(should_monitor_tunnels) { From 38b39caf13d67e70b1bc3cf6aad1c0e14fddfcfa Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 25 Apr 2018 14:58:53 +0200 Subject: [PATCH 25/84] added compilation flag for GXS dist sync --- libretroshare/src/gxs/rsgxsnetservice.cc | 8 ++++---- libretroshare/src/rsserver/rsinit.cc | 12 +++++++++--- retroshare.pri | 3 +++ 3 files changed, 16 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index c430449d2..cbd106c39 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -569,7 +569,7 @@ void RsGxsNetService::syncWithPeers() std::set peers; mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers); - if(mAllowDistSync) + if(mAllowDistSync && mGxsNetTunnel != NULL) { // Grab all online virtual peers of distant tunnels for the current service. @@ -740,7 +740,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) { // check if the item is to be sent to a distant peer or not - if(mAllowDistSync && mGxsNetTunnel->isDistantPeer( static_cast(si->PeerId()))) + if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer( static_cast(si->PeerId()))) { RsNxsSerialiser ser(mServType); @@ -764,7 +764,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) void RsGxsNetService::checkDistantSyncState() { - if(!mAllowDistSync) + if(!mAllowDistSync || mGxsNetTunnel==NULL) return ; RsGxsGrpMetaTemporaryMap grpMeta; @@ -1676,7 +1676,7 @@ RsItem *RsGxsNetService::generic_recvItem() uint32_t size = 0 ; RsGxsNetTunnelVirtualPeerId virtual_peer_id ; - while(mAllowDistSync && mGxsNetTunnel->receiveData(mServType,data,size,virtual_peer_id)) + while(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->receiveData(mServType,data,size,virtual_peer_id)) { RsNxsItem *item = dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; item->PeerId(virtual_peer_id) ; diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index c51abf01e..b8484ac8b 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1358,6 +1358,14 @@ int RsServer::StartupRetroShare() mWiki->setNetworkExchangeService(wiki_ns) ; #endif + /**** GXS Dist sync service ****/ +#ifdef RS_USE_GXS_DISTANT_SYNC + RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ; +#else + RsGxsNetTunnelService *mGxsNetTunnel = NULL; +#endif + + /**** Forum GXS service ****/ RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db", @@ -1371,14 +1379,12 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums, mGxsForums->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils); + pgpAuxUtils); //,mGxsNetTunnel,true,true,true); mGxsForums->setNetworkExchangeService(gxsforums_ns) ; /**** Channel GXS service ****/ - RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ; - RsGeneralDataService* gxschannels_ds = new RsDataService(currGxsDir + "/", "gxschannels_db", RS_SERVICE_GXS_TYPE_CHANNELS, NULL, rsInitConfig->gxs_passwd); diff --git a/retroshare.pri b/retroshare.pri index 446c1bd61..21c469aa2 100644 --- a/retroshare.pri +++ b/retroshare.pri @@ -9,6 +9,8 @@ no_retroshare_gui:CONFIG -= retroshare_gui # CONFIG *= retrotor +CONFIG *= gxsdistsync + # To disable RetroShare-nogui append the following # assignation to qmake command line "CONFIG+=no_retroshare_nogui" CONFIG *= retroshare_nogui @@ -214,6 +216,7 @@ unfinished { CONFIG += wikipoos } +gxsdistsync:DEFINES *= RS_USE_GXS_DISTANT_SYNC wikipoos:DEFINES *= RS_USE_WIKI rs_gxs:DEFINES *= RS_ENABLE_GXS libresapilocalserver:DEFINES *= LIBRESAPI_LOCAL_SERVER From 1a9a7622a2e3cfb967de4c7511c5e2b062a1eeb3 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 25 Apr 2018 17:58:37 +0200 Subject: [PATCH 26/84] made GxsNetTunnelService a parent class of GxsNetService and renamed public methods appropriately. --- libretroshare/src/gxs/rsgxsnetservice.cc | 27 ++++++++++++++---------- libretroshare/src/gxs/rsgxsnetservice.h | 5 ++--- libretroshare/src/gxs/rsgxsnettunnel.cc | 12 +++++------ libretroshare/src/gxs/rsgxsnettunnel.h | 13 ++++++------ libretroshare/src/rsserver/rsinit.cc | 17 ++++----------- 5 files changed, 34 insertions(+), 40 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index cbd106c39..ad8fa4c73 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -361,7 +361,7 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, const RsServiceInfo serviceInfo, RsGixsReputation* reputations, RsGcxs* circles, RsGixs *gixs, - PgpAuxUtils *pgpUtils, RsGxsNetTunnelService *mGxsNT, + PgpAuxUtils *pgpUtils, bool grpAutoSync, bool msgAutoSync, bool distSync, uint32_t default_store_period, uint32_t default_sync_period) : p3ThreadedService(), p3Config(), mTransactionN(0), mObserver(nxsObs), mDataStore(gds), @@ -370,7 +370,7 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, mSyncTs(0), mLastKeyPublishTs(0), mLastCleanRejectedMessages(0), mSYNC_PERIOD(SYNC_PERIOD), mCircles(circles), mGixs(gixs), - mReputations(reputations), mPgpUtils(pgpUtils),mGxsNetTunnel(mGxsNT), + mReputations(reputations), mPgpUtils(pgpUtils), mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync),mAllowDistSync(distSync), mServiceInfo(serviceInfo), mDefaultMsgStorePeriod(default_store_period), mDefaultMsgSyncPeriod(default_sync_period) @@ -569,12 +569,12 @@ void RsGxsNetService::syncWithPeers() std::set peers; mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers); - if(mAllowDistSync && mGxsNetTunnel != NULL) + if(mAllowDistSync) { // Grab all online virtual peers of distant tunnels for the current service. std::list vpids ; - mGxsNetTunnel->getVirtualPeers(mServType,vpids); + getVirtualPeers(mServType,vpids); for(auto it(vpids.begin());it!=vpids.end();++it) peers.insert(RsPeerId(*it)) ; @@ -668,6 +668,7 @@ void RsGxsNetService::syncWithPeers() const RsGxsGroupId& grpId = mmit->first; RsGxsCircleId encrypt_to_this_circle_id ; +#warning we should use this call in order to determine wether the peer can be sent group information about a specific group, otherwise we leak which group we are subscribed to if(!checkCanRecvMsgFromPeer(peerId, *meta,encrypt_to_this_circle_id)) continue; @@ -725,7 +726,7 @@ void RsGxsNetService::syncWithPeers() #ifdef NXS_NET_DEBUG_7 GXSNETDEBUG_PG(*sit,grpId) << " Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself - in clear " << std::endl; #endif - generic_sendItem(msg); + generic_sendItem(msg); #ifdef NXS_NET_DEBUG_5 GXSNETDEBUG_PG(*sit,grpId) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself" << std::endl; @@ -740,7 +741,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) { // check if the item is to be sent to a distant peer or not - if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer( static_cast(si->PeerId()))) + if(mAllowDistSync && isDistantPeer( static_cast(si->PeerId()))) { RsNxsSerialiser ser(mServType); @@ -756,7 +757,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) #endif ser.serialise(si,mem,&size) ; - mGxsNetTunnel->sendData(mem,size,static_cast(si->PeerId())); + sendTunnelData(mem,size,static_cast(si->PeerId())); } else sendItem(si) ; @@ -764,7 +765,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) void RsGxsNetService::checkDistantSyncState() { - if(!mAllowDistSync || mGxsNetTunnel==NULL) + if(!mAllowDistSync) return ; RsGxsGrpMetaTemporaryMap grpMeta; @@ -804,14 +805,14 @@ void RsGxsNetService::checkDistantSyncState() if(at_least_one_friend_is_supplier) { - mGxsNetTunnel->releasePeers(mServType,grpId); + releaseDistantPeers(mServType,grpId); #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___<< " Group " << grpId << ": suppliers among friends. Releasing peers." << std::endl; #endif } else { - mGxsNetTunnel->requestPeers(mServType,grpId); + requestDistantPeers(mServType,grpId); #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___<< " Group " << grpId << ": no suppliers among friends. Requesting peers." << std::endl; #endif @@ -1676,7 +1677,7 @@ RsItem *RsGxsNetService::generic_recvItem() uint32_t size = 0 ; RsGxsNetTunnelVirtualPeerId virtual_peer_id ; - while(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->receiveData(mServType,data,size,virtual_peer_id)) + while(mAllowDistSync && receiveTunnelData(mServType,data,size,virtual_peer_id)) { RsNxsItem *item = dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; item->PeerId(virtual_peer_id) ; @@ -1995,6 +1996,10 @@ void RsGxsNetService::data_tick() runVetting(); processExplicitGroupRequests(); + + // also tick distant traffic + + RsGxsNetTunnelService::data_tick(); } void RsGxsNetService::debugDump() diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index aac6c474a..04c0317a6 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -72,7 +72,7 @@ class RsGroupNetworkStatsRecord * Incoming transaction are in 3 different states * 1. START 2. RECEIVING 3. END */ -class RsGxsNetService : public RsNetworkExchangeService, public p3ThreadedService, public p3Config +class RsGxsNetService : public RsNetworkExchangeService, public RsGxsNetTunnelService, public p3ThreadedService, public p3Config { public: @@ -90,7 +90,7 @@ public: RsNxsObserver *nxsObs, // used to be = NULL. const RsServiceInfo serviceInfo, RsGixsReputation* reputations = NULL, RsGcxs* circles = NULL, RsGixs *gixs=NULL, - PgpAuxUtils *pgpUtils = NULL, RsGxsNetTunnelService *mGxsNT = NULL, + PgpAuxUtils *pgpUtils = NULL, bool grpAutoSync = true, bool msgAutoSync = true,bool distSync=false, uint32_t default_store_period = RS_GXS_DEFAULT_MSG_STORE_PERIOD, uint32_t default_sync_period = RS_GXS_DEFAULT_MSG_REQ_PERIOD); @@ -546,7 +546,6 @@ private: RsGixs *mGixs; RsGixsReputation* mReputations; PgpAuxUtils *mPgpUtils; - RsGxsNetTunnelService *mGxsNetTunnel; bool mGrpAutoSync; bool mAllowMsgSync; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 4de597285..f37195936 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -38,7 +38,7 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") { -#warning this is for testing only. In the final version this needs to be initialized with some random content. +#warning this is for testing only. In the final version this needs to be initialized with some random content, saved and re-used for a while (e.g. 1 month) memset(mRandomBias,0,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; } @@ -148,7 +148,7 @@ bool RsGxsNetTunnelService::isDistantPeer(const RsGxsNetTunnelVirtualPeerId& vir return mVirtualPeers.find(virtual_peer) != mVirtualPeers.end(); } -bool RsGxsNetTunnelService::receiveData(uint16_t service_id,unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) +bool RsGxsNetTunnelService::receiveTunnelData(uint16_t service_id,unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -174,7 +174,7 @@ bool RsGxsNetTunnelService::receiveData(uint16_t service_id,unsigned char *& dat return true; } -bool RsGxsNetTunnelService::sendData(unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) +bool RsGxsNetTunnelService::sendTunnelData(unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) { RS_STACK_MUTEX(mGxsNetTunnelMtx); // The item is serialized and encrypted using chacha20+SHA256, using the generic turtle encryption, and then sent to the turtle router. @@ -233,7 +233,7 @@ bool RsGxsNetTunnelService::getVirtualPeers(uint16_t service_id, std::list virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. }; -class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread +class RsGxsNetTunnelService: public RsTurtleClientService { public: RsGxsNetTunnelService() ; @@ -160,13 +161,13 @@ public: * \brief Manage tunnels for this group * @param group_id group for which tunnels should be released */ - bool requestPeers(uint16_t service_id, const RsGxsGroupId&group_id) ; + bool requestDistantPeers(uint16_t service_id, const RsGxsGroupId&group_id) ; /*! * \brief Stop managing tunnels for this group * @param group_id group for which tunnels should be released */ - bool releasePeers(uint16_t service_id,const RsGxsGroupId&group_id) ; + bool releaseDistantPeers(uint16_t service_id,const RsGxsGroupId&group_id) ; /*! * \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and @@ -182,7 +183,7 @@ public: * \return * true if succeeded. */ - bool sendData(unsigned char *& data, uint32_t data_len, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + bool sendTunnelData(unsigned char *& data, uint32_t data_len, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! * \brief receiveData @@ -194,7 +195,7 @@ public: * \return * true if something is returned. If not, data is set to NULL, data_len to 0. */ - bool receiveData(uint16_t service_id,unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + bool receiveTunnelData(uint16_t service_id,unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! * \brief isDistantPeer diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index b8484ac8b..12d60061e 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1358,14 +1358,6 @@ int RsServer::StartupRetroShare() mWiki->setNetworkExchangeService(wiki_ns) ; #endif - /**** GXS Dist sync service ****/ -#ifdef RS_USE_GXS_DISTANT_SYNC - RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ; -#else - RsGxsNetTunnelService *mGxsNetTunnel = NULL; -#endif - - /**** Forum GXS service ****/ RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db", @@ -1379,7 +1371,7 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums, mGxsForums->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils); //,mGxsNetTunnel,true,true,true); + pgpAuxUtils); mGxsForums->setNetworkExchangeService(gxsforums_ns) ; @@ -1395,7 +1387,7 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr, mGxsChannels, mGxsChannels->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils,mGxsNetTunnel,true,true,true); + pgpAuxUtils,true,true,true); mGxsChannels->setNetworkExchangeService(gxschannels_ns) ; @@ -1450,7 +1442,7 @@ int RsServer::StartupRetroShare() RsGxsNetService* gxstrans_ns = new RsGxsNetService( RS_SERVICE_TYPE_GXS_TRANS, gxstrans_ds, nxsMgr, mGxsTrans, mGxsTrans->getServiceInfo(), mReputations, mGxsCircles, - mGxsIdService, pgpAuxUtils,NULL,true,true,false,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD); + mGxsIdService, pgpAuxUtils,true,true,false,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD); mGxsTrans->setNetworkExchangeService(gxstrans_ns); pqih->addService(gxstrans_ns, true); @@ -1492,7 +1484,7 @@ int RsServer::StartupRetroShare() // connect components to turtle router. - mGxsNetTunnel->connectToTurtleRouter(tr) ; + gxschannels_ns->connectToTurtleRouter(tr) ; ftserver->connectToTurtleRouter(tr) ; ftserver->connectToFileDatabase(fdb) ; chatSrv->connectToGxsTunnelService(mGxsTunnels) ; @@ -1831,7 +1823,6 @@ int RsServer::StartupRetroShare() //rsWire = mWire; /*** start up GXS core runner ***/ - startServiceThread(mGxsNetTunnel, "gxs net tunnel"); startServiceThread(mGxsIdService, "gxs id"); startServiceThread(mGxsCircles, "gxs circle"); From da4b382edec3386e8a27c8255974c8a2becad3e2 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 25 Apr 2018 18:29:49 +0200 Subject: [PATCH 27/84] removed dependency on service in RsGxsNetTunnel --- libretroshare/src/gxs/rsgxsnetservice.cc | 8 +-- libretroshare/src/gxs/rsgxsnettunnel.cc | 76 ++++++++++-------------- libretroshare/src/gxs/rsgxsnettunnel.h | 27 ++++----- 3 files changed, 46 insertions(+), 65 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index ad8fa4c73..619adb5f7 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -574,7 +574,7 @@ void RsGxsNetService::syncWithPeers() // Grab all online virtual peers of distant tunnels for the current service. std::list vpids ; - getVirtualPeers(mServType,vpids); + getVirtualPeers(vpids); for(auto it(vpids.begin());it!=vpids.end();++it) peers.insert(RsPeerId(*it)) ; @@ -805,14 +805,14 @@ void RsGxsNetService::checkDistantSyncState() if(at_least_one_friend_is_supplier) { - releaseDistantPeers(mServType,grpId); + releaseDistantPeers(grpId); #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___<< " Group " << grpId << ": suppliers among friends. Releasing peers." << std::endl; #endif } else { - requestDistantPeers(mServType,grpId); + requestDistantPeers(grpId); #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___<< " Group " << grpId << ": no suppliers among friends. Requesting peers." << std::endl; #endif @@ -1677,7 +1677,7 @@ RsItem *RsGxsNetService::generic_recvItem() uint32_t size = 0 ; RsGxsNetTunnelVirtualPeerId virtual_peer_id ; - while(mAllowDistSync && receiveTunnelData(mServType,data,size,virtual_peer_id)) + while(mAllowDistSync && receiveTunnelData(data,size,virtual_peer_id)) { RsNxsItem *item = dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; item->PeerId(virtual_peer_id) ; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index f37195936..e3da0be70 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -135,8 +135,7 @@ RsGxsNetTunnelService::~RsGxsNetTunnelService() mVirtualPeers.clear(); for(auto it(mIncomingData.begin());it!=mIncomingData.end();++it) - for(auto it2((*it).second.begin());it2!=(*it).second.end();++it2) - delete (*it2).second; + delete (*it).second; mIncomingData.clear(); } @@ -148,11 +147,11 @@ bool RsGxsNetTunnelService::isDistantPeer(const RsGxsNetTunnelVirtualPeerId& vir return mVirtualPeers.find(virtual_peer) != mVirtualPeers.end(); } -bool RsGxsNetTunnelService::receiveTunnelData(uint16_t service_id,unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) +bool RsGxsNetTunnelService::receiveTunnelData(unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) { RS_STACK_MUTEX(mGxsNetTunnelMtx); - std::list >& lst(mIncomingData[service_id]) ; + std::list >& lst(mIncomingData); if(lst.empty()) { @@ -212,7 +211,7 @@ bool RsGxsNetTunnelService::sendTunnelData(unsigned char *& data,uint32_t data_l return true ; } -bool RsGxsNetTunnelService::getVirtualPeers(uint16_t service_id, std::list& peers) +bool RsGxsNetTunnelService::getVirtualPeers(std::list& peers) { // This function has two effects: // - return the virtual peers for this group @@ -223,17 +222,16 @@ bool RsGxsNetTunnelService::getVirtualPeers(uint16_t service_id, std::listsecond.providing_set.find(service_id) != it->second.providing_set.end()) - peers.push_back(it->first) ; + peers.push_back(it->first) ; #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " service " << std::hex << service_id << std::dec << " returning " << peers.size() << " peers." << std::endl; + GXS_NET_TUNNEL_DEBUG() << " returning " << peers.size() << " peers." << std::endl; #endif return true ; } -bool RsGxsNetTunnelService::requestDistantPeers(uint16_t service_id,const RsGxsGroupId& group_id) +bool RsGxsNetTunnelService::requestDistantPeers(const RsGxsGroupId& group_id) { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -243,19 +241,18 @@ bool RsGxsNetTunnelService::requestDistantPeers(uint16_t service_id,const RsGxsG ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE; ginfo.hash = calculateGroupHash(group_id) ; - ginfo.service_id = service_id; mHandledHashes[ginfo.hash] = group_id ; // we dont set the group policy here. It will only be set if no peers, or too few peers are available. #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " service " << std::hex << service_id << std::dec << " requesting peers for group " << group_id << std::endl; + GXS_NET_TUNNEL_DEBUG() << " requesting peers for group " << group_id << std::endl; #endif return true; } -bool RsGxsNetTunnelService::releaseDistantPeers(uint16_t service_id, const RsGxsGroupId& group_id) +bool RsGxsNetTunnelService::releaseDistantPeers(const RsGxsGroupId& group_id) { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -265,33 +262,31 @@ bool RsGxsNetTunnelService::releaseDistantPeers(uint16_t service_id, const RsGxs ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE; ginfo.hash = calculateGroupHash(group_id) ; - ginfo.service_id = service_id; mHandledHashes[ginfo.hash] = group_id ; // yes, we do not remove, because we're supposed to answer tunnel requests from other peers. mTurtle->stopMonitoringTunnels(ginfo.hash) ; #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " service " << std::hex << service_id << std::dec << " releasing peers for group " << group_id << std::endl; + GXS_NET_TUNNEL_DEBUG() << " releasing peers for group " << group_id << std::endl; #endif return true; } -RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId() const +RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId(const RsGxsGroupId& group_id) const { - assert(RsPeerId::SIZE_IN_BYTES <= Sha1CheckSum::SIZE_IN_BYTES) ; + assert(RsPeerId::SIZE_IN_BYTES <= Sha1CheckSum::SIZE_IN_BYTES) ;// so that we can build the virtual PeerId from a SHA1 sum. // We compute sha1( SSL_id | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId RsPeerId ssl_id = rsPeers->getOwnId() ; - unsigned char mem[RsPeerId::SIZE_IN_BYTES /*+ RsGxsGroupId::SIZE_IN_BYTES */ + RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE]; + unsigned char mem[RsGxsGroupId::SIZE_IN_BYTES + RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE]; - memcpy(mem ,ssl_id.toByteArray() ,RsPeerId::SIZE_IN_BYTES) ; - //memcpy(mem+RsPeerId::SIZE_IN_BYTES ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; - memcpy(mem+RsPeerId::SIZE_IN_BYTES /*+RsGxsGroupId::SIZE_IN_BYTES*/,mRandomBias ,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; + memcpy(mem ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; + memcpy(mem+RsGxsGroupId::SIZE_IN_BYTES,mRandomBias ,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; - return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsPeerId::SIZE_IN_BYTES+/*RsGxsGroupId::SIZE_IN_BYTES+*/ RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE).toByteArray()); + return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsGxsGroupId::SIZE_IN_BYTES+RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE).toByteArray()); } void RsGxsNetTunnelService::dump() const @@ -331,15 +326,13 @@ void RsGxsNetTunnelService::dump() const std::cerr << "Virtual peers: " << std::endl; for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) - { std::cerr << " GXS Peer:" << it->first << " Turtle:" << it->second.turtle_virtual_peer_id << " status: " << vpid_status_str[it->second.vpid_status] << " direction: " - << (int)it->second.side << " last seen " << time(NULL)-it->second.last_contact + << " group_id: " << it->second.group_id + << " direction: " << (int)it->second.side + << " last seen " << time(NULL)-it->second.last_contact << " ekey: " << RsUtil::BinToHex(it->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE,10) << std::endl; - for(auto it2(it->second.providing_set.begin());it2!=it->second.providing_set.end();++it2) - std::cerr << " service " << std::hex << it2->first << std::dec << " " << it2->second.provided_groups.size() << " groups" << std::endl; - } std::cerr << "Virtual peer turtle => GXS conversion table: " << std::endl; for(auto it(mTurtle2GxsPeer.begin());it!=mTurtle2GxsPeer.end();++it) std::cerr << " " << it->first << " => " << it->second << std::endl; @@ -350,8 +343,7 @@ void RsGxsNetTunnelService::dump() const std::cerr << "Incoming data: " << std::endl; for(auto it(mIncomingData.begin());it!=mIncomingData.end();++it) - for(auto it2(it->second.begin());it2!=it->second.end();++it2) - std::cerr << " service " << std::hex << it->first << std::dec << " peer id " << it2->first << " " << (void*)it2->second << std::endl; + std::cerr << " peer id " << it->first << " " << (void*)it->second << std::endl; } //===========================================================================================================================================// @@ -428,12 +420,9 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co return ; } - uint16_t service_id = mGroups[group_id].service_id ; - #ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << " item is a virtual peer id item with vpid = "<< pid_item->virtual_peer_id - << " for group " << group_id << " in service " << std::hex << service_id << std::dec - << ". Setting virtual peer." << std::endl; + << " for group " << group_id << ". Setting virtual peer." << std::endl; #endif // we receive a virtual peer id, so we need to update the local information for this peer id @@ -444,7 +433,7 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co vp_info.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; // status of the peer vp_info.side = direction; // client/server vp_info.last_contact = time(NULL); // last time some data was sent/recvd - vp_info.providing_set[service_id].provided_groups.insert(group_id); + vp_info.group_id = group_id; memcpy(vp_info.encryption_master_key,encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE); @@ -476,20 +465,18 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co return; } - uint16_t service_id = getRsItemService(getRsItemId(data)) ; - #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << "item contains generic data for service " << std::hex << service_id << std::dec << " for VPID " << gxs_vpid << ". Storing in incoming list" << std::endl; + GXS_NET_TUNNEL_DEBUG() << "item contains generic data for VPID " << gxs_vpid << ". Storing in incoming list" << std::endl; #endif - // push the data into the service incoming data list + // push the data into the incoming data list RsTlvBinaryData *bind = new RsTlvBinaryData; bind->tlvtype = 0; bind->bin_len = data_size; bind->bin_data = data; - mIncomingData[service_id].push_back(std::make_pair(gxs_vpid,bind)) ; + mIncomingData.push_back(std::make_pair(gxs_vpid,bind)) ; } } @@ -520,7 +507,7 @@ void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const Tur // We need to send our own virtual peer id to the other end of the tunnel - RsGxsNetTunnelVirtualPeerId net_service_virtual_peer = locked_makeVirtualPeerId(); + RsGxsNetTunnelVirtualPeerId net_service_virtual_peer = locked_makeVirtualPeerId(group_id); #ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << " sending back virtual peer name " << net_service_virtual_peer << " to end of tunnel" << std::endl; @@ -632,16 +619,17 @@ void RsGxsNetTunnelService::data_tick() void RsGxsNetTunnelService::sendKeepAlivePackets() { RS_STACK_MUTEX(mGxsNetTunnelMtx); - RsGxsNetTunnelVirtualPeerId own_gxs_vpid = locked_makeVirtualPeerId() ; - #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " sending keep-alive packets. Own GXS peer ID is " << own_gxs_vpid << std::endl; + GXS_NET_TUNNEL_DEBUG() << " sending keep-alive packets. " << std::endl; #endif // We send KA packets for each GXS virtual peer. The advantage is that unused tunnels will automatically die which eliminates duplicate tunnels // automatically. We only send from the client side. for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) + { + RsGxsNetTunnelVirtualPeerId own_gxs_vpid = locked_makeVirtualPeerId(it->second.group_id) ; + if(own_gxs_vpid < it->first) { #ifdef DEBUG_RSGXSNETTUNNEL @@ -662,6 +650,7 @@ void RsGxsNetTunnelService::sendKeepAlivePackets() else GXS_NET_TUNNEL_DEBUG() << " ignoring virtual peer " << it->first << std::endl; #endif + } } void RsGxsNetTunnelService::autowash() @@ -671,10 +660,9 @@ void RsGxsNetTunnelService::autowash() #ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << " performing per-group consistency test." << std::endl; #endif - RsGxsNetTunnelVirtualPeerId own_gxs_vpid = locked_makeVirtualPeerId() ; - for(auto it(mGroups.begin());it!=mGroups.end();++it) { + RsGxsNetTunnelVirtualPeerId own_gxs_vpid = locked_makeVirtualPeerId(it->first) ; RsGxsNetTunnelGroupInfo& ginfo(it->second) ; bool should_monitor_tunnels = false ; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 4826729f6..c63ab0135 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -100,11 +100,6 @@ typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; class RsGxsNetTunnelItem ; -struct RsGxsNetTunnelVirtualPeerProvidingSet -{ - std::set provided_groups ; -}; - struct RsGxsNetTunnelVirtualPeerInfo { enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. @@ -120,9 +115,9 @@ struct RsGxsNetTunnelVirtualPeerInfo uint8_t side ; // client/server uint8_t encryption_master_key[32]; - TurtleVirtualPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. + TurtleVirtualPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. - std::map providing_set; // partial list of groups provided by this virtual peer id, based on tunnel results, for each service + RsGxsGroupId group_id ; // group that virtual peer is providing }; struct RsGxsNetTunnelGroupInfo @@ -140,13 +135,12 @@ struct RsGxsNetTunnelGroupInfo RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group will explicitely request tunnels, if none available }; - RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0),service_id(0) {} + RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {} GroupPolicy group_policy ; GroupStatus group_status ; time_t last_contact ; TurtleFileHash hash ; - uint16_t service_id ; std::set virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. }; @@ -161,19 +155,19 @@ public: * \brief Manage tunnels for this group * @param group_id group for which tunnels should be released */ - bool requestDistantPeers(uint16_t service_id, const RsGxsGroupId&group_id) ; + bool requestDistantPeers(const RsGxsGroupId&group_id) ; /*! * \brief Stop managing tunnels for this group * @param group_id group for which tunnels should be released */ - bool releaseDistantPeers(uint16_t service_id,const RsGxsGroupId&group_id) ; + bool releaseDistantPeers(const RsGxsGroupId&group_id) ; /*! * \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and * alive. This function also "registers" the group which allows to handle tunnel requests in the server side. */ - bool getVirtualPeers(uint16_t service_id, std::list& peers) ; // returns the virtual peers for this group + bool getVirtualPeers(std::list& peers) ; // returns the virtual peers for this group /*! * \brief sendData @@ -187,15 +181,14 @@ public: /*! * \brief receiveData - * returns the next piece of data received fro the given service, and the virtual GXS peer that sended it. - * \param service_id service that provide the data + * returns the next piece of data received, and the virtual GXS peer that sended it. * \param data memory check containing the data. Memory ownership belongs to the client. * \param data_len length of memory chunk * \param virtual_peer peer who sent the data * \return * true if something is returned. If not, data is set to NULL, data_len to 0. */ - bool receiveTunnelData(uint16_t service_id,unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + bool receiveTunnelData(unsigned char *& data, uint32_t& data_len, RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! * \brief isDistantPeer @@ -247,7 +240,7 @@ private: std::list > mPendingTurtleItems ; // items that need to be sent off-turtle Mutex. - std::map > > mIncomingData; // list of incoming data items, per service. + std::list > mIncomingData; // list of incoming data items /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to @@ -261,7 +254,7 @@ private: * tunnel ID and turtle virtual peer id. This allows RsGxsNetService to keep sync-ing the data consistently. */ - RsGxsNetTunnelVirtualPeerId locked_makeVirtualPeerId() const ; + RsGxsNetTunnelVirtualPeerId locked_makeVirtualPeerId(const RsGxsGroupId& group_id) const ; static void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) ; From 57bb31ece62fcbba4a18313fbad7c594e913d70d Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 25 Apr 2018 18:52:07 +0200 Subject: [PATCH 28/84] added new checks in canSend and canRecv Msg/Grp to work with distant peers --- libretroshare/src/gxs/rsgxsnetservice.cc | 37 ++++++++++++++++++++++-- libretroshare/src/gxs/rsgxsnettunnel.cc | 12 ++++++-- libretroshare/src/gxs/rsgxsnettunnel.h | 3 +- 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 619adb5f7..33081acd2 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -668,7 +668,6 @@ void RsGxsNetService::syncWithPeers() const RsGxsGroupId& grpId = mmit->first; RsGxsCircleId encrypt_to_this_circle_id ; -#warning we should use this call in order to determine wether the peer can be sent group information about a specific group, otherwise we leak which group we are subscribed to if(!checkCanRecvMsgFromPeer(peerId, *meta,encrypt_to_this_circle_id)) continue; @@ -741,7 +740,9 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) { // check if the item is to be sent to a distant peer or not - if(mAllowDistSync && isDistantPeer( static_cast(si->PeerId()))) + RsGxsGroupId tmp_grpId; + + if(mAllowDistSync && isDistantPeer( static_cast(si->PeerId()),tmp_grpId)) { RsNxsSerialiser ser(mServType); @@ -4091,6 +4092,17 @@ bool RsGxsNetService::canSendGrpId(const RsPeerId& sslId, const RsGxsGrpMetaData #ifdef NXS_NET_DEBUG_4 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId()"<< std::endl; #endif + // check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for + + RsGxsGroupId peer_grp ; + if(isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl; +#endif + return false ; + } + // first do the simple checks uint8_t circleType = grpMeta.mCircleType; @@ -4144,6 +4156,17 @@ bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxs GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::checkCanRecvMsgFromPeer()"; GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " peer Id = " << sslId << ", grpId=" << grpMeta.mGroupId <& msgMetas, co #ifdef NXS_NET_DEBUG_4 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendMsgIds() CIRCLE VETTING" << std::endl; #endif + // check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for + + RsGxsGroupId peer_grp ; + if(isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) + { +#ifdef NXS_NET_DEBUG_4 + GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl; +#endif + return false ; + } // first do the simple checks uint8_t circleType = grpMeta.mCircleType; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index e3da0be70..00e3fd852 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -140,11 +140,19 @@ RsGxsNetTunnelService::~RsGxsNetTunnelService() mIncomingData.clear(); } -bool RsGxsNetTunnelService::isDistantPeer(const RsGxsNetTunnelVirtualPeerId& virtual_peer) +bool RsGxsNetTunnelService::isDistantPeer(const RsGxsNetTunnelVirtualPeerId& virtual_peer, RsGxsGroupId& group_id) { RS_STACK_MUTEX(mGxsNetTunnelMtx); - return mVirtualPeers.find(virtual_peer) != mVirtualPeers.end(); + auto it = mVirtualPeers.find(virtual_peer) ; + + if(it != mVirtualPeers.end()) + { + group_id = it->second.group_id ; + return true ; + } + else + return false ; } bool RsGxsNetTunnelService::receiveTunnelData(unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index c63ab0135..9448b9508 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -193,10 +193,11 @@ public: /*! * \brief isDistantPeer * returns wether the peer is in the list of available distant peers or not + * \param group_id returned by the service to indicate which group this peer id is designed for. * \return true if the peer is a distant GXS peer. */ - bool isDistantPeer(const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + bool isDistantPeer(const RsGxsNetTunnelVirtualPeerId& virtual_peer,RsGxsGroupId& group_id) ; /*! * \brief dumps all information about monitored groups. From 606537a0923fec8af3db0a83f1f78d661fa59b05 Mon Sep 17 00:00:00 2001 From: RetroPooh Date: Thu, 26 Apr 2018 23:28:14 +0300 Subject: [PATCH 29/84] reorder cases for nicer diff --- retroshare-gui/src/gui/RetroShareLink.cpp | 202 +++++++++++----------- 1 file changed, 101 insertions(+), 101 deletions(-) diff --git a/retroshare-gui/src/gui/RetroShareLink.cpp b/retroshare-gui/src/gui/RetroShareLink.cpp index ed9484f08..744826bbb 100644 --- a/retroshare-gui/src/gui/RetroShareLink.cpp +++ b/retroshare-gui/src/gui/RetroShareLink.cpp @@ -1386,107 +1386,6 @@ static void processList(const QStringList &list, const QString &textSingular, co ++countUnknown; break; - case TYPE_FILE: - { - FileInfo fi1; - if(links.size()==1 && rsFiles->alreadyHaveFile(RsFileHash(link.hash().toStdString()), fi1) - && !link.name().endsWith(RsCollection::ExtensionString)) - { - /* fallthrough */ - } - else - { - col.merge_in(link.name(),link.size(),RsFileHash(link.hash().toStdString())) ; - fileLinkFound = true; - break; - } - } - //break; - case TYPE_EXTRAFILE: - { -#ifdef DEBUG_RSLINK - std::cerr << " RetroShareLink::process FileRequest : fileName : " << link.name().toUtf8().constData() << ". fileHash : " << link.hash().toStdString() << ". fileSize : " << link.size() << std::endl; -#endif - - needNotifySuccess = true; - std::list srcIds; - - // Add the link built-in source. This is needed for EXTRA files, where the source is specified in the link. - - if(link.type() == TYPE_EXTRAFILE) - { -#ifdef DEBUG_RSLINK - std::cerr << " RetroShareLink::process Adding built-in source " << link.SSLId().toStdString() << std::endl; -#endif - srcIds.push_back(RsPeerId(link.SSLId().toStdString())) ; - } - - // Get a list of available direct sources, in case the file is browsable only. - // - FileInfo finfo ; - rsFiles->FileDetails(RsFileHash(link.hash().toStdString()), RS_FILE_HINTS_REMOTE, finfo) ; - - for(std::vector::const_iterator it(finfo.peers.begin());it!=finfo.peers.end();++it) - { -#ifdef DEBUG_RSLINK - std::cerr << " adding peerid " << (*it).peerId << std::endl ; -#endif - srcIds.push_back((*it).peerId) ; - } - - QString cleanname = link.name() ; - static const QString bad_chars_str = "/\\\"*:?<>|" ; - - for(int i=0;ialreadyHaveFile(RsFileHash(link.hash().toStdString()), fi)) { - /* make path for downloaded file */ - std::string path; - path = fi.path;//Shared files has path with filename included - - //Seems that all FileInfo get .path==filepath+filename - //if (fi.downloadStatus == FT_STATE_COMPLETE) - // path = fi.path + "/" + fi.fname; - - QFileInfo qinfo; - qinfo.setFile(QString::fromUtf8(path.c_str())); - if (qinfo.exists() && qinfo.isFile() && !dontOpenNextFile) { - QString question = ""; - question += QObject::tr("Warning: Retroshare is about to ask your system to open this file. "); - question += QObject::tr("Before you do so, please make sure that this file does not contain malicious executable code."); - question += "

" + cleanname + ""; - - QMessageBox mb(QObject::tr("Confirmation"), question, QMessageBox::Warning, QMessageBox::Yes, QMessageBox::No, links.size()>1 ? QMessageBox::NoToAll : 0, 0); - int ret = mb.exec(); - if(ret == QMessageBox::Yes) { - ++countFileOpened; - bFileOpened = true; - /* open file with a suitable application */ - if (!RsUrlHandler::openUrl(QUrl::fromLocalFile(qinfo.absoluteFilePath()))) { - std::cerr << "RetroShareLink::process(): can't open file " << path << std::endl; - } - } else if (ret == QMessageBox::NoToAll) { - dontOpenNextFile = true; - } - needNotifySuccess = false; - } - } - - if (rsFiles->FileRequest(cleanname.toUtf8().constData(), RsFileHash(link.hash().toStdString()), link.size(), "", RS_FILE_REQ_ANONYMOUS_ROUTING, srcIds)) { - fileAdded.append(link.name()); - } else { - if (!bFileOpened && links.size()>1) fileExist.append(link.name());} - } - break; - case TYPE_PERSON: { #ifdef DEBUG_RSLINK @@ -1641,6 +1540,107 @@ static void processList(const QStringList &list, const QString &textSingular, co } break ; + case TYPE_FILE: + { + FileInfo fi1; + if(links.size()==1 && rsFiles->alreadyHaveFile(RsFileHash(link.hash().toStdString()), fi1) + && !link.name().endsWith(RsCollection::ExtensionString)) + { + /* fallthrough */ + } + else + { + col.merge_in(link.name(),link.size(),RsFileHash(link.hash().toStdString())) ; + fileLinkFound = true; + break; + } + } + //break; + case TYPE_EXTRAFILE: + { +#ifdef DEBUG_RSLINK + std::cerr << " RetroShareLink::process FileRequest : fileName : " << link.name().toUtf8().constData() << ". fileHash : " << link.hash().toStdString() << ". fileSize : " << link.size() << std::endl; +#endif + + needNotifySuccess = true; + std::list srcIds; + + // Add the link built-in source. This is needed for EXTRA files, where the source is specified in the link. + + if(link.type() == TYPE_EXTRAFILE) + { +#ifdef DEBUG_RSLINK + std::cerr << " RetroShareLink::process Adding built-in source " << link.SSLId().toStdString() << std::endl; +#endif + srcIds.push_back(RsPeerId(link.SSLId().toStdString())) ; + } + + // Get a list of available direct sources, in case the file is browsable only. + // + FileInfo finfo ; + rsFiles->FileDetails(RsFileHash(link.hash().toStdString()), RS_FILE_HINTS_REMOTE, finfo) ; + + for(std::vector::const_iterator it(finfo.peers.begin());it!=finfo.peers.end();++it) + { +#ifdef DEBUG_RSLINK + std::cerr << " adding peerid " << (*it).peerId << std::endl ; +#endif + srcIds.push_back((*it).peerId) ; + } + + QString cleanname = link.name() ; + static const QString bad_chars_str = "/\\\"*:?<>|" ; + + for(int i=0;ialreadyHaveFile(RsFileHash(link.hash().toStdString()), fi)) { + /* make path for downloaded file */ + std::string path; + path = fi.path;//Shared files has path with filename included + + //Seems that all FileInfo get .path==filepath+filename + //if (fi.downloadStatus == FT_STATE_COMPLETE) + // path = fi.path + "/" + fi.fname; + + QFileInfo qinfo; + qinfo.setFile(QString::fromUtf8(path.c_str())); + if (qinfo.exists() && qinfo.isFile() && !dontOpenNextFile) { + QString question = ""; + question += QObject::tr("Warning: Retroshare is about to ask your system to open this file. "); + question += QObject::tr("Before you do so, please make sure that this file does not contain malicious executable code."); + question += "

" + cleanname + ""; + + QMessageBox mb(QObject::tr("Confirmation"), question, QMessageBox::Warning, QMessageBox::Yes, QMessageBox::No, links.size()>1 ? QMessageBox::NoToAll : 0, 0); + int ret = mb.exec(); + if(ret == QMessageBox::Yes) { + ++countFileOpened; + bFileOpened = true; + /* open file with a suitable application */ + if (!RsUrlHandler::openUrl(QUrl::fromLocalFile(qinfo.absoluteFilePath()))) { + std::cerr << "RetroShareLink::process(): can't open file " << path << std::endl; + } + } else if (ret == QMessageBox::NoToAll) { + dontOpenNextFile = true; + } + needNotifySuccess = false; + } + } + + if (rsFiles->FileRequest(cleanname.toUtf8().constData(), RsFileHash(link.hash().toStdString()), link.size(), "", RS_FILE_REQ_ANONYMOUS_ROUTING, srcIds)) { + fileAdded.append(link.name()); + } else { + if (!bFileOpened && links.size()>1) fileExist.append(link.name());} + } + break; + //TYPE_PRIVATE_CHAT case TYPE_PUBLIC_MSG: From 5be57046f1c88a4ca14196e6f9bad8c8cd5ada31 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 27 Apr 2018 00:00:29 +0200 Subject: [PATCH 30/84] added load/save of random bias in GxsNetTunnel service --- libretroshare/src/gxs/rsgxsnetservice.cc | 26 ++++++++++++------- libretroshare/src/gxs/rsgxsnettunnel.cc | 8 +++--- libretroshare/src/gxs/rsgxsnettunnel.h | 15 +++++++---- libretroshare/src/retroshare/rsids.h | 2 ++ libretroshare/src/rsitems/rsgxsupdateitems.cc | 11 +++++++- libretroshare/src/rsitems/rsgxsupdateitems.h | 13 ++++++++++ 6 files changed, 56 insertions(+), 19 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 33081acd2..bf396a42c 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -1522,8 +1522,8 @@ class StoreHere { public: - StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap& cmm, RsGxsNetService::ServerMsgMap& smm,RsGxsNetService::GrpConfigMap& gcm, RsGxsServerGrpUpdate& sgm) - : mClientGrpMap(cgm), mClientMsgMap(cmm), mServerMsgMap(smm), mGrpConfigMap(gcm), mServerGrpUpdate(sgm) + StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap& cmm, RsGxsNetService::ServerMsgMap& smm,RsGxsNetService::GrpConfigMap& gcm, RsGxsServerGrpUpdate& sgm,Bias20Bytes& mrb) + : mClientGrpMap(cgm), mClientMsgMap(cmm), mServerMsgMap(smm), mGrpConfigMap(gcm), mServerGrpUpdate(sgm), mRandomBias(mrb) {} template void check_store(ID_type id,UpdateMap& map,ItemClass& item) @@ -1536,11 +1536,12 @@ public: void operator() (RsItem* item) { - RsGxsMsgUpdateItem* mui; - RsGxsGrpUpdateItem* gui; - RsGxsServerGrpUpdateItem* gsui; - RsGxsServerMsgUpdateItem* msui; - RsGxsGrpConfigItem* mgci; + RsGxsMsgUpdateItem *mui; + RsGxsGrpUpdateItem *gui; + RsGxsServerGrpUpdateItem *gsui; + RsGxsServerMsgUpdateItem *msui; + RsGxsGrpConfigItem *mgci; + RsGxsTunnelRandomBiasItem *rbsi; if((mui = dynamic_cast(item)) != NULL) check_store(mui->peerID,mClientMsgMap,*mui); @@ -1552,6 +1553,8 @@ public: check_store(msui->grpId,mServerMsgMap, *msui); else if((gsui = dynamic_cast(item)) != NULL) mServerGrpUpdate = *gsui; + else if((rbsi = dynamic_cast(item))!=NULL) + mRandomBias = rbsi->mRandomBias; else std::cerr << "Type not expected!" << std::endl; @@ -1566,7 +1569,7 @@ private: RsGxsNetService::GrpConfigMap& mGrpConfigMap; RsGxsServerGrpUpdate& mServerGrpUpdate; - + Bias20Bytes& mRandomBias ; }; bool RsGxsNetService::loadList(std::list &load) @@ -1575,7 +1578,7 @@ bool RsGxsNetService::loadList(std::list &load) // The delete is done in StoreHere, if necessary - std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mServerGrpConfigMap, mGrpServerUpdate)); + std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mServerGrpConfigMap, mGrpServerUpdate,mRandomBias)); // We reset group statistics here. This is the best place since we know at this point which are all unsubscribed groups. @@ -1652,6 +1655,11 @@ bool RsGxsNetService::saveList(bool& cleanup, std::list& save) save.push_back(it); + RsGxsTunnelRandomBiasItem *it2 = new RsGxsTunnelRandomBiasItem(mServType) ; + it2->mRandomBias = mRandomBias; + + save.push_back(it2) ; + cleanup = true; return true; } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 00e3fd852..7d4a0018f 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -39,7 +39,7 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") { #warning this is for testing only. In the final version this needs to be initialized with some random content, saved and re-used for a while (e.g. 1 month) - memset(mRandomBias,0,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; + mRandomBias.clear(); } //===========================================================================================================================================// @@ -289,12 +289,12 @@ RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId(cons RsPeerId ssl_id = rsPeers->getOwnId() ; - unsigned char mem[RsGxsGroupId::SIZE_IN_BYTES + RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE]; + unsigned char mem[RsGxsGroupId::SIZE_IN_BYTES + mRandomBias.SIZE_IN_BYTES]; memcpy(mem ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; - memcpy(mem+RsGxsGroupId::SIZE_IN_BYTES,mRandomBias ,RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE) ; + memcpy(mem+RsGxsGroupId::SIZE_IN_BYTES,mRandomBias.toByteArray(),mRandomBias.SIZE_IN_BYTES) ; - return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsGxsGroupId::SIZE_IN_BYTES+RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE).toByteArray()); + return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsGxsGroupId::SIZE_IN_BYTES+mRandomBias.SIZE_IN_BYTES).toByteArray()); } void RsGxsNetTunnelService::dump() const diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 9448b9508..5d90d9fe3 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -23,6 +23,8 @@ * */ +#pragma once + #include #include @@ -224,15 +226,17 @@ protected: void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; p3turtle *mTurtle ; + + static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 20 ; + static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE = 32 ; + + Bias20Bytes mRandomBias ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. private: void autowash() ; void sendKeepAlivePackets() ; void handleIncoming(RsGxsNetTunnelItem *item) ; void flush_pending_items(); - static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 20 ; - static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE = 32 ; - std::map mGroups ; // groups on the client and server side std::map mVirtualPeers ; // current virtual peers, which group they provide, and how to talk to them through turtle @@ -259,8 +263,9 @@ private: static void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) ; - uint8_t mRandomBias[RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE] ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. - mutable RsMutex mGxsNetTunnelMtx; + + friend class RsGxsTunnelRandomBiasItem ; + friend class StoreHere ; }; diff --git a/libretroshare/src/retroshare/rsids.h b/libretroshare/src/retroshare/rsids.h index 21a8fd754..828136dbc 100644 --- a/libretroshare/src/retroshare/rsids.h +++ b/libretroshare/src/retroshare/rsids.h @@ -238,12 +238,14 @@ static const uint32_t RS_GENERIC_ID_GXS_TUNNEL_ID_TYPE = 0x0010 ; static const uint32_t RS_GENERIC_ID_GXS_DISTANT_CHAT_ID_TYPE = 0x0011 ; static const uint32_t RS_GENERIC_ID_NODE_GROUP_ID_TYPE = 0x0012 ; static const uint32_t RS_GENERIC_ID_SHA256_ID_TYPE = 0x0013 ; +static const uint32_t RS_GENERIC_ID_20_BYTES_UNTYPED = 0x0014 ; typedef t_RsGenericIdType< SSL_ID_SIZE , false, RS_GENERIC_ID_SSL_ID_TYPE> SSLIdType ; typedef t_RsGenericIdType< PGP_KEY_ID_SIZE , true, RS_GENERIC_ID_PGP_ID_TYPE> PGPIdType ; typedef t_RsGenericIdType< SHA1_SIZE , false, RS_GENERIC_ID_SHA1_ID_TYPE> Sha1CheckSum ; typedef t_RsGenericIdType< SHA256_SIZE , false, RS_GENERIC_ID_SHA256_ID_TYPE> Sha256CheckSum ; typedef t_RsGenericIdType< PGP_KEY_FINGERPRINT_SIZE, true, RS_GENERIC_ID_PGP_FINGERPRINT_TYPE> PGPFingerprintType ; +typedef t_RsGenericIdType< SHA1_SIZE , true, RS_GENERIC_ID_20_BYTES_UNTYPED> Bias20Bytes ; typedef t_RsGenericIdType< CERT_SIGN_LEN , false, RS_GENERIC_ID_GXS_GROUP_ID_TYPE > GXSGroupId ; typedef t_RsGenericIdType< CERT_SIGN_LEN , false, RS_GENERIC_ID_GXS_ID_TYPE > GXSId ; diff --git a/libretroshare/src/rsitems/rsgxsupdateitems.cc b/libretroshare/src/rsitems/rsgxsupdateitems.cc index 50f154cb2..eee847704 100644 --- a/libretroshare/src/rsitems/rsgxsupdateitems.cc +++ b/libretroshare/src/rsitems/rsgxsupdateitems.cc @@ -44,6 +44,7 @@ RsItem* RsGxsUpdateSerialiser::create_item(uint16_t service,uint8_t item_subtype case RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE: return new RsGxsServerGrpUpdateItem(SERVICE_TYPE); case RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE: return new RsGxsServerMsgUpdateItem(SERVICE_TYPE); case RS_PKT_SUBTYPE_GXS_GRP_CONFIG: return new RsGxsGrpConfigItem(SERVICE_TYPE); + case RS_PKT_SUBTYPE_GXS_RANDOM_BIAS: return new RsGxsTunnelRandomBiasItem(SERVICE_TYPE); default: return NULL ; } @@ -76,6 +77,11 @@ void RsGxsServerGrpUpdateItem::clear() grpUpdateTS = 0; } +void RsGxsTunnelRandomBiasItem::clear() +{ + mRandomBias.clear() ; +} + /**********************************************************************************************/ /* SERIALISER */ /**********************************************************************************************/ @@ -134,5 +140,8 @@ void RsGxsGrpConfigItem::serial_process(RsGenericSerializer::SerializeJob j,RsGe RsTypeSerializer::serial_process(j,ctx,msg_send_delay,"msg_send_delay") ; RsTypeSerializer::serial_process(j,ctx,msg_req_delay,"msg_req_delay") ; } - +void RsGxsTunnelRandomBiasItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +{ + RsTypeSerializer::serial_process(j,ctx,mRandomBias,"random bias") ; +} diff --git a/libretroshare/src/rsitems/rsgxsupdateitems.h b/libretroshare/src/rsitems/rsgxsupdateitems.h index c9725426c..e73dc2197 100644 --- a/libretroshare/src/rsitems/rsgxsupdateitems.h +++ b/libretroshare/src/rsitems/rsgxsupdateitems.h @@ -39,6 +39,7 @@ #include "gxs/rsgxs.h" #include "gxs/rsgxsdata.h" +#include "gxs/rsgxsnettunnel.h" #include "serialiser/rstlvidset.h" @@ -48,6 +49,7 @@ const uint8_t RS_PKT_SUBTYPE_GXS_MSG_UPDATE = 0x03; const uint8_t RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE = 0x04; const uint8_t RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE = 0x08; const uint8_t RS_PKT_SUBTYPE_GXS_GRP_CONFIG = 0x09; +const uint8_t RS_PKT_SUBTYPE_GXS_RANDOM_BIAS = 0x0a; class RsGxsNetServiceItem: public RsItem { @@ -186,6 +188,17 @@ public: RsGxsGroupId grpId; }; +class RsGxsTunnelRandomBiasItem: public RsGxsNetServiceItem +{ +public: + explicit RsGxsTunnelRandomBiasItem(uint16_t servType) : RsGxsNetServiceItem(servType, RS_PKT_SUBTYPE_GXS_RANDOM_BIAS) { clear();} + virtual ~RsGxsTunnelRandomBiasItem() {} + + virtual void clear(); + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); + + Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization. +}; class RsGxsUpdateSerialiser : public RsServiceSerializer { From 99739783503a05731916f66b2ae6715785b79a67 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 29 Apr 2018 16:19:45 +0200 Subject: [PATCH 31/84] improved management of tunnels and virtual peers --- libretroshare/src/gxs/rsgxsnetservice.cc | 7 ++- libretroshare/src/gxs/rsgxsnetservice.h | 2 + libretroshare/src/gxs/rsgxsnettunnel.cc | 73 +++++++++++++++--------- libretroshare/src/gxs/rsgxsnettunnel.h | 11 +++- 4 files changed, 61 insertions(+), 32 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index bf396a42c..50045b556 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -273,14 +273,14 @@ ***/ #define NXS_NET_DEBUG_0 1 -#define NXS_NET_DEBUG_1 1 +//#define NXS_NET_DEBUG_1 1 //#define NXS_NET_DEBUG_2 1 //#define NXS_NET_DEBUG_3 1 //#define NXS_NET_DEBUG_4 1 //#define NXS_NET_DEBUG_5 1 //#define NXS_NET_DEBUG_6 1 //#define NXS_NET_DEBUG_7 1 -#define NXS_NET_DEBUG_8 1 +//#define NXS_NET_DEBUG_8 1 //#define NXS_FRAG @@ -2008,7 +2008,8 @@ void RsGxsNetService::data_tick() // also tick distant traffic - RsGxsNetTunnelService::data_tick(); + if(mAllowDistSync) + RsGxsNetTunnelService::data_tick(); } void RsGxsNetService::debugDump() diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 04c0317a6..d7fe23cc3 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -104,6 +104,8 @@ public: public: + virtual uint16_t serviceType() const { return mServType ; } + /*! * Use this to set how far back synchronisation and storage of messages should take place * @param age the max age a sync/storage item can to be allowed in a synchronisation diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 7d4a0018f..052dffacc 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -32,7 +32,7 @@ #define DEBUG_RSGXSNETTUNNEL 1 #define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } -#define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL : " << __FUNCTION__ << " : " +#define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL(" << std::hex << serviceType() << std::dec << ") : " << __FUNCTION__ << " : " #define GXS_NET_TUNNEL_ERROR() std::cerr << "(EE) GXS_NET_TUNNEL ERROR : " @@ -204,6 +204,8 @@ bool RsGxsNetTunnelService::sendTunnelData(unsigned char *& data,uint32_t data_l return false ; } + it->second.last_contact = time(NULL) ; + // 2 - encrypt and send the item. RsTurtleGenericDataItem *encrypted_turtle_item = NULL ; @@ -247,7 +249,9 @@ bool RsGxsNetTunnelService::requestDistantPeers(const RsGxsGroupId& group_id) RsGxsNetTunnelGroupInfo& ginfo( mGroups[group_id] ) ; - ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE; + if(ginfo.group_policy < RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE) + ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE; + ginfo.hash = calculateGroupHash(group_id) ; mHandledHashes[ginfo.hash] = group_id ; @@ -287,14 +291,18 @@ RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId(cons // We compute sha1( SSL_id | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId +#ifdef DEBUG_RSGXSNETTUNNEL + // /!\ this is for testing only! Remove this when done! Can not be done at initialization when rsPeer is not started. RsPeerId ssl_id = rsPeers->getOwnId() ; + mRandomBias = Bias20Bytes(RsDirUtil::sha1sum(ssl_id.toByteArray(),ssl_id.SIZE_IN_BYTES)) ; +#endif - unsigned char mem[RsGxsGroupId::SIZE_IN_BYTES + mRandomBias.SIZE_IN_BYTES]; + unsigned char mem[group_id.SIZE_IN_BYTES + mRandomBias.SIZE_IN_BYTES]; - memcpy(mem ,group_id.toByteArray(),RsGxsGroupId::SIZE_IN_BYTES) ; - memcpy(mem+RsGxsGroupId::SIZE_IN_BYTES,mRandomBias.toByteArray(),mRandomBias.SIZE_IN_BYTES) ; + memcpy(mem ,group_id.toByteArray(),group_id.SIZE_IN_BYTES) ; + memcpy(mem+group_id.SIZE_IN_BYTES,mRandomBias.toByteArray(),mRandomBias.SIZE_IN_BYTES) ; - return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,RsGxsGroupId::SIZE_IN_BYTES+mRandomBias.SIZE_IN_BYTES).toByteArray()); + return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,group_id.SIZE_IN_BYTES+mRandomBias.SIZE_IN_BYTES).toByteArray()); } void RsGxsNetTunnelService::dump() const @@ -308,10 +316,11 @@ void RsGxsNetTunnelService::dump() const std::string("[VPIDS_AVAILABLE ]") }; - static std::string group_policy_str[3] = { - std::string("[UNKNOWN]"), - std::string("[PASSIVE]"), - std::string("[ACTIVE ]"), + static std::string group_policy_str[4] = { + std::string("[UNKNOWN ]"), + std::string("[PASSIVE ]"), + std::string("[ACTIVE ]"), + std::string("[REQUESTING]"), }; static std::string vpid_status_str[3] = { std::string("[UNKNOWN ]"), @@ -319,39 +328,39 @@ void RsGxsNetTunnelService::dump() const std::string("[ACTIVE ]") }; - std::cerr << "GxsNetTunnelService dump: " << std::endl; - std::cerr << "Managed GXS groups: " << std::endl; + std::cerr << "GxsNetTunnelService dump (this=" << (void*)this << ". serv=" << std::hex << serviceType() << std::dec <<") : " << std::endl; + std::cerr << " Managed GXS groups: " << std::endl; for(auto it(mGroups.begin());it!=mGroups.end();++it) { - std::cerr << " " << it->first << " hash: " << it->second.hash << " policy: " << group_policy_str[it->second.group_policy] << " status: " << group_status_str[it->second.group_status] << " Last contact: " << time(NULL) - it->second.last_contact << " secs ago" << std::endl; + std::cerr << " " << it->first << " hash: " << it->second.hash << " policy: " << group_policy_str[it->second.group_policy] << " status: " << group_status_str[it->second.group_status] << " Last contact: " << time(NULL) - it->second.last_contact << " secs ago" << std::endl; if(!it->second.virtual_peers.empty()) - std::cerr << " virtual peers:" << std::endl; + std::cerr << " virtual peers:" << std::endl; for(auto it2(it->second.virtual_peers.begin());it2!=it->second.virtual_peers.end();++it2) - std::cerr << " " << *it2 << std::endl; + std::cerr << " " << *it2 << std::endl; } - std::cerr << "Virtual peers: " << std::endl; + std::cerr << " Virtual peers: " << std::endl; for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) - std::cerr << " GXS Peer:" << it->first << " Turtle:" << it->second.turtle_virtual_peer_id + std::cerr << " GXS Peer:" << it->first << " Turtle:" << it->second.turtle_virtual_peer_id << " status: " << vpid_status_str[it->second.vpid_status] << " direction: " << " group_id: " << it->second.group_id << " direction: " << (int)it->second.side - << " last seen " << time(NULL)-it->second.last_contact + << " last seen " << time(NULL)-it->second.last_contact << " secs ago" << " ekey: " << RsUtil::BinToHex(it->second.encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE,10) << std::endl; - std::cerr << "Virtual peer turtle => GXS conversion table: " << std::endl; + std::cerr << " Virtual peer turtle => GXS conversion table: " << std::endl; for(auto it(mTurtle2GxsPeer.begin());it!=mTurtle2GxsPeer.end();++it) - std::cerr << " " << it->first << " => " << it->second << std::endl; + std::cerr << " " << it->first << " => " << it->second << std::endl; - std::cerr << "Hashes: " << std::endl; + std::cerr << " Hashes: " << std::endl; for(auto it(mHandledHashes.begin());it!=mHandledHashes.end();++it) - std::cerr << " hash: " << it->first << " GroupId: " << it->second << std::endl; + std::cerr << " hash: " << it->first << " GroupId: " << it->second << std::endl; - std::cerr << "Incoming data: " << std::endl; + std::cerr << " Incoming data: " << std::endl; for(auto it(mIncomingData.begin());it!=mIncomingData.end();++it) - std::cerr << " peer id " << it->first << " " << (void*)it->second << std::endl; + std::cerr << " peer id " << it->first << " " << (void*)it->second << std::endl; } //===========================================================================================================================================// @@ -472,6 +481,8 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co free(data); return; } + it2->second.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; // status of the peer + it2->second.last_contact = time(NULL); // last time some data was sent/recvd #ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << "item contains generic data for VPID " << gxs_vpid << ". Storing in incoming list" << std::endl; @@ -638,10 +649,14 @@ void RsGxsNetTunnelService::sendKeepAlivePackets() { RsGxsNetTunnelVirtualPeerId own_gxs_vpid = locked_makeVirtualPeerId(it->second.group_id) ; +#ifdef DEBUG_RSGXSNETTUNNEL + GXS_NET_TUNNEL_DEBUG() << " virtual peer " << it->first << " through tunnel " << it->second.turtle_virtual_peer_id << " for group " << it->second.group_id ; +#endif + if(own_gxs_vpid < it->first) { #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << " sending to virtual peer " << it->first << " through tunnel " << it->second.turtle_virtual_peer_id << std::endl; + std::cerr << ": sent!" << std::endl; #endif RsGxsNetTunnelKeepAliveItem pitem ; RsTemporaryMemory tmpmem( RsGxsNetTunnelSerializer().size(&pitem) ) ; @@ -653,10 +668,12 @@ void RsGxsNetTunnelService::sendKeepAlivePackets() if(p3turtle::encryptData(tmpmem,len,it->second.encryption_master_key,encrypted_turtle_item)) mPendingTurtleItems.push_back(std::make_pair(it->second.turtle_virtual_peer_id,encrypted_turtle_item)) ; + + it->second.last_contact = time(NULL) ; } #ifdef DEBUG_RSGXSNETTUNNEL else - GXS_NET_TUNNEL_DEBUG() << " ignoring virtual peer " << it->first << std::endl; + std::cerr << ": ignored!" << std::endl; #endif } } @@ -679,7 +696,7 @@ void RsGxsNetTunnelService::autowash() #endif // check whether the group already has GXS virtual peers with suppliers. If so, we can set the GRP policy as passive. - if(ginfo.group_policy == RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE) + if(ginfo.group_policy >= RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE) { bool found = false ; @@ -699,10 +716,12 @@ void RsGxsNetTunnelService::autowash() std::cerr << " active, with client-side peers : "; #endif should_monitor_tunnels = false ; + ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE ; } else { should_monitor_tunnels = true ; + ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING ; #ifdef DEBUG_RSGXSNETTUNNEL std::cerr << " active, and no client-side peers available : " ; #endif diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 5d90d9fe3..825ff891c 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -134,7 +134,8 @@ struct RsGxsNetTunnelGroupInfo enum GroupPolicy { RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN = 0x00, // nothing has been set RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE = 0x01, // group is available for server side tunnels, but does not explicitely request tunnels - RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group will explicitely request tunnels, if none available + RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group will only explicitely request tunnels if none available + RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING = 0x03, // group explicitely requests tunnels }; RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {} @@ -153,6 +154,12 @@ public: RsGxsNetTunnelService() ; virtual ~RsGxsNetTunnelService() ; + /*! + * \brief serviceType + * \return returns the service that is currently using this as a subclass. + */ + virtual uint16_t serviceType() const = 0 ; + /*! * \brief Manage tunnels for this group * @param group_id group for which tunnels should be released @@ -230,7 +237,7 @@ protected: static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 20 ; static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE = 32 ; - Bias20Bytes mRandomBias ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. + mutable Bias20Bytes mRandomBias ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time. private: void autowash() ; void sendKeepAlivePackets() ; From c5ba0e975fe78377eb628cb1ed6f15adc1512009 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 29 Apr 2018 19:20:14 +0200 Subject: [PATCH 32/84] fixed TS in tunnel management --- libretroshare/src/gxs/rsgxsnettunnel.cc | 26 +++++++++++++++---------- libretroshare/src/gxs/rsgxsnettunnel.h | 4 ++++ 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 052dffacc..96232ad6f 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -40,6 +40,10 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") { #warning this is for testing only. In the final version this needs to be initialized with some random content, saved and re-used for a while (e.g. 1 month) mRandomBias.clear(); + + mLastKeepAlive = time(NULL) + (lrand48()%20); // adds some variance in order to avoid doing all this tasks at once across services + mLastAutoWash = time(NULL) + (lrand48()%20); + mLastDump = time(NULL) + (lrand48()%20); } //===========================================================================================================================================// @@ -554,6 +558,8 @@ void RsGxsNetTunnelService::removeVirtualPeer(const TurtleFileHash& hash, const #ifdef DEBUG_RSGXSNETTUNNEL GXS_NET_TUNNEL_DEBUG() << " removing virtual peer " << vpid << " for hash " << hash << std::endl; #endif + mTurtle2GxsPeer.erase(vpid); + auto it = mHandledHashes.find(hash) ; if(it == mHandledHashes.end()) @@ -616,23 +622,23 @@ void RsGxsNetTunnelService::data_tick() // cleanup - static time_t last_autowash = time(NULL); - - if(last_autowash + 5 < now) + if(mLastAutoWash + 5 < now) { autowash(); - last_autowash = now; + mLastAutoWash = now; } - static time_t last_dump = time(NULL); - - if(last_dump + 10 < now) + if(mLastKeepAlive + 20 < now) { - last_dump = now; - dump(); - + mLastKeepAlive = now ; sendKeepAlivePackets() ; } + + if(mLastDump + 10 < now) + { + mLastDump = now; + dump(); + } } void RsGxsNetTunnelService::sendKeepAlivePackets() diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 825ff891c..d0ab17c75 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -274,5 +274,9 @@ private: friend class RsGxsTunnelRandomBiasItem ; friend class StoreHere ; + + time_t mLastKeepAlive ; + time_t mLastAutoWash ; + time_t mLastDump ; }; From ba0819f8d0a4b4b3a699f51d29d6f8197b3c74ba Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 1 May 2018 15:17:41 +0200 Subject: [PATCH 33/84] added additional debug info to test proper distant request of GXS ids --- libretroshare/src/gxs/rsgxsnetservice.cc | 25 +++++++++++++++++------ libretroshare/src/gxs/rsgxsnettunnel.h | 2 +- libretroshare/src/services/p3idservice.cc | 3 ++- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 50045b556..7538a1604 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -273,14 +273,14 @@ ***/ #define NXS_NET_DEBUG_0 1 -//#define NXS_NET_DEBUG_1 1 +#define NXS_NET_DEBUG_1 1 //#define NXS_NET_DEBUG_2 1 //#define NXS_NET_DEBUG_3 1 //#define NXS_NET_DEBUG_4 1 -//#define NXS_NET_DEBUG_5 1 +#define NXS_NET_DEBUG_5 1 //#define NXS_NET_DEBUG_6 1 //#define NXS_NET_DEBUG_7 1 -//#define NXS_NET_DEBUG_8 1 +#define NXS_NET_DEBUG_8 1 //#define NXS_FRAG @@ -318,7 +318,7 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING = 0x05 ; || defined(NXS_NET_DEBUG_8) static const RsPeerId peer_to_print = RsPeerId(std::string("")) ; -static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("ff8d59ef38cad0429f34cc21749dda71")) ; // use this to allow to this group id only, or "" for all IDs +static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("")) ; // use this to allow to this group id only, or "" for all IDs static const uint32_t service_to_print = RS_SERVICE_GXS_TYPE_CHANNELS ; // use this to allow to this service id only, or 0 for all services // warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums) @@ -3275,7 +3275,7 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) { #ifdef NXS_NET_DEBUG_1 - GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendGrpsTransaction() Generating Grp data send fron TransN: " << tr->mTransaction->transactionNumber << std::endl; + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendGrpsTransaction() Generating Grp data send from TransN: " << tr->mTransaction->transactionNumber << std::endl; #endif // go groups requested in transaction tr @@ -3288,7 +3288,12 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) { RsNxsSyncGrpItem* item = dynamic_cast(*lit); if (item) + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),item->grpId) << "locked_genSendGrpsTransaction() retrieving data for group \"" << item->grpId << "\"" << std::endl; +#endif grps[item->grpId] = NULL; + } else { #ifdef NXS_NET_DEBUG_1 @@ -3300,7 +3305,12 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) if(!grps.empty()) mDataStore->retrieveNxsGrps(grps, false, false); else + { +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "RsGxsNetService::locked_genSendGrpsTransaction(): no group to request! This is unexpected" << std::endl; +#endif return; + } NxsTransaction* newTr = new NxsTransaction(); newTr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM; @@ -3317,6 +3327,9 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr) mit->second->transactionNumber = transN; newTr->mItems.push_back(mit->second); mit->second = NULL ; // avoids deletion +#ifdef NXS_NET_DEBUG_1 + GXSNETDEBUG_PG(tr->mTransaction->PeerId(),mit->first) << "RsGxsNetService::locked_genSendGrpsTransaction(): adding grp data of group \"" << mit->first << "\" to transaction" << std::endl; +#endif } if(newTr->mItems.empty()){ @@ -4769,7 +4782,7 @@ void RsGxsNetService::processExplicitGroupRequests() for(; git != groupIdList.end(); ++git) { #ifdef NXS_NET_DEBUG_0 - GXSNETDEBUG_PG(peerId,*git) << " group request for grp ID " << *git << " to peer " << peerId << std::endl; + GXSNETDEBUG_P_(peerId) << " group request for grp ID " << *git << " to peer " << peerId << std::endl; #endif RsNxsSyncGrpItem* item = new RsNxsSyncGrpItem(mServType); item->grpId = *git; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index d0ab17c75..4c776b9fa 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -176,7 +176,7 @@ public: * \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and * alive. This function also "registers" the group which allows to handle tunnel requests in the server side. */ - bool getVirtualPeers(std::list& peers) ; // returns the virtual peers for this group + bool getVirtualPeers(std::list& peers) ; // returns the virtual peers for this service /*! * \brief sendData diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 71b0e2274..51bce5550 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -47,6 +47,7 @@ #include #include +#define DEBUG_IDS 1 /**** * #define DEBUG_IDS 1 * #define DEBUG_RECOGN 1 @@ -277,7 +278,7 @@ void p3IdService::timeStampKey(const RsGxsId& gxs_id, const RsIdentityUsage& rea return ; } #ifdef DEBUG_IDS - std::cerr << "(II) time stamping key " << gxs_id << " for the following reason: " << reason << std::endl; + std::cerr << "(II) time stamping key " << gxs_id << " for the following reason: " << reason.mUsageCode << std::endl; #endif RS_STACK_MUTEX(mIdMtx) ; From 4d6fed643a3fd92a12e1d2d55437c7495db7cec2 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 1 May 2018 20:10:56 +0200 Subject: [PATCH 34/84] reverted to single GxsTunnelService shared for all services --- libretroshare/src/gxs/rsgxsnetservice.cc | 51 +++----- libretroshare/src/gxs/rsgxsnetservice.h | 5 +- libretroshare/src/gxs/rsgxsnettunnel.cc | 121 +++++++++++++++--- libretroshare/src/gxs/rsgxsnettunnel.h | 27 ++-- libretroshare/src/rsitems/rsgxsupdateitems.cc | 10 -- libretroshare/src/rsitems/rsgxsupdateitems.h | 12 -- libretroshare/src/rsserver/rsinit.cc | 25 +++- 7 files changed, 157 insertions(+), 94 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 7538a1604..e18423c0d 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -361,7 +361,7 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs, const RsServiceInfo serviceInfo, RsGixsReputation* reputations, RsGcxs* circles, RsGixs *gixs, - PgpAuxUtils *pgpUtils, + PgpAuxUtils *pgpUtils, RsGxsNetTunnelService *mGxsNT, bool grpAutoSync, bool msgAutoSync, bool distSync, uint32_t default_store_period, uint32_t default_sync_period) : p3ThreadedService(), p3Config(), mTransactionN(0), mObserver(nxsObs), mDataStore(gds), @@ -370,7 +370,7 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, mSyncTs(0), mLastKeyPublishTs(0), mLastCleanRejectedMessages(0), mSYNC_PERIOD(SYNC_PERIOD), mCircles(circles), mGixs(gixs), - mReputations(reputations), mPgpUtils(pgpUtils), + mReputations(reputations), mPgpUtils(pgpUtils), mGxsNetTunnel(mGxsNT), mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync),mAllowDistSync(distSync), mServiceInfo(serviceInfo), mDefaultMsgStorePeriod(default_store_period), mDefaultMsgSyncPeriod(default_sync_period) @@ -569,12 +569,12 @@ void RsGxsNetService::syncWithPeers() std::set peers; mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers); - if(mAllowDistSync) + if(mAllowDistSync && mGxsNetTunnel != NULL) { // Grab all online virtual peers of distant tunnels for the current service. std::list vpids ; - getVirtualPeers(vpids); + mGxsNetTunnel->getVirtualPeers(vpids); for(auto it(vpids.begin());it!=vpids.end();++it) peers.insert(RsPeerId(*it)) ; @@ -742,7 +742,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) RsGxsGroupId tmp_grpId; - if(mAllowDistSync && isDistantPeer( static_cast(si->PeerId()),tmp_grpId)) + if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer( static_cast(si->PeerId()),tmp_grpId)) { RsNxsSerialiser ser(mServType); @@ -758,7 +758,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) #endif ser.serialise(si,mem,&size) ; - sendTunnelData(mem,size,static_cast(si->PeerId())); + mGxsNetTunnel->sendTunnelData(mServType,mem,size,static_cast(si->PeerId())); } else sendItem(si) ; @@ -766,7 +766,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) void RsGxsNetService::checkDistantSyncState() { - if(!mAllowDistSync) + if(!mAllowDistSync || mGxsNetTunnel==NULL) return ; RsGxsGrpMetaTemporaryMap grpMeta; @@ -806,14 +806,14 @@ void RsGxsNetService::checkDistantSyncState() if(at_least_one_friend_is_supplier) { - releaseDistantPeers(grpId); + mGxsNetTunnel->releaseDistantPeers(mServType,grpId); #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___<< " Group " << grpId << ": suppliers among friends. Releasing peers." << std::endl; #endif } else { - requestDistantPeers(grpId); + mGxsNetTunnel->requestDistantPeers(mServType,grpId); #ifdef NXS_NET_DEBUG_8 GXSNETDEBUG___<< " Group " << grpId << ": no suppliers among friends. Requesting peers." << std::endl; #endif @@ -1522,8 +1522,8 @@ class StoreHere { public: - StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap& cmm, RsGxsNetService::ServerMsgMap& smm,RsGxsNetService::GrpConfigMap& gcm, RsGxsServerGrpUpdate& sgm,Bias20Bytes& mrb) - : mClientGrpMap(cgm), mClientMsgMap(cmm), mServerMsgMap(smm), mGrpConfigMap(gcm), mServerGrpUpdate(sgm), mRandomBias(mrb) + StoreHere(RsGxsNetService::ClientGrpMap& cgm, RsGxsNetService::ClientMsgMap& cmm, RsGxsNetService::ServerMsgMap& smm,RsGxsNetService::GrpConfigMap& gcm, RsGxsServerGrpUpdate& sgm) + : mClientGrpMap(cgm), mClientMsgMap(cmm), mServerMsgMap(smm), mGrpConfigMap(gcm), mServerGrpUpdate(sgm) {} template void check_store(ID_type id,UpdateMap& map,ItemClass& item) @@ -1541,7 +1541,6 @@ public: RsGxsServerGrpUpdateItem *gsui; RsGxsServerMsgUpdateItem *msui; RsGxsGrpConfigItem *mgci; - RsGxsTunnelRandomBiasItem *rbsi; if((mui = dynamic_cast(item)) != NULL) check_store(mui->peerID,mClientMsgMap,*mui); @@ -1553,8 +1552,6 @@ public: check_store(msui->grpId,mServerMsgMap, *msui); else if((gsui = dynamic_cast(item)) != NULL) mServerGrpUpdate = *gsui; - else if((rbsi = dynamic_cast(item))!=NULL) - mRandomBias = rbsi->mRandomBias; else std::cerr << "Type not expected!" << std::endl; @@ -1569,7 +1566,6 @@ private: RsGxsNetService::GrpConfigMap& mGrpConfigMap; RsGxsServerGrpUpdate& mServerGrpUpdate; - Bias20Bytes& mRandomBias ; }; bool RsGxsNetService::loadList(std::list &load) @@ -1578,12 +1574,11 @@ bool RsGxsNetService::loadList(std::list &load) // The delete is done in StoreHere, if necessary - std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mServerGrpConfigMap, mGrpServerUpdate,mRandomBias)); + std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mServerGrpConfigMap, mGrpServerUpdate)); + time_t now = time(NULL); // We reset group statistics here. This is the best place since we know at this point which are all unsubscribed groups. - time_t now = time(NULL); - for(GrpConfigMap::iterator it(mServerGrpConfigMap.begin());it!=mServerGrpConfigMap.end();++it) { // At each reload, we reset the count of visible messages. It will be rapidely restored to its real value from friends. @@ -1637,6 +1632,7 @@ struct get_second : public std::unary_function& save) { RS_STACK_MUTEX(mNxsMutex) ; @@ -1655,11 +1651,6 @@ bool RsGxsNetService::saveList(bool& cleanup, std::list& save) save.push_back(it); - RsGxsTunnelRandomBiasItem *it2 = new RsGxsTunnelRandomBiasItem(mServType) ; - it2->mRandomBias = mRandomBias; - - save.push_back(it2) ; - cleanup = true; return true; } @@ -1686,7 +1677,7 @@ RsItem *RsGxsNetService::generic_recvItem() uint32_t size = 0 ; RsGxsNetTunnelVirtualPeerId virtual_peer_id ; - while(mAllowDistSync && receiveTunnelData(data,size,virtual_peer_id)) + while(mAllowDistSync && mGxsNetTunnel!=NULL && mGxsNetTunnel->receiveTunnelData(mServType,data,size,virtual_peer_id)) { RsNxsItem *item = dynamic_cast(RsNxsSerialiser(mServType).deserialise(data,&size)) ; item->PeerId(virtual_peer_id) ; @@ -2005,11 +1996,6 @@ void RsGxsNetService::data_tick() runVetting(); processExplicitGroupRequests(); - - // also tick distant traffic - - if(mAllowDistSync) - RsGxsNetTunnelService::data_tick(); } void RsGxsNetService::debugDump() @@ -4117,8 +4103,9 @@ bool RsGxsNetService::canSendGrpId(const RsPeerId& sslId, const RsGxsGrpMetaData // check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for RsGxsGroupId peer_grp ; - if(isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) + if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) { +#warning (cyril) make sure that this is not a problem for cross-service sending of items #ifdef NXS_NET_DEBUG_4 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl; #endif @@ -4181,7 +4168,7 @@ bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxs // check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for RsGxsGroupId peer_grp ; - if(isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) + if(mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) { #ifdef NXS_NET_DEBUG_4 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl; @@ -4535,7 +4522,7 @@ bool RsGxsNetService::canSendMsgIds(std::vector& msgMetas, co // check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for RsGxsGroupId peer_grp ; - if(isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) + if(mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) { #ifdef NXS_NET_DEBUG_4 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl; diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index d7fe23cc3..6ead3f4c7 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -72,7 +72,7 @@ class RsGroupNetworkStatsRecord * Incoming transaction are in 3 different states * 1. START 2. RECEIVING 3. END */ -class RsGxsNetService : public RsNetworkExchangeService, public RsGxsNetTunnelService, public p3ThreadedService, public p3Config +class RsGxsNetService : public RsNetworkExchangeService, public p3ThreadedService, public p3Config { public: @@ -90,7 +90,7 @@ public: RsNxsObserver *nxsObs, // used to be = NULL. const RsServiceInfo serviceInfo, RsGixsReputation* reputations = NULL, RsGcxs* circles = NULL, RsGixs *gixs=NULL, - PgpAuxUtils *pgpUtils = NULL, + PgpAuxUtils *pgpUtils = NULL, RsGxsNetTunnelService *mGxsNT = NULL, bool grpAutoSync = true, bool msgAutoSync = true,bool distSync=false, uint32_t default_store_period = RS_GXS_DEFAULT_MSG_STORE_PERIOD, uint32_t default_sync_period = RS_GXS_DEFAULT_MSG_REQ_PERIOD); @@ -548,6 +548,7 @@ private: RsGixs *mGixs; RsGixsReputation* mReputations; PgpAuxUtils *mPgpUtils; + RsGxsNetTunnelService *mGxsNetTunnel; bool mGrpAutoSync; bool mAllowMsgSync; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 96232ad6f..bb1abd6e1 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -32,7 +32,7 @@ #define DEBUG_RSGXSNETTUNNEL 1 #define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } -#define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL(" << std::hex << serviceType() << std::dec << ") : " << __FUNCTION__ << " : " +#define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL: " << __FUNCTION__ << " : " #define GXS_NET_TUNNEL_ERROR() std::cerr << "(EE) GXS_NET_TUNNEL ERROR : " @@ -54,6 +54,7 @@ const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ; class RsGxsNetTunnelItem: public RsItem { @@ -91,6 +92,21 @@ public: virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) {} }; +class RsGxsNetTunnelRandomBiasItem: public RsGxsNetTunnelItem +{ +public: + explicit RsGxsNetTunnelRandomBiasItem() : RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS) { clear();} + virtual ~RsGxsNetTunnelRandomBiasItem() {} + + virtual void clear() { mRandomBias.clear() ; } + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,mRandomBias,"random bias") ; + } + + Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization. +}; + class RsGxsNetTunnelSerializer: public RsServiceSerializer { public: @@ -108,6 +124,7 @@ public: { case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER: return new RsGxsNetTunnelVirtualPeerItem ; case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ; + case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ; default: GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << item_subtype << std::dec << " is not handled!" << std::endl; return NULL ; @@ -139,7 +156,8 @@ RsGxsNetTunnelService::~RsGxsNetTunnelService() mVirtualPeers.clear(); for(auto it(mIncomingData.begin());it!=mIncomingData.end();++it) - delete (*it).second; + for(auto it2(it->second.begin());it2!=it->second.end();++it2) + delete it2->second; mIncomingData.clear(); } @@ -159,11 +177,11 @@ bool RsGxsNetTunnelService::isDistantPeer(const RsGxsNetTunnelVirtualPeerId& vir return false ; } -bool RsGxsNetTunnelService::receiveTunnelData(unsigned char *& data,uint32_t& data_len,RsGxsNetTunnelVirtualPeerId& virtual_peer) +bool RsGxsNetTunnelService::receiveTunnelData(uint16_t service_id, unsigned char *& data, uint32_t& data_len, RsGxsNetTunnelVirtualPeerId& virtual_peer) { RS_STACK_MUTEX(mGxsNetTunnelMtx); - std::list >& lst(mIncomingData); + std::list >& lst(mIncomingData[service_id]); if(lst.empty()) { @@ -185,7 +203,7 @@ bool RsGxsNetTunnelService::receiveTunnelData(unsigned char *& data,uint32_t& da return true; } -bool RsGxsNetTunnelService::sendTunnelData(unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) +bool RsGxsNetTunnelService::sendTunnelData(uint16_t service_id,unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) { RS_STACK_MUTEX(mGxsNetTunnelMtx); // The item is serialized and encrypted using chacha20+SHA256, using the generic turtle encryption, and then sent to the turtle router. @@ -245,7 +263,7 @@ bool RsGxsNetTunnelService::getVirtualPeers(std::listgetOwnId() ; + mRandomBias = Bias20Bytes(RsDirUtil::sha1sum(ssl_id.toByteArray(),ssl_id.SIZE_IN_BYTES)) ; +#else + mRandomBias = Bias20Bytes::random(); +#endif + IndicateConfigChanged(); + } + + return mRandomBias ; +} + +RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId(const RsGxsGroupId& group_id) { assert(RsPeerId::SIZE_IN_BYTES <= Sha1CheckSum::SIZE_IN_BYTES) ;// so that we can build the virtual PeerId from a SHA1 sum. // We compute sha1( SSL_id | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId -#ifdef DEBUG_RSGXSNETTUNNEL - // /!\ this is for testing only! Remove this when done! Can not be done at initialization when rsPeer is not started. - RsPeerId ssl_id = rsPeers->getOwnId() ; - mRandomBias = Bias20Bytes(RsDirUtil::sha1sum(ssl_id.toByteArray(),ssl_id.SIZE_IN_BYTES)) ; -#endif + Bias20Bytes rb(locked_randomBias()); - unsigned char mem[group_id.SIZE_IN_BYTES + mRandomBias.SIZE_IN_BYTES]; + unsigned char mem[group_id.SIZE_IN_BYTES + rb.SIZE_IN_BYTES]; memcpy(mem ,group_id.toByteArray(),group_id.SIZE_IN_BYTES) ; - memcpy(mem+group_id.SIZE_IN_BYTES,mRandomBias.toByteArray(),mRandomBias.SIZE_IN_BYTES) ; + memcpy(mem+group_id.SIZE_IN_BYTES,rb.toByteArray(),rb.SIZE_IN_BYTES) ; - return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,group_id.SIZE_IN_BYTES+mRandomBias.SIZE_IN_BYTES).toByteArray()); + return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,group_id.SIZE_IN_BYTES+rb.SIZE_IN_BYTES).toByteArray()); } void RsGxsNetTunnelService::dump() const @@ -332,7 +363,7 @@ void RsGxsNetTunnelService::dump() const std::string("[ACTIVE ]") }; - std::cerr << "GxsNetTunnelService dump (this=" << (void*)this << ". serv=" << std::hex << serviceType() << std::dec <<") : " << std::endl; + std::cerr << "GxsNetTunnelService dump (this=" << (void*)this << ": " << std::endl; std::cerr << " Managed GXS groups: " << std::endl; for(auto it(mGroups.begin());it!=mGroups.end();++it) @@ -364,7 +395,12 @@ void RsGxsNetTunnelService::dump() const std::cerr << " Incoming data: " << std::endl; for(auto it(mIncomingData.begin());it!=mIncomingData.end();++it) - std::cerr << " peer id " << it->first << " " << (void*)it->second << std::endl; + { + std::cerr << " service " << std::hex << it->first << std::dec << std::endl; + + for(auto it2(it->second.begin());it2!=it->second.end();++it2) + std::cerr << " peer id " << it2->first << " " << (void*)it2->second << std::endl; + } } //===========================================================================================================================================// @@ -639,6 +675,8 @@ void RsGxsNetTunnelService::data_tick() mLastDump = now; dump(); } + + rstime::rs_usleep(1*1000*1000) ; // 1 sec } void RsGxsNetTunnelService::sendKeepAlivePackets() @@ -759,4 +797,51 @@ void RsGxsNetTunnelService::autowash() } } +bool RsGxsNetTunnelService::saveList(bool& cleanup, std::list& save) +{ + RsGxsNetTunnelRandomBiasItem *it2 = new RsGxsNetTunnelRandomBiasItem() ; + { + RS_STACK_MUTEX(mGxsNetTunnelMtx); + it2->mRandomBias = mRandomBias; + } + + save.push_back(it2) ; + cleanup = true ; + + return true; +} + +bool RsGxsNetTunnelService::loadList(std::list &load) +{ + RsGxsNetTunnelRandomBiasItem *rbsi ; + + for(auto it(load.begin());it!=load.end();++it) + { + if((rbsi = dynamic_cast(*it))!=NULL) + { + RS_STACK_MUTEX(mGxsNetTunnelMtx); + mRandomBias = rbsi->mRandomBias; + } + else + GXS_NET_TUNNEL_ERROR() << " unknown item in config file: type=" << std::hex << (*it)->PacketId() << std::dec << std::endl; + + delete *it; + } + + return true; +} + +RsSerialiser *RsGxsNetTunnelService::setupSerialiser() +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + static RsSerialiser *ser = NULL ; // this is not so nice, but this method is only called from p3Config, so there's no really need of a data race + + if(!ser) + { + ser = new RsSerialiser ; + ser->addSerialType(new RsGxsNetTunnelSerializer) ; + } + + return ser ; +} diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 4c776b9fa..a36883cbb 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -148,29 +148,23 @@ struct RsGxsNetTunnelGroupInfo std::set virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. }; -class RsGxsNetTunnelService: public RsTurtleClientService +class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread, public p3Config { public: RsGxsNetTunnelService() ; virtual ~RsGxsNetTunnelService() ; - /*! - * \brief serviceType - * \return returns the service that is currently using this as a subclass. - */ - virtual uint16_t serviceType() const = 0 ; - /*! * \brief Manage tunnels for this group * @param group_id group for which tunnels should be released */ - bool requestDistantPeers(const RsGxsGroupId&group_id) ; + bool requestDistantPeers(uint16_t service_id,const RsGxsGroupId&group_id) ; /*! * \brief Stop managing tunnels for this group * @param group_id group for which tunnels should be released */ - bool releaseDistantPeers(const RsGxsGroupId&group_id) ; + bool releaseDistantPeers(uint16_t service_id, const RsGxsGroupId&group_id) ; /*! * \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and @@ -186,7 +180,7 @@ public: * \return * true if succeeded. */ - bool sendTunnelData(unsigned char *& data, uint32_t data_len, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + bool sendTunnelData(uint16_t service_id,unsigned char *& data, uint32_t data_len, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! * \brief receiveData @@ -197,7 +191,7 @@ public: * \return * true if something is returned. If not, data is set to NULL, data_len to 0. */ - bool receiveTunnelData(unsigned char *& data, uint32_t& data_len, RsGxsNetTunnelVirtualPeerId& virtual_peer) ; + bool receiveTunnelData(uint16_t service_id, unsigned char *& data, uint32_t& data_len, RsGxsNetTunnelVirtualPeerId& virtual_peer) ; /*! * \brief isDistantPeer @@ -224,6 +218,12 @@ public: void data_tick() ; + // Overloads p3Config + + RsSerialiser *setupSerialiser(); + bool saveList(bool& cleanup, std::list& save); + bool loadList(std::list &load); + protected: // interaction with turtle router @@ -231,6 +231,7 @@ protected: virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ; void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; + const Bias20Bytes& locked_randomBias() ; p3turtle *mTurtle ; @@ -252,7 +253,7 @@ private: std::list > mPendingTurtleItems ; // items that need to be sent off-turtle Mutex. - std::list > mIncomingData; // list of incoming data items + std::map > > mIncomingData; // list of incoming data items /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to @@ -266,7 +267,7 @@ private: * tunnel ID and turtle virtual peer id. This allows RsGxsNetService to keep sync-ing the data consistently. */ - RsGxsNetTunnelVirtualPeerId locked_makeVirtualPeerId(const RsGxsGroupId& group_id) const ; + RsGxsNetTunnelVirtualPeerId locked_makeVirtualPeerId(const RsGxsGroupId& group_id) ; static void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) ; diff --git a/libretroshare/src/rsitems/rsgxsupdateitems.cc b/libretroshare/src/rsitems/rsgxsupdateitems.cc index eee847704..007407210 100644 --- a/libretroshare/src/rsitems/rsgxsupdateitems.cc +++ b/libretroshare/src/rsitems/rsgxsupdateitems.cc @@ -44,7 +44,6 @@ RsItem* RsGxsUpdateSerialiser::create_item(uint16_t service,uint8_t item_subtype case RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE: return new RsGxsServerGrpUpdateItem(SERVICE_TYPE); case RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE: return new RsGxsServerMsgUpdateItem(SERVICE_TYPE); case RS_PKT_SUBTYPE_GXS_GRP_CONFIG: return new RsGxsGrpConfigItem(SERVICE_TYPE); - case RS_PKT_SUBTYPE_GXS_RANDOM_BIAS: return new RsGxsTunnelRandomBiasItem(SERVICE_TYPE); default: return NULL ; } @@ -77,11 +76,6 @@ void RsGxsServerGrpUpdateItem::clear() grpUpdateTS = 0; } -void RsGxsTunnelRandomBiasItem::clear() -{ - mRandomBias.clear() ; -} - /**********************************************************************************************/ /* SERIALISER */ /**********************************************************************************************/ @@ -140,8 +134,4 @@ void RsGxsGrpConfigItem::serial_process(RsGenericSerializer::SerializeJob j,RsGe RsTypeSerializer::serial_process(j,ctx,msg_send_delay,"msg_send_delay") ; RsTypeSerializer::serial_process(j,ctx,msg_req_delay,"msg_req_delay") ; } -void RsGxsTunnelRandomBiasItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) -{ - RsTypeSerializer::serial_process(j,ctx,mRandomBias,"random bias") ; -} diff --git a/libretroshare/src/rsitems/rsgxsupdateitems.h b/libretroshare/src/rsitems/rsgxsupdateitems.h index e73dc2197..ec21a14c8 100644 --- a/libretroshare/src/rsitems/rsgxsupdateitems.h +++ b/libretroshare/src/rsitems/rsgxsupdateitems.h @@ -188,18 +188,6 @@ public: RsGxsGroupId grpId; }; -class RsGxsTunnelRandomBiasItem: public RsGxsNetServiceItem -{ -public: - explicit RsGxsTunnelRandomBiasItem(uint16_t servType) : RsGxsNetServiceItem(servType, RS_PKT_SUBTYPE_GXS_RANDOM_BIAS) { clear();} - virtual ~RsGxsTunnelRandomBiasItem() {} - - virtual void clear(); - virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); - - Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization. -}; - class RsGxsUpdateSerialiser : public RsServiceSerializer { public: diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 12d60061e..4140eaa0a 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1358,6 +1358,14 @@ int RsServer::StartupRetroShare() mWiki->setNetworkExchangeService(wiki_ns) ; #endif + /**** GXS Dist sync service ****/ + +#ifdef RS_USE_GXS_DISTANT_SYNC + RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ; +#else + RsGxsNetTunnelService *mGxsNetTunnel = NULL ; +#endif + /**** Forum GXS service ****/ RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db", @@ -1371,7 +1379,7 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr, mGxsForums, mGxsForums->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils); + pgpAuxUtils);//,mGxsNetTunnel,true,true,true); mGxsForums->setNetworkExchangeService(gxsforums_ns) ; @@ -1387,7 +1395,7 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr, mGxsChannels, mGxsChannels->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils,true,true,true); + pgpAuxUtils,mGxsNetTunnel,true,true,true); mGxsChannels->setNetworkExchangeService(gxschannels_ns) ; @@ -1442,7 +1450,7 @@ int RsServer::StartupRetroShare() RsGxsNetService* gxstrans_ns = new RsGxsNetService( RS_SERVICE_TYPE_GXS_TRANS, gxstrans_ds, nxsMgr, mGxsTrans, mGxsTrans->getServiceInfo(), mReputations, mGxsCircles, - mGxsIdService, pgpAuxUtils,true,true,false,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD); + mGxsIdService, pgpAuxUtils,NULL,true,true,false,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD); mGxsTrans->setNetworkExchangeService(gxstrans_ns); pqih->addService(gxstrans_ns, true); @@ -1475,16 +1483,17 @@ int RsServer::StartupRetroShare() pqih -> addService(fdb,true); pqih -> addService(ftserver,true); - mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ; - mGxsTunnels->connectToTurtleRouter(tr) ; - rsGxsTunnel = mGxsTunnels; + mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ; + mGxsTunnels->connectToTurtleRouter(tr) ; + rsGxsTunnel = mGxsTunnels; + + mGxsNetTunnel->connectToTurtleRouter(tr) ; rsDisc = mDisc; rsMsgs = new p3Msgs(msgSrv, chatSrv); // connect components to turtle router. - gxschannels_ns->connectToTurtleRouter(tr) ; ftserver->connectToTurtleRouter(tr) ; ftserver->connectToFileDatabase(fdb) ; chatSrv->connectToGxsTunnelService(mGxsTunnels) ; @@ -1615,6 +1624,7 @@ int RsServer::StartupRetroShare() //mConfigMgr->addConfiguration("ftserver.cfg", ftserver); // mConfigMgr->addConfiguration("gpg_prefs.cfg", AuthGPG::getAuthGPG()); + mConfigMgr->addConfiguration("gxsnettunnel.cfg", mGxsNetTunnel); mConfigMgr->loadConfiguration(); mConfigMgr->addConfiguration("peers.cfg", mPeerMgr); @@ -1824,6 +1834,7 @@ int RsServer::StartupRetroShare() /*** start up GXS core runner ***/ + startServiceThread(mGxsNetTunnel, "gxs net tunnel"); startServiceThread(mGxsIdService, "gxs id"); startServiceThread(mGxsCircles, "gxs circle"); startServiceThread(mPosted, "gxs posted"); From 8d5c013a17b8cce4a4724234f95731f7d08bcfd5 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 3 May 2018 23:21:59 +0200 Subject: [PATCH 35/84] added proper synchronization of GxsIds through tunnels of another service --- libretroshare/src/gxs/rsgxsnetservice.cc | 29 ++++- libretroshare/src/gxs/rsgxsnetservice.h | 1 + libretroshare/src/gxs/rsgxsnettunnel.cc | 149 +++++++++++----------- libretroshare/src/gxs/rsgxsnettunnel.h | 2 + libretroshare/src/gxs/rsnxs.h | 8 ++ libretroshare/src/rsserver/rsinit.cc | 21 +-- libretroshare/src/services/p3idservice.cc | 2 +- 7 files changed, 127 insertions(+), 85 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index e18423c0d..8a46fc89e 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -766,7 +766,7 @@ void RsGxsNetService::generic_sendItem(RsNxsItem *si) void RsGxsNetService::checkDistantSyncState() { - if(!mAllowDistSync || mGxsNetTunnel==NULL) + if(!mAllowDistSync || mGxsNetTunnel==NULL || !mGrpAutoSync) return ; RsGxsGrpMetaTemporaryMap grpMeta; @@ -835,6 +835,17 @@ void RsGxsNetService::syncGrpStatistics() std::set online_peers; mNetMgr->getOnlineList(mServiceInfo.mServiceType, online_peers); + if(mAllowDistSync && mGxsNetTunnel != NULL) + { + // Grab all online virtual peers of distant tunnels for the current service. + + std::list vpids ; + mGxsNetTunnel->getVirtualPeers(vpids); + + for(auto it(vpids.begin());it!=vpids.end();++it) + online_peers.insert(RsPeerId(*it)) ; + } + // Go through group statistics and groups without information are re-requested to random peers selected // among the ones who provided the group info. @@ -4168,7 +4179,7 @@ bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxs // check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for RsGxsGroupId peer_grp ; - if(mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) + if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) { #ifdef NXS_NET_DEBUG_4 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl; @@ -4522,7 +4533,7 @@ bool RsGxsNetService::canSendMsgIds(std::vector& msgMetas, co // check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for RsGxsGroupId peer_grp ; - if(mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) + if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId) { #ifdef NXS_NET_DEBUG_4 GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl; @@ -5064,6 +5075,18 @@ bool RsGxsNetService::removeGroups(const std::list& groups) return true ; } +bool RsGxsNetService::isDistantPeer(const RsPeerId& pid) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + if(!mAllowDistSync || mGxsNetTunnel == NULL) + return false ; + + RsGxsGroupId group_id ; + + return mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(pid),group_id); +} + bool RsGxsNetService::stampMsgServerUpdateTS(const RsGxsGroupId& gid) { RS_STACK_MUTEX(mNxsMutex) ; diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 6ead3f4c7..fe1c77010 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -170,6 +170,7 @@ public: virtual bool getGroupServerUpdateTS(const RsGxsGroupId& gid,time_t& grp_server_update_TS,time_t& msg_server_update_TS) ; virtual bool stampMsgServerUpdateTS(const RsGxsGroupId& gid) ; virtual bool removeGroups(const std::list& groups); + virtual bool isDistantPeer(const RsPeerId& pid); /* p3Config methods */ public: diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index bb1abd6e1..b81b6a7c0 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -275,6 +275,7 @@ bool RsGxsNetTunnelService::requestDistantPeers(uint16_t service_id, const RsGxs ginfo.group_policy = RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE; ginfo.hash = calculateGroupHash(group_id) ; + ginfo.service_id = service_id; mHandledHashes[ginfo.hash] = group_id ; @@ -307,39 +308,6 @@ bool RsGxsNetTunnelService::releaseDistantPeers(uint16_t service_id,const RsGxsG return true; } -const Bias20Bytes& RsGxsNetTunnelService::locked_randomBias() -{ - if(mRandomBias.isNull()) - { -#ifdef DEBUG_RSGXSNETTUNNEL -#warning /!\ this is for testing only! Remove this when done! Can not be done at initialization when rsPeer is not started. - RsPeerId ssl_id = rsPeers->getOwnId() ; - mRandomBias = Bias20Bytes(RsDirUtil::sha1sum(ssl_id.toByteArray(),ssl_id.SIZE_IN_BYTES)) ; -#else - mRandomBias = Bias20Bytes::random(); -#endif - IndicateConfigChanged(); - } - - return mRandomBias ; -} - -RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId(const RsGxsGroupId& group_id) -{ - assert(RsPeerId::SIZE_IN_BYTES <= Sha1CheckSum::SIZE_IN_BYTES) ;// so that we can build the virtual PeerId from a SHA1 sum. - - // We compute sha1( SSL_id | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId - - Bias20Bytes rb(locked_randomBias()); - - unsigned char mem[group_id.SIZE_IN_BYTES + rb.SIZE_IN_BYTES]; - - memcpy(mem ,group_id.toByteArray(),group_id.SIZE_IN_BYTES) ; - memcpy(mem+group_id.SIZE_IN_BYTES,rb.toByteArray(),rb.SIZE_IN_BYTES) ; - - return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,group_id.SIZE_IN_BYTES+rb.SIZE_IN_BYTES).toByteArray()); -} - void RsGxsNetTunnelService::dump() const { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -379,7 +347,7 @@ void RsGxsNetTunnelService::dump() const std::cerr << " Virtual peers: " << std::endl; for(auto it(mVirtualPeers.begin());it!=mVirtualPeers.end();++it) std::cerr << " GXS Peer:" << it->first << " Turtle:" << it->second.turtle_virtual_peer_id - << " status: " << vpid_status_str[it->second.vpid_status] << " direction: " + << " status: " << vpid_status_str[it->second.vpid_status] << " group_id: " << it->second.group_id << " direction: " << (int)it->second.side << " last seen " << time(NULL)-it->second.last_contact << " secs ago" @@ -466,7 +434,9 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co return ; } - if(getRsItemService(getRsItemId(data)) == RS_SERVICE_TYPE_GXS_NET_TUNNEL) + uint16_t service_id = getRsItemService(getRsItemId(data)); + + if(service_id == RS_SERVICE_TYPE_GXS_NET_TUNNEL) { RsItem *decrypted_item = RsGxsNetTunnelSerializer().deserialise(data,&data_size); RsGxsNetTunnelVirtualPeerItem *pid_item = dynamic_cast(decrypted_item) ; @@ -491,52 +461,61 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co vp_info.side = direction; // client/server vp_info.last_contact = time(NULL); // last time some data was sent/recvd vp_info.group_id = group_id; + vp_info.service_id = mGroups[group_id].service_id ; memcpy(vp_info.encryption_master_key,encryption_master_key,RS_GXS_TUNNEL_CONST_EKEY_SIZE); vp_info.turtle_virtual_peer_id = turtle_virtual_peer_id; // turtle peer to use when sending data to this vpid. free(data); + + return ; } - else + + + // item is a generic data item for the client. Let's store the data in the appropriate incoming data queue. + + auto it = mTurtle2GxsPeer.find(turtle_virtual_peer_id) ; + + if(it == mTurtle2GxsPeer.end()) { - // item is a generic data item for the client. Let's store the data in the appropriate incoming data queue. + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for vpid " << turtle_virtual_peer_id << " but this vpid is unknown!" << std::endl; + free(data); + return; + } - auto it = mTurtle2GxsPeer.find(turtle_virtual_peer_id) ; + RsGxsNetTunnelVirtualPeerId gxs_vpid = it->second ; - if(it == mTurtle2GxsPeer.end()) - { - GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for vpid " << turtle_virtual_peer_id << " but this vpid is unknown!" << std::endl; - free(data); - return; - } + auto it2 = mVirtualPeers.find(gxs_vpid) ; - RsGxsNetTunnelVirtualPeerId gxs_vpid = it->second ; + if(it2 == mVirtualPeers.end()) + { + GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for GXS vpid " << gxs_vpid << " but the virtual peer id is missing!" << std::endl; + free(data); + return; + } + it2->second.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; // status of the peer + it2->second.last_contact = time(NULL); // last time some data was sent/recvd - auto it2 = mVirtualPeers.find(gxs_vpid) ; - - if(it2 == mVirtualPeers.end()) - { - GXS_NET_TUNNEL_ERROR() << "item received by GxsNetTunnel for GXS vpid " << gxs_vpid << " but the virtual peer id is missing!" << std::endl; - free(data); - return; - } - it2->second.vpid_status = RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE ; // status of the peer - it2->second.last_contact = time(NULL); // last time some data was sent/recvd + if(service_id != it2->second.service_id && service_id != RS_SERVICE_GXS_TYPE_GXSID) + { + GXS_NET_TUNNEL_ERROR() << " received an item for VPID " << gxs_vpid << " openned for service " << it2->second.service_id << " that is not for that service nor for GXS Id service (service=" << std::hex << service_id << std::dec << ". Rejecting!" << std::endl; + free(data); + return ; + } #ifdef DEBUG_RSGXSNETTUNNEL - GXS_NET_TUNNEL_DEBUG() << "item contains generic data for VPID " << gxs_vpid << ". Storing in incoming list" << std::endl; + GXS_NET_TUNNEL_DEBUG() << "item contains generic data for VPID " << gxs_vpid << ". service_id = " << std::hex << service_id << std::dec << ". Storing in incoming list" << std::endl; #endif - // push the data into the incoming data list + // push the data into the incoming data list - RsTlvBinaryData *bind = new RsTlvBinaryData; - bind->tlvtype = 0; - bind->bin_len = data_size; - bind->bin_data = data; + RsTlvBinaryData *bind = new RsTlvBinaryData; + bind->tlvtype = 0; + bind->bin_len = data_size; + bind->bin_data = data; - mIncomingData.push_back(std::make_pair(gxs_vpid,bind)) ; - } + mIncomingData[service_id].push_back(std::make_pair(gxs_vpid,bind)) ; } void RsGxsNetTunnelService::addVirtualPeer(const TurtleFileHash& hash, const TurtleVirtualPeerId& vpid,RsTurtleGenericTunnelItem::Direction dir) @@ -679,6 +658,40 @@ void RsGxsNetTunnelService::data_tick() rstime::rs_usleep(1*1000*1000) ; // 1 sec } +const Bias20Bytes& RsGxsNetTunnelService::locked_randomBias() +{ + if(mRandomBias.isNull()) + { +#ifdef DEBUG_RSGXSNETTUNNEL +#warning /!\ this is for testing only! Remove this when done! Can not be done at initialization when rsPeer is not started. + RsPeerId ssl_id = rsPeers->getOwnId() ; + mRandomBias = Bias20Bytes(RsDirUtil::sha1sum(ssl_id.toByteArray(),ssl_id.SIZE_IN_BYTES)) ; +#else + mRandomBias = Bias20Bytes::random(); +#endif + IndicateConfigChanged(); + } + + return mRandomBias ; +} + +RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId(const RsGxsGroupId& group_id) +{ + assert(RsPeerId::SIZE_IN_BYTES <= Sha1CheckSum::SIZE_IN_BYTES) ;// so that we can build the virtual PeerId from a SHA1 sum. + + // We compute sha1( SSL_id | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId + + Bias20Bytes rb(locked_randomBias()); + + unsigned char mem[group_id.SIZE_IN_BYTES + rb.SIZE_IN_BYTES]; + + memcpy(mem ,group_id.toByteArray(),group_id.SIZE_IN_BYTES) ; + memcpy(mem+group_id.SIZE_IN_BYTES,rb.toByteArray(),rb.SIZE_IN_BYTES) ; + + return RsGxsNetTunnelVirtualPeerId(RsDirUtil::sha1sum(mem,group_id.SIZE_IN_BYTES+rb.SIZE_IN_BYTES).toByteArray()); +} + + void RsGxsNetTunnelService::sendKeepAlivePackets() { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -834,14 +847,8 @@ bool RsGxsNetTunnelService::loadList(std::list &load) RsSerialiser *RsGxsNetTunnelService::setupSerialiser() { - RS_STACK_MUTEX(mGxsNetTunnelMtx); - static RsSerialiser *ser = NULL ; // this is not so nice, but this method is only called from p3Config, so there's no really need of a data race - - if(!ser) - { - ser = new RsSerialiser ; - ser->addSerialType(new RsGxsNetTunnelSerializer) ; - } + RsSerialiser *ser = new RsSerialiser ; + ser->addSerialType(new RsGxsNetTunnelSerializer) ; return ser ; } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index a36883cbb..abe7aba1c 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -120,6 +120,7 @@ struct RsGxsNetTunnelVirtualPeerInfo TurtleVirtualPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. RsGxsGroupId group_id ; // group that virtual peer is providing + uint16_t service_id ; // this is used for checkng consistency of the incoming data }; struct RsGxsNetTunnelGroupInfo @@ -144,6 +145,7 @@ struct RsGxsNetTunnelGroupInfo GroupStatus group_status ; time_t last_contact ; TurtleFileHash hash ; + uint16_t service_id ; std::set virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. }; diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 283c96e65..faec2577a 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -166,6 +166,14 @@ public: */ virtual bool stampMsgServerUpdateTS(const RsGxsGroupId& gid) =0; + /*! + * \brief isDistantPeer + * \param pid peer that is a virtual peer provided by GxsNetTunnel + * \return + * true if the peer exists (adn therefore is online) + */ + virtual bool isDistantPeer(const RsPeerId& pid)=0; + /*! * \brief removeGroups * Removes time stamp information from the list of groups. This allows to re-sync them if suppliers are present. diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 4140eaa0a..7fd67fa27 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1279,6 +1279,14 @@ int RsServer::StartupRetroShare() RsNxsNetMgr* nxsMgr = new RsNxsNetMgrImpl(serviceCtrl); + /**** GXS Dist sync service ****/ + +#ifdef RS_USE_GXS_DISTANT_SYNC + RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ; +#else + RsGxsNetTunnelService *mGxsNetTunnel = NULL ; +#endif + /**** Identity service ****/ RsGeneralDataService* gxsid_ds = new RsDataService(currGxsDir + "/", "gxsid_db", @@ -1300,9 +1308,10 @@ int RsServer::StartupRetroShare() RS_SERVICE_GXS_TYPE_GXSID, gxsid_ds, nxsMgr, mGxsIdService, mGxsIdService->getServiceInfo(), mReputations, mGxsCircles,mGxsIdService, - pgpAuxUtils,NULL, - false,false); // don't synchronise group automatic (need explicit group request) + pgpAuxUtils,mGxsNetTunnel, + false,false,true); // don't synchronise group automatic (need explicit group request) // don't sync messages at all. + // allow distsync, so that we can grab GXS id requests for other services // Normally we wouldn't need this (we do in other service): // mGxsIdService->setNetworkExchangeService(gxsid_ns) ; @@ -1358,14 +1367,6 @@ int RsServer::StartupRetroShare() mWiki->setNetworkExchangeService(wiki_ns) ; #endif - /**** GXS Dist sync service ****/ - -#ifdef RS_USE_GXS_DISTANT_SYNC - RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ; -#else - RsGxsNetTunnelService *mGxsNetTunnel = NULL ; -#endif - /**** Forum GXS service ****/ RsGeneralDataService* gxsforums_ds = new RsDataService(currGxsDir + "/", "gxsforums_db", diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 51bce5550..5618568fa 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -2700,7 +2700,7 @@ void p3IdService::requestIdsFromNet() bool request_can_proceed = false ; for(cit2 = peers.begin(); cit2 != peers.end(); ++cit2) - if(rsPeers->isOnline(*cit2)) // make sure that the peer in online, so that we know that the request has some chance to succeed. + if(rsPeers->isOnline(*cit2) || mNes->isDistantPeer(*cit2)) // make sure that the peer in online, so that we know that the request has some chance to succeed. { requests[*cit2].push_back(cit->first); request_can_proceed = true ; From 0ada4d48958a4ee049046293969a005b8825c220 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 5 May 2018 18:08:27 +0200 Subject: [PATCH 36/84] improved GxsNetTunnel comment section --- libretroshare/src/gxs/rsgxsnettunnel.h | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index abe7aba1c..46314da6f 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -35,7 +35,7 @@ * to RsGxsNetService. * * It is the responsibility of RsGxsNetService to activate/desactivate tunnels for each particular group, depending on wether the group - * is already available at friends or not. + * is already available at regular friends or not. * * Tunnel management is done at the level of groups rather than services, because we would like to keep the possibility to not * request tunnels for some groups which do not need it, and only request tunnels for specific groups that cannot be provided @@ -47,7 +47,6 @@ // * encrypt tunnel data using chacha20+HMAC-SHA256 using AEAD( GroupId, 96bits IV, tunnel ID ) (similar to what FT does) // * when tunnel is established, exchange virtual peer names: vpid = H( GroupID | Random bias ) // * when vpid is known, notify the client (GXS net service) which can use the virtual peer to sync -// // * only use a single tunnel per virtual peer ID // - // Client ------------------ TR(H(GroupId)) --------------> Server | @@ -67,7 +66,7 @@ // Client <------------------- GXS Data ------------------> Server | // - // Notes: -// * tunnels are established symetrically. If a distant peers wants to sync the same group, they'll have to open a single tunnel, with a different ID. +// * tunnels are established symmetrically. If a distant peers wants to sync the same group, they'll have to open a single tunnel, with a different ID. // Groups therefore have two states: // - managed : the group can be used to answer tunnel requests. If server tunnels are established, the group will be synced with these peers // - tunneled: the group will actively request tunnels. If tunnels are established both ways, the same virtual peers will be used so the tunnels are "merged". @@ -90,13 +89,18 @@ // // * virtual peers are also shared among services. This reduces the required amount of tunnels and tunnel requests to send. // +// * tunnels for a given service may also be used by the net service of p3IdService in order to explicitely request missing GxsIds. +// So GxsNetTunnel somehow allows different services to use the same tunnel. However we make sure that this traffic is limited to only p3IdService. +// // How do we know that a group needs distant sync? // * look into GrpConfigMap for suppliers. Suppliers is cleared at load. // * last_update_TS in GrpConfigMap is randomised so it cannot be used // * we need a way to know that there's no suppliers for good reasons (not that we just started) // // Security -// * +// * the question of sync-ing with distant anonymous peers is a bit tricky because these peers can try to generate fake requests or change which messages are available +// and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen. +// typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; @@ -128,8 +132,7 @@ struct RsGxsNetTunnelGroupInfo enum GroupStatus { RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE = 0x01, // no virtual peers requested, just waiting -// RS_GXS_NET_TUNNEL_GRP_STATUS_TUNNELS_REQUESTED = 0x02, // virtual peers requested, and waiting for turtle to answer - RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x03 // some virtual peers are available. Data can be read/written + RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x02 // some virtual peers are available. Data can be read/written }; enum GroupPolicy { From e7182013bf18f848a42b3f270e9a6e5c0ac35574 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 5 May 2018 18:41:41 +0200 Subject: [PATCH 37/84] added items for generic search result items for GXS --- libretroshare/src/retroshare/rsturtle.h | 8 ++++++ libretroshare/src/turtle/rsturtleitem.h | 37 +++++++++++++++++++++++-- 2 files changed, 43 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/retroshare/rsturtle.h b/libretroshare/src/retroshare/rsturtle.h index 7cef4fd77..4609905bc 100644 --- a/libretroshare/src/retroshare/rsturtle.h +++ b/libretroshare/src/retroshare/rsturtle.h @@ -33,7 +33,9 @@ #include #include +#include "serialiser/rstlvbinary.h" #include "retroshare/rstypes.h" +#include "retroshare/rsgxsifacetypes.h" namespace RsRegularExpression { class LinearizedExpression ; } class RsTurtleClientService ; @@ -52,6 +54,12 @@ struct TurtleFileInfo std::string name ; uint64_t size ; }; +struct TurtleGxsInfo +{ + RsGxsGroupId group_id ; + std::string name ; + RsTlvBinaryData meta ; +}; struct TurtleTunnelRequestDisplayInfo { uint32_t request_id ; // Id of the request diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 248082652..1eded28dd 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -16,7 +16,7 @@ #include "serialiser/rsserializer.h" const uint8_t RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST = 0x01 ; -const uint8_t RS_TURTLE_SUBTYPE_SEARCH_RESULT = 0x02 ; +const uint8_t RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT = 0x02 ; const uint8_t RS_TURTLE_SUBTYPE_OPEN_TUNNEL = 0x03 ; const uint8_t RS_TURTLE_SUBTYPE_TUNNEL_OK = 0x04 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_REQUEST = 0x07 ; @@ -27,6 +27,7 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ; const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ; const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST = 0x15 ; +const uint8_t RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT = 0x16 ; // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ; // unused // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ; @@ -49,7 +50,23 @@ class RsTurtleItem: public RsItem class RsTurtleSearchResultItem: public RsTurtleItem { public: - RsTurtleSearchResultItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_SEARCH_RESULT), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT) ;} + RsTurtleSearchResultItem(uint8_t subtype) : RsTurtleItem(subtype), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT) ;} + + TurtleSearchRequestId request_id ; // Randomly generated request id. + + uint16_t depth ; // The depth of a search result is obfuscated in this way: + // If the actual depth is 1, this field will be 1. + // If the actual depth is > 1, this field is a larger arbitrary integer. + + virtual void clear() =0; + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)=0; + protected: +}; + +class RsTurtleFTSearchResultItem: public RsTurtleSearchResultItem +{ + public: + RsTurtleFTSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT){} TurtleSearchRequestId request_id ; // Randomly generated request id. @@ -61,7 +78,23 @@ class RsTurtleSearchResultItem: public RsTurtleItem void clear() { result.clear() ; } protected: void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; +class RsTurtleGxsSearchResultItem: public RsTurtleSearchResultItem +{ + public: + RsTurtleGxsSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT){} + + TurtleSearchRequestId request_id ; // Randomly generated request id. + + uint16_t depth ; // The depth of a search result is obfuscated in this way: + // If the actual depth is 1, this field will be 1. + // If the actual depth is > 1, this field is a larger arbitrary integer. + std::list result ; + + void clear() { result.clear() ; } + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; class RsTurtleSearchRequestItem: public RsTurtleItem From 90d7f55c404cca1829f8b978d084f5dbfd271bc3 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 31 May 2018 15:41:54 +0200 Subject: [PATCH 38/84] finished implementing GXS search items --- libretroshare/src/pqi/p3notify.cc | 3 +- libretroshare/src/pqi/p3notify.h | 1 + libretroshare/src/retroshare/rsnotify.h | 1 + libretroshare/src/retroshare/rsturtle.h | 3 +- libretroshare/src/turtle/p3turtle.cc | 206 +++++++++++++---------- libretroshare/src/turtle/rsturtleitem.cc | 67 +++++++- libretroshare/src/turtle/rsturtleitem.h | 195 +++++++++++++-------- retroshare-gui/src/gui/notifyqt.cpp | 5 + retroshare-gui/src/gui/notifyqt.h | 1 + 9 files changed, 322 insertions(+), 160 deletions(-) diff --git a/libretroshare/src/pqi/p3notify.cc b/libretroshare/src/pqi/p3notify.cc index 43caf9758..2461d5f1c 100644 --- a/libretroshare/src/pqi/p3notify.cc +++ b/libretroshare/src/pqi/p3notify.cc @@ -231,7 +231,8 @@ void p3Notify::notifyChatLobbyTimeShift (int time_shift) void p3Notify::notifyCustomState (const std::string& peer_id , const std::string& status_string ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyCustomState (peer_id,status_string) ; } void p3Notify::notifyHashingInfo (uint32_t type , const std::string& fileinfo ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyHashingInfo (type,fileinfo) ; } void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list& files ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,files) ; } -void p3Notify::notifyPeerHasNewAvatar (std::string peer_id ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; } +void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list& groups ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,groups) ; } +void p3Notify::notifyPeerHasNewAvatar (std::string peer_id ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; } void p3Notify::notifyOwnAvatarChanged () { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnAvatarChanged() ; } void p3Notify::notifyOwnStatusMessageChanged() { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnStatusMessageChanged() ; } void p3Notify::notifyDiskFull (uint32_t location , uint32_t size_limit_in_MB ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyDiskFull (location,size_limit_in_MB) ; } diff --git a/libretroshare/src/pqi/p3notify.h b/libretroshare/src/pqi/p3notify.h index 820c4a6bb..22e1f143d 100644 --- a/libretroshare/src/pqi/p3notify.h +++ b/libretroshare/src/pqi/p3notify.h @@ -106,6 +106,7 @@ class p3Notify: public RsNotify void notifyCustomState (const std::string& /* peer_id */, const std::string& /* status_string */) ; void notifyHashingInfo (uint32_t /* type */, const std::string& /* fileinfo */) ; void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* files */) ; + void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* groups */) ; void notifyPeerHasNewAvatar (std::string /* peer_id */) ; void notifyOwnAvatarChanged () ; void notifyOwnStatusMessageChanged () ; diff --git a/libretroshare/src/retroshare/rsnotify.h b/libretroshare/src/retroshare/rsnotify.h index b73c577e9..6d9db3ef7 100644 --- a/libretroshare/src/retroshare/rsnotify.h +++ b/libretroshare/src/retroshare/rsnotify.h @@ -228,6 +228,7 @@ class NotifyClient virtual void notifyCustomState (const std::string& /* peer_id */, const std::string& /* status_string */) {} virtual void notifyHashingInfo (uint32_t /* type */, const std::string& /* fileinfo */) {} virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* files */) {} + virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* groups */) {} virtual void notifyPeerHasNewAvatar (std::string /* peer_id */) {} virtual void notifyOwnAvatarChanged () {} virtual void notifyOwnStatusMessageChanged () {} diff --git a/libretroshare/src/retroshare/rsturtle.h b/libretroshare/src/retroshare/rsturtle.h index 4609905bc..0467b452d 100644 --- a/libretroshare/src/retroshare/rsturtle.h +++ b/libretroshare/src/retroshare/rsturtle.h @@ -57,8 +57,9 @@ struct TurtleFileInfo struct TurtleGxsInfo { RsGxsGroupId group_id ; + uint16_t service_id ; std::string name ; - RsTlvBinaryData meta ; + //RsTlvBinaryData meta ;// is that actually needed? Not sure. Better if it's not. }; struct TurtleTunnelRequestDisplayInfo { diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index ca337f241..11cb8017a 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -71,7 +71,7 @@ void TS_dumpState() ; // an in-depth test would be better to get an idea of what the ideal values // could ever be. // -// update of 14-03-11: +// update of 14-03-11: // - I raised the cache time for tunnel requests. This avoids inconsistencies such as: // * tunnel requests bouncing back while the original request is not in the cache anymore // * special case of this for own file transfer: an outgoing tunnel is built with no end. @@ -155,13 +155,15 @@ void p3turtle::getItemNames(std::map& names) const { names.clear(); - names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST ] = "Search request"; - names[RS_TURTLE_SUBTYPE_SEARCH_RESULT ] = "Search result"; + names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST ] = "Filename substring search request"; + names[RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST ] = "GXS search request"; + names[RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT ] = "File search result"; + names[RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT ] = "GXS search result"; names[RS_TURTLE_SUBTYPE_OPEN_TUNNEL ] = "Tunnel request"; names[RS_TURTLE_SUBTYPE_TUNNEL_OK ] = "Tunnel response"; names[RS_TURTLE_SUBTYPE_FILE_REQUEST ] = "Data request"; names[RS_TURTLE_SUBTYPE_FILE_DATA ] = "Data chunk"; - names[RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST ] = "RegExp search"; + names[RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST ] = "Filename RegExp search request"; names[RS_TURTLE_SUBTYPE_GENERIC_DATA ] = "Generic data"; names[RS_TURTLE_SUBTYPE_FILE_MAP ] = "Chunk map"; names[RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST ] = "Chunk map request"; @@ -169,7 +171,7 @@ void p3turtle::getItemNames(std::map& names) const names[RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST ] = "Chunk CRC request"; } -void p3turtle::setEnabled(bool b) +void p3turtle::setEnabled(bool b) { RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ _turtle_routing_enabled = b; @@ -188,7 +190,7 @@ bool p3turtle::enabled() const } -void p3turtle::setSessionEnabled(bool b) +void p3turtle::setSessionEnabled(bool b) { RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ _turtle_routing_session_enabled = b; @@ -246,7 +248,7 @@ int p3turtle::tick() RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ _last_tunnel_management_time = now ; - // Update traffic statistics. The constants are important: they allow a smooth variation of the + // Update traffic statistics. The constants are important: they allow a smooth variation of the // traffic speed, which is used to moderate tunnel requests statistics. // _traffic_info = _traffic_info*0.9 + _traffic_info_buffer* (0.1 / (float)TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS) ; @@ -374,8 +376,8 @@ void p3turtle::manageTunnels() // - the hash hasn't been tunneled for more than REGULAR_TUNNEL_DIGGING_TIME seconds, even if downloading. // // Candidate hashes are sorted, by olderness. The older gets tunneled first. At most MAX_TUNNEL_REQS_PER_SECOND are - // treated at once, as this method is called every second. - // Note: Because REGULAR_TUNNEL_DIGGING_TIME is larger than EMPTY_TUNNELS_DIGGING_TIME, files being downloaded get + // treated at once, as this method is called every second. + // Note: Because REGULAR_TUNNEL_DIGGING_TIME is larger than EMPTY_TUNNELS_DIGGING_TIME, files being downloaded get // re-tunneled in priority. As this happens less, they don't obliterate tunneling for files that have no tunnels yet. std::vector > hashes_to_digg ; @@ -556,7 +558,7 @@ void p3turtle::autoWash() // Now remove all the virtual peers ids at the client services. Off mutex! // - + for(uint32_t i=0;i::iterator it(_incoming_file_hashes.find(hash)) ; @@ -663,7 +665,7 @@ void p3turtle::locked_closeTunnel(TurtleTunnelId tid,std::vector& load) RsConfigKeyValueSet *vitem = dynamic_cast(*it) ; if(vitem != NULL) - for(std::list::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) + for(std::list::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit) { if(kit->key == "TURTLE_CONFIG_MAX_TR_RATE") { @@ -828,7 +830,7 @@ int p3turtle::handleIncoming() RsTurtleGenericTunnelItem *gti = dynamic_cast(item) ; if(gti != NULL) - routeGenericTunnelItem(gti) ; /// Generic packets, that travel through established tunnels. + routeGenericTunnelItem(gti) ; /// Generic packets, that travel through established tunnels. else /// These packets should be destroyed by the client. { /// Special packets that require specific treatment, because tunnels do not exist for these packets. @@ -840,7 +842,8 @@ int p3turtle::handleIncoming() case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast(item)) ; break ; - case RS_TURTLE_SUBTYPE_SEARCH_RESULT : handleSearchResult(dynamic_cast(item)) ; + case RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT : + case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : handleSearchResult(dynamic_cast(item)) ; break ; case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : handleTunnelRequest(dynamic_cast(item)) ; @@ -867,7 +870,7 @@ int p3turtle::handleIncoming() void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) { RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ - + // take a look at the item and test against inconsistent values // - If the item destimation is @@ -875,7 +878,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) std::cerr << "Received search request from peer " << item->PeerId() << ": " << std::endl ; item->print(std::cerr,0) ; #endif - + uint32_t item_size = RsTurtleSerialiser().size(item); if(item_size > TURTLE_MAX_SEARCH_REQ_ACCEPTED_SERIAL_SIZE) @@ -886,7 +889,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) std::cerr << " Caught a turtle search item with arbitrary large size from " << item->PeerId() << " of size " << item_size << " and depth " << item->depth << ". This is not allowed => dropping." << std::endl; return ; } - + if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE) { #ifdef P3TURTLE_DEBUG @@ -924,49 +927,12 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) #ifdef P3TURTLE_DEBUG std::cerr << " Request not from us. Performing local search" << std::endl ; #endif + std::list search_results ; - std::list result ; + item->performLocalSearch(req,search_results) ; - item->performLocalSearch(result) ; - - RsTurtleSearchResultItem *res_item = NULL ; - uint32_t item_size = 0 ; - -#ifdef P3TURTLE_DEBUG - if(!result.empty()) - std::cerr << " " << result.size() << " matches found. Sending back to origin (" << item->PeerId() << ")." << std::endl ; -#endif - while(!result.empty() && req.result_count < TURTLE_SEARCH_RESULT_MAX_HITS) - { - // Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity. - // - static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ; - - if(res_item == NULL) - { - res_item = new RsTurtleSearchResultItem ; - item_size = 0 ; - - res_item->depth = 0 ; - res_item->request_id = item->request_id ; - res_item->PeerId(item->PeerId()) ; // send back to the same guy - } - res_item->result.push_back(result.front()) ; - - ++req.result_count ; // increase hit number for this particular search request. - - item_size += 8 /* size */ + result.front().hash.serial_size() + result.front().name.size() ; - result.pop_front() ; - - if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || result.empty() || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) - { -#ifdef P3TURTLE_DEBUG - std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ; -#endif - sendItem(res_item) ; - res_item = NULL ; - } - } + for(auto it(search_results.begin());it!=search_results.end();++it) + sendItem(*it) ; } // if enough has been sent back already, do not sarch further @@ -1011,14 +977,14 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) // Copy current item and modify it. RsTurtleSearchRequestItem *fwd_item = item->clone() ; - // increase search depth, except in some rare cases, to prevent correlation between + // increase search depth, except in some rare cases, to prevent correlation between // TR sniffing and friend names. The strategy is to not increase depth if the depth // is 1: // If B receives a TR of depth 1 from A, B cannot deduice that A is downloading the // file, since A might have shifted the depth. // if(!random_dshift) - ++(fwd_item->depth) ; + ++(fwd_item->depth) ; fwd_item->PeerId(*it) ; @@ -1055,7 +1021,7 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) if(it->second.origin == _own_id) { - it->second.result_count += item->result.size() ; + it->second.result_count += item->count() ; returnSearchResult(item) ; // Yes, so send upward. } else @@ -1065,7 +1031,7 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) #endif // We update the total count forwarded back, and chop it to TURTLE_SEARCH_RESULT_MAX_HITS. - uint32_t n = item->result.size(); // not so good! + uint32_t n = item->count(); // not so good! if(it->second.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) { @@ -1076,14 +1042,14 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) if(it->second.result_count + n > TURTLE_SEARCH_RESULT_MAX_HITS) { for(uint32_t i=it->second.result_count + n; i>TURTLE_SEARCH_RESULT_MAX_HITS;--i) - item->result.pop_back() ; + item->pop() ; it->second.result_count = TURTLE_SEARCH_RESULT_MAX_HITS ; } else it->second.result_count += n ; - RsTurtleSearchResultItem *fwd_item = new RsTurtleSearchResultItem(*item) ; // copy the item + RsTurtleSearchResultItem *fwd_item = item->duplicate(); // Normally here, we should setup the forward adress, so that the owner's // of the files found can be further reached by a tunnel. @@ -1154,8 +1120,8 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item) // Let's figure out whether this packet is for us or not. - if(item->PeerId() == tunnel.local_dst && tunnel.local_src != _own_id) //direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT && - { + if(item->PeerId() == tunnel.local_dst && tunnel.local_src != _own_id) //direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT && + { #ifdef P3TURTLE_DEBUG std::cerr << " Forwarding generic item to peer " << tunnel.local_src << std::endl ; #endif @@ -1193,7 +1159,7 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item) // The packet was not forwarded, so it is for us. Let's treat it. // This is done off-mutex, to avoid various deadlocks // - + handleRecvGenericTunnelItem(item) ; delete item ; @@ -1316,13 +1282,13 @@ void p3turtle::sendTurtleData(const RsPeerId& virtual_peer_id,RsTurtleGenericTun if(tunnel.local_src == _own_id) { - item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_SERVER) ; + item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_SERVER) ; item->PeerId(tunnel.local_dst) ; _traffic_info_buffer.data_dn_Bps += ss ; } else if(tunnel.local_dst == _own_id) { - item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ; + item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ; item->PeerId(tunnel.local_src) ; _traffic_info_buffer.data_up_Bps += ss ; } @@ -1460,7 +1426,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item) RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ std::map::iterator it = _tunnel_requests_origins.find(item->request_id) ; - + if(it != _tunnel_requests_origins.end()) { #ifdef P3TURTLE_DEBUG @@ -1563,7 +1529,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item) // - the tunnel id will now be unique for a given route // - allows a better balance of bandwidth for a given transfer // - avoid the waste of items that get lost when re-routing a tunnel - + #ifdef P3TURTLE_DEBUG std::cerr << "Perturbating partial tunnel id. Original=" << std::hex << item->partial_tunnel_id ; #endif @@ -1594,7 +1560,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item) forward_probability = 1.0f / nb_online_ids ; // Setting forward_probability to 1/nb_online_ids forces at most one TR up per TR dn. But if we are overflooded by - // TR dn, we still need to control them to avoid flooding the pqiHandler outqueue. So we additionally moderate the + // TR dn, we still need to control them to avoid flooding the pqiHandler outqueue. So we additionally moderate the // forward probability so as to reduct the output rate accordingly. // if(_traffic_info.tr_dn_Bps / (float)TUNNEL_REQUEST_PACKET_SIZE > _max_tr_up_rate) @@ -1620,7 +1586,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item) // Copy current item and modify it. RsTurtleOpenTunnelItem *fwd_item = new RsTurtleOpenTunnelItem(*item) ; - // increase search depth, except in some rare cases, to prevent correlation between + // increase search depth, except in some rare cases, to prevent correlation between // TR sniffing and friend names. The strategy is to not increase depth if the depth // is 1: // If B receives a TR of depth 1 from A, B cannot deduice that A is downloading the @@ -1777,7 +1743,61 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item) // ------------------------------ IO with libretroshare ----------------------------// // -----------------------------------------------------------------------------------// // -void RsTurtleStringSearchRequestItem::performLocalSearch(std::list& result) const +void RsTurtleFileSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const +{ +#ifdef P3TURTLE_DEBUG + std::cerr << "Performing rsFiles->search()" << std::endl ; +#endif + // now, search! + std::list initialResults ; + search(initialResults) ; + +#ifdef P3TURTLE_DEBUG + std::cerr << initialResults.size() << " matches found." << std::endl ; +#endif + result.clear() ; + RsTurtleFTSearchResultItem *res_item = NULL ; + uint32_t item_size = 0 ; + + static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ; + + for(auto it(initialResults.begin());it!=initialResults.end();++it) + { + if(res_item == NULL) + { + res_item = new RsTurtleFTSearchResultItem ; + item_size = 0 ; + + res_item->depth = 0 ; + res_item->request_id = request_id ; + res_item->PeerId(PeerId()) ; // send back to the same guy + + result.push_back(res_item) ; + } + res_item->result.push_back(*it); + + // Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity. + // + ++req.result_count ; // increase hit number for this particular search request. + + item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ; + + if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + { +#ifdef P3TURTLE_DEBUG + std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ; +#endif + res_item = NULL ; // forces creation of a new item. + } + } +} + +void RsTurtleGxsSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const +{ + std::cerr << "(EE) Missing code to perform actual GXS search" << std::endl; +} + +void RsTurtleStringSearchRequestItem::search(std::list& result) const { /* call to core */ std::list initialResults; @@ -1800,7 +1820,7 @@ void RsTurtleStringSearchRequestItem::performLocalSearch(std::list::const_iterator it(initialResults.begin());it!=initialResults.end();++it) { // retain only file type - if (it->type == DIR_TYPE_DIR) + if (it->type == DIR_TYPE_DIR) { #ifdef P3TURTLE_DEBUG std::cerr << " Skipping directory " << it->name << std::endl ; @@ -1816,7 +1836,7 @@ void RsTurtleStringSearchRequestItem::performLocalSearch(std::list& result) const +void RsTurtleRegExpSearchRequestItem::search(std::list& result) const { /* call to core */ std::list initialResults; @@ -1839,7 +1859,7 @@ void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list::const_iterator it(initialResults.begin());it!=initialResults.end();++it) { // retain only file type - if (it->type == DIR_TYPE_DIR) + if (it->type == DIR_TYPE_DIR) { #ifdef P3TURTLE_DEBUG std::cerr << " Skipping directory " << it->name << std::endl ; @@ -1953,13 +1973,25 @@ void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *clie void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item) { - // just cout for now, but it should be notified to the gui - #ifdef P3TURTLE_DEBUG std::cerr << " Returning result for search request " << HEX_PRINT(item->request_id) << " upwards." << std::endl ; #endif - RsServer::notify()->notifyTurtleSearchResult(item->request_id,item->result) ; + RsTurtleFTSearchResultItem *ft_sr = dynamic_cast(item) ; + + if(ft_sr != NULL) + { + RsServer::notify()->notifyTurtleSearchResult(ft_sr->request_id,ft_sr->result) ; + return ; + } + + RsTurtleGxsSearchResultItem *gxs_sr = dynamic_cast(item) ; + + if(gxs_sr != NULL) + { + RsServer::notify()->notifyTurtleSearchResult(gxs_sr->request_id,gxs_sr->result) ; + return ; + } } /// Warning: this function should never be called while the turtle mutex is locked. @@ -2072,7 +2104,7 @@ std::string p3turtle::getPeerNameForVirtualPeerId(const RsPeerId& virtual_peer_i std::map::const_iterator it(_virtual_peers.find(virtual_peer_id)) ; if(it != _virtual_peers.end()) { - std::map::iterator it2( _local_tunnels.find(it->second) ) ; + std::map::iterator it2( _local_tunnels.find(it->second) ) ; if(it2 != _local_tunnels.end()) { if(it2->second.local_src == _own_id) @@ -2103,7 +2135,7 @@ bool p3turtle::encryptData(const unsigned char *clear_data,uint32_t clear_data_s TURTLE_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) << std::endl; #endif - uint32_t item_serialized_size = clear_data_size;//RsGenericSerializer().size(clear_item) ; + uint32_t item_serialized_size = clear_data_size;//RsGenericSerializer().size(clear_item) ; uint32_t total_data_size = ENCRYPTED_TURTLE_HEADER_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_EDATA_SIZE + item_serialized_size + ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE ; #ifdef SERVER_DEBUG @@ -2124,7 +2156,7 @@ bool p3turtle::encryptData(const unsigned char *clear_data,uint32_t clear_data_s edata[0] = 0xae ; edata[1] = 0xad ; - edata[2] = ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256 ; // means AEAD_chacha20_sha256 + edata[2] = ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256 ; // means AEAD_chacha20_sha256 edata[3] = 0x01 ; offset += ENCRYPTED_TURTLE_HEADER_SIZE; @@ -2141,7 +2173,7 @@ bool p3turtle::encryptData(const unsigned char *clear_data,uint32_t clear_data_s offset += ENCRYPTED_TURTLE_EDATA_SIZE ; - memcpy(&edata[offset],clear_data,clear_data_size); + memcpy(&edata[offset],clear_data,clear_data_size); #ifdef SERVER_DEBUG TURTLE_DEBUG() << " clear item : " << RsUtil::BinToHex(&edata[offset],std::min(50,(int)total_data_size-(int)offset)) << "(...)" << std::endl; @@ -2291,12 +2323,12 @@ void p3turtle::getInfo( std::vector >& hashes_info, tunnel.push_back(printNumber(it->first,true)) ; std::string name; - if(mLinkMgr->getPeerName(it->second.local_src,name)) + if(mLinkMgr->getPeerName(it->second.local_src,name)) tunnel.push_back(name) ; else tunnel.push_back(it->second.local_src.toStdString()) ; - if(mLinkMgr->getPeerName(it->second.local_dst,name)) + if(mLinkMgr->getPeerName(it->second.local_dst,name)) tunnel.push_back(name) ; else tunnel.push_back(it->second.local_dst.toStdString()); diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index 35b250117..ca99b24d4 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -28,7 +28,9 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c { case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST : return new RsTurtleStringSearchRequestItem(); case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST : return new RsTurtleRegExpSearchRequestItem(); - case RS_TURTLE_SUBTYPE_SEARCH_RESULT : return new RsTurtleSearchResultItem(); + case RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST : return new RsTurtleGxsSearchRequestItem(); + case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : return new RsTurtleFTSearchResultItem(); + case RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT : return new RsTurtleGxsSearchResultItem(); case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : return new RsTurtleOpenTunnelItem(); case RS_TURTLE_SUBTYPE_TUNNEL_OK : return new RsTurtleTunnelOkItem(); case RS_TURTLE_SUBTYPE_GENERIC_DATA : return new RsTurtleGenericDataItem(); @@ -61,7 +63,13 @@ void RsTurtleRegExpSearchRequestItem::serial_process(RsGenericSerializer::Serial RsTypeSerializer::serial_process(j,ctx,depth,"depth") ; RsTypeSerializer::serial_process(j,ctx,expr,"expr") ; } - +void RsTurtleGxsSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +{ + RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; + RsTypeSerializer::serial_process(j,ctx,depth,"depth") ; + RsTypeSerializer::serial_process(j,ctx,service_id,"service_id") ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ; +} template<> uint32_t RsTypeSerializer::serial_size(const RsRegularExpression::LinearizedExpression& r) { uint32_t s = 0 ; @@ -140,13 +148,18 @@ template<> void RsTypeSerializer::print_data(const std::string& n, const RsRegul std::cerr << " [RegExpr ] " << n << ", tokens=" << expr._tokens.size() << " ints=" << expr._ints.size() << " strings=" << expr._strings.size() << std::endl; } -void RsTurtleSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +void RsTurtleFTSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +{ + RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; + RsTypeSerializer::serial_process(j,ctx,depth ,"depth") ; + RsTypeSerializer::serial_process (j,ctx,result ,"result") ; +} +void RsTurtleGxsSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; RsTypeSerializer::serial_process(j,ctx,depth ,"depth") ; RsTypeSerializer::serial_process (j,ctx,result ,"result") ; } - template<> uint32_t RsTypeSerializer::serial_size(const TurtleFileInfo& i) { uint32_t s = 0 ; @@ -193,6 +206,52 @@ template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleF std::cerr << " [FileInfo ] " << n << " size=" << i.size << " hash=" << i.hash << ", name=" << i.name << std::endl; } +template<> uint32_t RsTypeSerializer::serial_size(const TurtleGxsInfo& i) +{ + uint32_t s = 0 ; + + s += 2 ; // service_id + s += i.group_id.SIZE_IN_BYTES ; + s += GetTlvStringSize(i.name) ; + + return s; +} + +template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,TurtleGxsInfo& i) +{ + uint32_t saved_offset = offset ; + bool ok = true ; + + ok &= getRawUInt16(data, size, &offset, &i.service_id); // service_id + ok &= i.group_id.deserialise(data, size, offset); // group_id + ok &= GetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name); // group name + + if(!ok) + offset = saved_offset ; + + return ok; +} + +template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const TurtleGxsInfo& i) +{ + uint32_t saved_offset = offset ; + bool ok = true ; + + ok &= setRawUInt16(data, size, &offset, i.service_id); // service_id + ok &= i.group_id.serialise(data, size, offset); // group_id + ok &= SetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name); // group name + + if(!ok) + offset = saved_offset ; + + return ok; +} + +template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleGxsInfo& i) +{ + std::cerr << " [GXS Info ] " << n << " group_id=" << i.group_id << " service=" << std::hex << i.service_id << std::dec << ", name=" << i.name << std::endl; +} + void RsTurtleOpenTunnelItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process (j,ctx,file_hash ,"file_hash") ; diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 1eded28dd..1353a77c5 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -23,6 +23,7 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_REQUEST = 0x07 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA = 0x08 ; const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST = 0x09 ; const uint8_t RS_TURTLE_SUBTYPE_GENERIC_DATA = 0x0a ; +const uint8_t RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST = 0x0b ; const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ; const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ; @@ -32,6 +33,8 @@ const uint8_t RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT = 0x16 ; // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ; // unused // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ; +class TurtleSearchRequestInfo ; + /***********************************************************************************/ /* Basic Turtle Item Class */ /***********************************************************************************/ @@ -47,92 +50,81 @@ class RsTurtleItem: public RsItem /* Specific packets */ /***********************************************************************************/ -class RsTurtleSearchResultItem: public RsTurtleItem -{ - public: - RsTurtleSearchResultItem(uint8_t subtype) : RsTurtleItem(subtype), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT) ;} +// Class hierarchy is +// +// RsTurtleItem +// | +// +---- RsTurtleSearchRequestItem +// | | +// | +---- RsTurtleFileSearchRequestItem +// | | | +// | | +---- RsTurtleFileSearchRequestItem +// | | | +// | | +---- RsTurtleStringSearchRequestItem +// | | | +// | | +---- RsTurtleReqExpSearchRequestItem +// | | +// | +---- RsTurtleGxsSearchRequestItem +// | +// +---- RsTurtleSearchResultItem +// | +// +---- RsTurtleFTSearchResultItem +// | +// +---- RsTurtleGxsSearchResultItem +// - TurtleSearchRequestId request_id ; // Randomly generated request id. - - uint16_t depth ; // The depth of a search result is obfuscated in this way: - // If the actual depth is 1, this field will be 1. - // If the actual depth is > 1, this field is a larger arbitrary integer. - - virtual void clear() =0; - virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)=0; - protected: -}; - -class RsTurtleFTSearchResultItem: public RsTurtleSearchResultItem -{ - public: - RsTurtleFTSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT){} - - TurtleSearchRequestId request_id ; // Randomly generated request id. - - uint16_t depth ; // The depth of a search result is obfuscated in this way: - // If the actual depth is 1, this field will be 1. - // If the actual depth is > 1, this field is a larger arbitrary integer. - std::list result ; - - void clear() { result.clear() ; } - protected: - void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); -}; - -class RsTurtleGxsSearchResultItem: public RsTurtleSearchResultItem -{ - public: - RsTurtleGxsSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT){} - - TurtleSearchRequestId request_id ; // Randomly generated request id. - - uint16_t depth ; // The depth of a search result is obfuscated in this way: - // If the actual depth is 1, this field will be 1. - // If the actual depth is > 1, this field is a larger arbitrary integer. - std::list result ; - - void clear() { result.clear() ; } - protected: - void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); -}; +class RsTurtleSearchResultItem ; class RsTurtleSearchRequestItem: public RsTurtleItem { public: RsTurtleSearchRequestItem(uint32_t subtype) : RsTurtleItem(subtype), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_REQUEST) ;} + virtual ~RsTurtleSearchRequestItem() {} + virtual RsTurtleSearchRequestItem *clone() const = 0 ; // used for cloning in routing methods - virtual void performLocalSearch(std::list&) const = 0 ; // abstracts the search method - + virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const = 0 ; // abstracts the search method + virtual std::string GetKeywords() = 0; - + uint32_t request_id ; // randomly generated request id. uint16_t depth ; // Used for limiting search depth. }; -class RsTurtleStringSearchRequestItem: public RsTurtleSearchRequestItem +class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem { public: - RsTurtleStringSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST) {} - + RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {} + virtual ~RsTurtleFileSearchRequestItem() {} + + virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method + virtual void search(std::list &) const =0; +}; + +class RsTurtleStringSearchRequestItem: public RsTurtleFileSearchRequestItem +{ + public: + RsTurtleStringSearchRequestItem() : RsTurtleFileSearchRequestItem(RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST) {} + virtual ~RsTurtleStringSearchRequestItem() {} + std::string match_string ; // string to match - + std::string GetKeywords() { return match_string; } - + virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleStringSearchRequestItem(*this) ; } - virtual void performLocalSearch(std::list&) const ; void clear() { match_string.clear() ; } protected: + virtual void search(std::list &) const ; void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; -class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem +class RsTurtleRegExpSearchRequestItem: public RsTurtleFileSearchRequestItem { public: - RsTurtleRegExpSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {} + RsTurtleRegExpSearchRequestItem() : RsTurtleFileSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {} + virtual ~RsTurtleRegExpSearchRequestItem() {} RsRegularExpression::LinearizedExpression expr ; // Reg Exp in linearised mode @@ -143,15 +135,84 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem delete ex; return exs; } - - virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; } - virtual void performLocalSearch(std::list&) const ; + virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; } void clear() { expr = RsRegularExpression::LinearizedExpression(); } + protected: + virtual void search(std::list &) const ; + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; + +class RsTurtleGxsSearchRequestItem: public RsTurtleSearchRequestItem +{ + public: + RsTurtleGxsSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST) {} + virtual ~RsTurtleGxsSearchRequestItem() {} + + virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method + + std::string match_string ; // string to match + uint16_t service_id ; // searvice to search + + std::string GetKeywords() { return match_string; } + + virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleGxsSearchRequestItem(*this) ; } + + void clear() { match_string.clear() ; } + protected: void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; +class RsTurtleSearchResultItem: public RsTurtleItem +{ + public: + RsTurtleSearchResultItem(uint8_t subtype) : RsTurtleItem(subtype), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT) ;} + + TurtleSearchRequestId request_id ; // Randomly generated request id. + uint16_t depth ; // The depth of a search result is obfuscated in this way: + // If the actual depth is 1, this field will be 1. + // If the actual depth is > 1, this field is a larger arbitrary integer. + + virtual void clear() =0; + virtual uint32_t count() const =0; + virtual void pop() =0; + + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)=0; + virtual RsTurtleSearchResultItem *duplicate() const =0; +}; + +class RsTurtleFTSearchResultItem: public RsTurtleSearchResultItem +{ + public: + RsTurtleFTSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT){} + + std::list result ; + + void clear() { result.clear() ; } + uint32_t count() const { return result.size() ; } + virtual void pop() { result.pop_back() ;} + virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleFTSearchResultItem(*this) ; } + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; + +class RsTurtleGxsSearchResultItem: public RsTurtleSearchResultItem +{ + public: + RsTurtleGxsSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT){} + + std::list result ; + + void clear() { result.clear() ; } + uint32_t count() const { return result.size() ; } + virtual void pop() { result.pop_back() ;} + virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleGxsSearchResultItem(*this) ; } + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; + + /***********************************************************************************/ /* Turtle Tunnel Item classes */ /***********************************************************************************/ @@ -200,24 +261,24 @@ class RsTurtleGenericTunnelItem: public RsTurtleItem /// Does this packet stamps tunnels when it passes through ? /// This is used for keeping trace weither tunnels are active or not. - + virtual bool shouldStampTunnel() const = 0 ; - /// All tunnels derived from RsTurtleGenericTunnelItem should have a tunnel id to + /// All tunnels derived from RsTurtleGenericTunnelItem should have a tunnel id to /// indicate which tunnel they are travelling through. - + virtual TurtleTunnelId tunnelId() const { return tunnel_id ; } /// Indicate weither the packet is a client packet (goign back to the /// client) or a server packet (going to the server. Typically file /// requests are server packets, whereas file data are client packets. - + virtual Direction travelingDirection() const { return direction ; } virtual void setTravelingDirection(Direction d) { direction = d; } Direction direction ; // This does not need to be serialised. It's only used by the client services, optionnally, // and is set by the turtle router according to which direction the item travels. - + uint32_t tunnel_id ; // Id of the tunnel to travel through }; diff --git a/retroshare-gui/src/gui/notifyqt.cpp b/retroshare-gui/src/gui/notifyqt.cpp index 3810fefc3..da8dac48d 100644 --- a/retroshare-gui/src/gui/notifyqt.cpp +++ b/retroshare-gui/src/gui/notifyqt.cpp @@ -563,6 +563,11 @@ void NotifyQt::notifyChatCleared(const ChatId& chat_id) emit chatCleared(chat_id); } +void NotifyQt::notifyTurtleSearchResult(uint32_t search_id,const std::list& found_groups) +{ + std::cerr << "(EE) missing code to handle GXS turtle search result." << std::endl; +} + void NotifyQt::notifyTurtleSearchResult(uint32_t search_id,const std::list& files) { { diff --git a/retroshare-gui/src/gui/notifyqt.h b/retroshare-gui/src/gui/notifyqt.h index b179d9999..597231703 100644 --- a/retroshare-gui/src/gui/notifyqt.h +++ b/retroshare-gui/src/gui/notifyqt.h @@ -47,6 +47,7 @@ class NotifyQt: public QObject, public NotifyClient virtual void notifyCustomState(const std::string& peer_id, const std::string& status_string); virtual void notifyHashingInfo(uint32_t type, const std::string& fileinfo); virtual void notifyTurtleSearchResult(uint32_t search_id,const std::list& found_files); + virtual void notifyTurtleSearchResult(uint32_t search_id,const std::list& found_groups); virtual void notifyPeerHasNewAvatar(std::string peer_id) ; virtual void notifyOwnAvatarChanged() ; virtual void notifyChatLobbyEvent(uint64_t /* lobby id */, uint32_t /* event type */, const RsGxsId & /*nickname*/, const std::string& /* any string */) ; From b5c1b8210be58db6a4416599fccc9dcab54ca453 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 2 Jun 2018 18:12:27 +0200 Subject: [PATCH 39/84] added classes for both group data and group summary results in turtle search --- libretroshare/src/turtle/p3turtle.cc | 28 +++++++++--- libretroshare/src/turtle/rsturtleitem.cc | 24 +++++++++-- libretroshare/src/turtle/rsturtleitem.h | 54 +++++++++++++++++++++--- 3 files changed, 89 insertions(+), 17 deletions(-) diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 11cb8017a..5208b32d8 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -158,7 +158,8 @@ void p3turtle::getItemNames(std::map& names) const names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST ] = "Filename substring search request"; names[RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST ] = "GXS search request"; names[RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT ] = "File search result"; - names[RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT ] = "GXS search result"; + names[RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY ] = "GXS group summary"; + names[RS_TURTLE_SUBTYPE_GXS_GROUP_DATA ] = "GXS group data"; names[RS_TURTLE_SUBTYPE_OPEN_TUNNEL ] = "Tunnel request"; names[RS_TURTLE_SUBTYPE_TUNNEL_OK ] = "Tunnel response"; names[RS_TURTLE_SUBTYPE_FILE_REQUEST ] = "Data request"; @@ -842,7 +843,8 @@ int p3turtle::handleIncoming() case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast(item)) ; break ; - case RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT : + case RS_TURTLE_SUBTYPE_GXS_GROUP_DATA : + case RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY : case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : handleSearchResult(dynamic_cast(item)) ; break ; @@ -1794,9 +1796,15 @@ void RsTurtleFileSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo & void RsTurtleGxsSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const { - std::cerr << "(EE) Missing code to perform actual GXS search" << std::endl; + std::cerr << "(EE) p3turtle: Missing code to perform actual GXS search" << std::endl; } +void RsTurtleGxsGroupRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const +{ + std::cerr << "(EE) p3turtle: Missing code to perform actual retrieval of GXS group" << std::endl; +} + + void RsTurtleStringSearchRequestItem::search(std::list& result) const { /* call to core */ @@ -1985,11 +1993,19 @@ void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item) return ; } - RsTurtleGxsSearchResultItem *gxs_sr = dynamic_cast(item) ; + RsTurtleGxsSearchResultGroupSummaryItem *gxs_sr_gs = dynamic_cast(item) ; - if(gxs_sr != NULL) + if(gxs_sr_gs != NULL) { - RsServer::notify()->notifyTurtleSearchResult(gxs_sr->request_id,gxs_sr->result) ; + RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gs->request_id,gxs_sr_gs->result) ; + return ; + } + RsTurtleGxsSearchResultGroupDataItem *gxs_sr_gd = dynamic_cast(item) ; + + if(gxs_sr_gd != NULL) + { +#warning MISSING CODE HERE TO HANDLE ENCRYPTED INCOMING GROUP DATA. + //RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gd->request_id,gxs_sr_gd->encrypted_nxs_group) ; return ; } } diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index ca99b24d4..175996575 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -28,13 +28,16 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c { case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST : return new RsTurtleStringSearchRequestItem(); case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST : return new RsTurtleRegExpSearchRequestItem(); - case RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST : return new RsTurtleGxsSearchRequestItem(); case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : return new RsTurtleFTSearchResultItem(); - case RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT : return new RsTurtleGxsSearchResultItem(); case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : return new RsTurtleOpenTunnelItem(); case RS_TURTLE_SUBTYPE_TUNNEL_OK : return new RsTurtleTunnelOkItem(); case RS_TURTLE_SUBTYPE_GENERIC_DATA : return new RsTurtleGenericDataItem(); + case RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST : return new RsTurtleGxsSearchRequestItem(); + case RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST : return new RsTurtleGxsGroupRequestItem(); + case RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY : return new RsTurtleGxsSearchResultGroupSummaryItem(); + case RS_TURTLE_SUBTYPE_GXS_GROUP_DATA : return new RsTurtleGxsSearchResultGroupDataItem(); + default: break ; } @@ -56,7 +59,6 @@ void RsTurtleStringSearchRequestItem::serial_process(RsGenericSerializer::Serial RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; RsTypeSerializer::serial_process(j,ctx,depth ,"depth") ; } - void RsTurtleRegExpSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; @@ -70,6 +72,14 @@ void RsTurtleGxsSearchRequestItem::serial_process(RsGenericSerializer::Serialize RsTypeSerializer::serial_process(j,ctx,service_id,"service_id") ; RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ; } +void RsTurtleGxsGroupRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +{ + RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; + RsTypeSerializer::serial_process(j,ctx,depth,"depth") ; + RsTypeSerializer::serial_process(j,ctx,service_id,"service_id") ; + RsTypeSerializer::serial_process(j,ctx,hashed_group_id,"hashed_group_id") ; +} + template<> uint32_t RsTypeSerializer::serial_size(const RsRegularExpression::LinearizedExpression& r) { uint32_t s = 0 ; @@ -154,12 +164,18 @@ void RsTurtleFTSearchResultItem::serial_process(RsGenericSerializer::SerializeJo RsTypeSerializer::serial_process(j,ctx,depth ,"depth") ; RsTypeSerializer::serial_process (j,ctx,result ,"result") ; } -void RsTurtleGxsSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +void RsTurtleGxsSearchResultGroupSummaryItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; RsTypeSerializer::serial_process(j,ctx,depth ,"depth") ; RsTypeSerializer::serial_process (j,ctx,result ,"result") ; } +void RsTurtleGxsSearchResultGroupDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +{ + RsTypeSerializer::serial_process (j,ctx,request_id,"request_id") ; + RsTypeSerializer::serial_process (j,ctx,depth ,"depth") ; + RsTypeSerializer::serial_process(j,ctx,encrypted_nxs_group,"encrypted_nxs_group") ; +} template<> uint32_t RsTypeSerializer::serial_size(const TurtleFileInfo& i) { uint32_t s = 0 ; diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 1353a77c5..885ab04de 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -24,11 +24,13 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA = 0x08 ; const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST = 0x09 ; const uint8_t RS_TURTLE_SUBTYPE_GENERIC_DATA = 0x0a ; const uint8_t RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST = 0x0b ; +const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST = 0x0c ; const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ; const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ; const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST = 0x15 ; -const uint8_t RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT = 0x16 ; +const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY = 0x16 ; +const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_DATA = 0x17 ; // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ; // unused // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ; @@ -65,12 +67,18 @@ class RsTurtleItem: public RsItem // | | +---- RsTurtleReqExpSearchRequestItem // | | // | +---- RsTurtleGxsSearchRequestItem +// | | +// | +---- RsTurtleGxsGroupRequestItem // | // +---- RsTurtleSearchResultItem // | // +---- RsTurtleFTSearchResultItem // | // +---- RsTurtleGxsSearchResultItem +// | +// +---- RsTurtleGxsSearchResultGroupSummaryItem +// | +// +---- RsTurtleGxsSearchResultGroupDataItem // class RsTurtleSearchResultItem ; @@ -149,21 +157,38 @@ class RsTurtleGxsSearchRequestItem: public RsTurtleSearchRequestItem RsTurtleGxsSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST) {} virtual ~RsTurtleGxsSearchRequestItem() {} - virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method - std::string match_string ; // string to match uint16_t service_id ; // searvice to search std::string GetKeywords() { return match_string; } + virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleGxsSearchRequestItem(*this) ; } - void clear() { match_string.clear() ; } protected: void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; +class RsTurtleGxsGroupRequestItem: public RsTurtleSearchRequestItem +{ + public: + RsTurtleGxsGroupRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST) {} + virtual ~RsTurtleGxsGroupRequestItem() {} + + uint16_t service_id ; // searvice to search + Sha1CheckSum hashed_group_id ; // the group ID is hashed in order to keep it private. + + virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleGxsGroupRequestItem(*this) ; } + + virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method + void clear() { hashed_group_id.clear() ; } + std::string GetKeywords() { return hashed_group_id.toStdString(); } + + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; + class RsTurtleSearchResultItem: public RsTurtleItem { public: @@ -197,21 +222,36 @@ class RsTurtleFTSearchResultItem: public RsTurtleSearchResultItem void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; -class RsTurtleGxsSearchResultItem: public RsTurtleSearchResultItem +class RsTurtleGxsSearchResultGroupSummaryItem: public RsTurtleSearchResultItem { public: - RsTurtleGxsSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_SEARCH_RESULT){} + RsTurtleGxsSearchResultGroupSummaryItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY){} + virtual ~RsTurtleGxsSearchResultGroupSummaryItem() {} std::list result ; void clear() { result.clear() ; } uint32_t count() const { return result.size() ; } virtual void pop() { result.pop_back() ;} - virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleGxsSearchResultItem(*this) ; } + virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleGxsSearchResultGroupSummaryItem(*this) ; } protected: void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; +class RsTurtleGxsSearchResultGroupDataItem: public RsTurtleSearchResultItem +{ + public: + RsTurtleGxsSearchResultGroupDataItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_GROUP_DATA){} + virtual ~RsTurtleGxsSearchResultGroupDataItem() {} + RsTlvBinaryData encrypted_nxs_group; // data is encrypted with group ID. + + uint32_t count() const { return 1 ; } + virtual void pop() { clear(); } + void clear() { encrypted_nxs_group.TlvClear() ; } + virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleGxsSearchResultGroupDataItem(*this) ; } + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; /***********************************************************************************/ /* Turtle Tunnel Item classes */ From 9a64f80182f8c03fe65586f4bc098f65decdd268 Mon Sep 17 00:00:00 2001 From: sehraf Date: Sun, 3 Jun 2018 17:11:11 +0200 Subject: [PATCH 40/84] handle backspace when entering password --- retroshare-nogui/src/TerminalApiClient.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/retroshare-nogui/src/TerminalApiClient.cpp b/retroshare-nogui/src/TerminalApiClient.cpp index f4ffa4af1..b90ed7dbc 100644 --- a/retroshare-nogui/src/TerminalApiClient.cpp +++ b/retroshare-nogui/src/TerminalApiClient.cpp @@ -103,6 +103,15 @@ static std::string readStringFromKeyboard(bool passwd_mode) while((c=getchar()) != '\n') { + // handle backspace + if (c == 127) { + if(s.length()!=0) { + std::cout << "\b \b"; + s.resize(s.length()-1); + } + continue; + } + if(passwd_mode) putchar('*') ; else From 7caf06b57d69e834f234909d75ff8cef8d761b3b Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 6 Jun 2018 23:15:29 +0200 Subject: [PATCH 41/84] added structures for generic turtle search and access functions in Gxs objects --- libresapi/src/api/FileSearchHandler.cpp | 4 +- libretroshare/src/ft/ftserver.cc | 43 +++- libretroshare/src/ft/ftserver.h | 8 +- libretroshare/src/grouter/p3grouter.cc | 7 +- libretroshare/src/grouter/p3grouter.h | 2 +- libretroshare/src/gxs/rsgenexchange.cc | 8 + libretroshare/src/gxs/rsgenexchange.h | 9 + libretroshare/src/gxs/rsgxsnetservice.cc | 9 + libretroshare/src/gxs/rsgxsnetservice.h | 2 + libretroshare/src/gxs/rsgxsnettunnel.cc | 24 ++- libretroshare/src/gxs/rsgxsnettunnel.h | 22 +- libretroshare/src/gxs/rsnxs.h | 2 + libretroshare/src/gxstunnel/p3gxstunnel.cc | 4 +- libretroshare/src/gxstunnel/p3gxstunnel.h | 2 +- libretroshare/src/pqi/p3notify.cc | 3 +- libretroshare/src/pqi/p3notify.h | 3 +- libretroshare/src/retroshare/rsfiles.h | 3 + libretroshare/src/retroshare/rsgxschannels.h | 3 + libretroshare/src/retroshare/rsnotify.h | 3 +- libretroshare/src/retroshare/rsturtle.h | 11 +- libretroshare/src/rsitems/rsgxsitems.cc | 56 +++++ libretroshare/src/rsitems/rsgxsitems.h | 76 +++++++ libretroshare/src/services/p3gxschannels.cc | 7 + libretroshare/src/services/p3gxschannels.h | 3 + libretroshare/src/turtle/p3turtle.cc | 198 +++++++++++------- libretroshare/src/turtle/p3turtle.h | 19 +- libretroshare/src/turtle/rsturtleitem.cc | 94 +++------ libretroshare/src/turtle/rsturtleitem.h | 95 +++------ .../src/turtle/turtleclientservice.h | 47 ++++- .../src/gui/FileTransfer/SearchDialog.cpp | 6 +- retroshare-gui/src/gui/notifyqt.h | 2 + 31 files changed, 513 insertions(+), 262 deletions(-) diff --git a/libresapi/src/api/FileSearchHandler.cpp b/libresapi/src/api/FileSearchHandler.cpp index 0382005df..eb414fc9f 100644 --- a/libresapi/src/api/FileSearchHandler.cpp +++ b/libresapi/src/api/FileSearchHandler.cpp @@ -190,9 +190,9 @@ void FileSearchHandler::handleCreateSearch(Request &req, Response &resp) // i have no idea what the reasons for two different search modes are // rs-gui does it, so do we if(words.size() == 1) - search_id = mTurtle->turtleSearch(words.front()); + search_id = rsFiles->turtleSearch(words.front()); else - search_id = mTurtle->turtleSearch(lin_exp); + search_id = rsFiles->turtleSearch(lin_exp); } std::list results; diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 47718caaa..3172cfd89 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -1317,7 +1317,7 @@ bool ftServer::encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHas // Decrypts the given item using aead-chacha20-poly1305 -bool ftServer::decryptItem(RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item) +bool ftServer::decryptItem(const RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item) { #ifndef USE_NEW_METHOD unsigned char *data = NULL ; @@ -1453,9 +1453,34 @@ bool ftServer::findRealHash(const RsFileHash& hash, RsFileHash& real_hash) return false ; } +TurtleSearchRequestId ftServer::turtleSearch(const std::string& string_to_match) +{ + return mTurtleRouter->turtleSearch(string_to_match) ; +} +TurtleSearchRequestId ftServer::turtleSearch(const RsRegularExpression::LinearizedExpression& expr) +{ + return mTurtleRouter->turtleSearch(expr) ; +} + +#warning we should do this here, but for now it's done by turtle router. +// // Dont delete the item. The client (p3turtle) is doing it after calling this. +// // +// void ftServer::receiveSearchResult(RsTurtleSearchResultItem *item) +// { +// RsTurtleFTSearchResultItem *ft_sr = dynamic_cast(item) ; +// +// if(ft_sr == NULL) +// { +// FTSERVER_ERROR() << "(EE) ftServer::receiveSearchResult(): item cannot be cast to a RsTurtleFTSearchResultItem" << std::endl; +// return ; +// } +// +// RsServer::notify()->notifyTurtleSearchResult(ft_sr->request_id,ft_sr->result) ; +// } + // Dont delete the item. The client (p3turtle) is doing it after calling this. // -void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, +void ftServer::receiveTurtleData(const RsTurtleGenericTunnelItem *i, const RsFileHash& hash, const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem::Direction direction) @@ -1475,7 +1500,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, } RsTurtleGenericTunnelItem *decrypted_item ; - if(!decryptItem(dynamic_cast(i),real_hash,decrypted_item)) + if(!decryptItem(dynamic_cast(i),real_hash,decrypted_item)) { FTSERVER_ERROR() << "(EE) decryption error." << std::endl; return ; @@ -1491,7 +1516,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, { case RS_TURTLE_SUBTYPE_FILE_REQUEST: { - RsTurtleFileRequestItem *item = dynamic_cast(i) ; + const RsTurtleFileRequestItem *item = dynamic_cast(i) ; if (item) { #ifdef SERVER_DEBUG @@ -1504,7 +1529,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, case RS_TURTLE_SUBTYPE_FILE_DATA : { - RsTurtleFileDataItem *item = dynamic_cast(i) ; + const RsTurtleFileDataItem *item = dynamic_cast(i) ; if (item) { #ifdef SERVER_DEBUG @@ -1512,7 +1537,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, #endif getMultiplexer()->recvData(virtual_peer_id,hash,0,item->chunk_offset,item->chunk_size,item->chunk_data) ; - item->chunk_data = NULL ; // this prevents deletion in the destructor of RsFileDataItem, because data will be deleted + const_cast(item)->chunk_data = NULL ; // this prevents deletion in the destructor of RsFileDataItem, because data will be deleted // down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData } } @@ -1520,7 +1545,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, case RS_TURTLE_SUBTYPE_FILE_MAP : { - RsTurtleFileMapItem *item = dynamic_cast(i) ; + const RsTurtleFileMapItem *item = dynamic_cast(i) ; if (item) { #ifdef SERVER_DEBUG @@ -1543,7 +1568,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, case RS_TURTLE_SUBTYPE_CHUNK_CRC : { - RsTurtleChunkCrcItem *item = dynamic_cast(i) ; + const RsTurtleChunkCrcItem *item = dynamic_cast(i) ; if (item) { #ifdef SERVER_DEBUG @@ -1556,7 +1581,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i, case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST: { - RsTurtleChunkCrcRequestItem *item = dynamic_cast(i) ; + const RsTurtleChunkCrcRequestItem *item = dynamic_cast(i) ; if (item) { #ifdef SERVER_DEBUG diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 191b201b4..0833127cf 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -96,7 +96,8 @@ public: // Implements RsTurtleClientService // virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ; - virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; + virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; + //virtual void receiveSearchResult(RsTurtleSearchResultItem *item);// TODO virtual RsItem *create_item(uint16_t service,uint8_t item_type) const ; virtual RsServiceSerializer *serializer() { return this ; } @@ -143,6 +144,9 @@ public: virtual void setFilePermDirectDL(uint32_t perm) ; virtual uint32_t filePermDirectDL() ; + virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ; + virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ; + /*** * Control of Downloads Priority. ***/ @@ -250,7 +254,7 @@ public: static void deriveEncryptionKey(const RsFileHash& hash, uint8_t *key); bool encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHash& hash,RsTurtleGenericDataItem *& encrypted_item); - bool decryptItem(RsTurtleGenericDataItem *encrypted_item, const RsFileHash& hash, RsTurtleGenericTunnelItem *&decrypted_item); + bool decryptItem(const RsTurtleGenericDataItem *encrypted_item, const RsFileHash& hash, RsTurtleGenericTunnelItem *&decrypted_item); /*************** Internal Transfer Fns *************************/ virtual int tick(); diff --git a/libretroshare/src/grouter/p3grouter.cc b/libretroshare/src/grouter/p3grouter.cc index 75e8fd4d6..656da0f9c 100644 --- a/libretroshare/src/grouter/p3grouter.cc +++ b/libretroshare/src/grouter/p3grouter.cc @@ -482,7 +482,7 @@ void p3GRouter::handleLowLevelTransactionAckItem(RsGRouterTransactionAcknItem *t #endif } -void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/) +void p3GRouter::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/) { #ifdef GROUTER_DEBUG std::cerr << "p3GRouter::receiveTurtleData() " << std::endl; @@ -496,7 +496,7 @@ void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFile // - possibly packs multi-item blocks back together // - converts it into a grouter generic item (by deserialising it) - RsTurtleGenericDataItem *item = dynamic_cast(gitem) ; + const RsTurtleGenericDataItem *item = dynamic_cast(gitem) ; if(item == NULL) { @@ -510,7 +510,8 @@ void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFile // Items come out of the pipe in order. We need to recover all chunks before we de-serialise the content and have it handled by handleIncoming() - RsItem *itm = RsGRouterSerialiser().deserialise(item->data_bytes,&item->data_size) ; + uint32_t size = item->data_size ; + RsItem *itm = RsGRouterSerialiser().deserialise(item->data_bytes,&size); if(itm == NULL) { diff --git a/libretroshare/src/grouter/p3grouter.h b/libretroshare/src/grouter/p3grouter.h index 71b2fc138..8a3a54000 100644 --- a/libretroshare/src/grouter/p3grouter.h +++ b/libretroshare/src/grouter/p3grouter.h @@ -209,7 +209,7 @@ protected: //===================================================// virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) ; - virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/); + virtual void receiveTurtleData(const RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/); virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) ; virtual void removeVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id) ; diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 11137f8b0..ba330e334 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -3384,3 +3384,11 @@ void RsGenExchange::removeDeleteExistingMessages( std::list& msgs, Gx } } +void RsGenExchange::turtleGroupRequest(const RsGxsGroupId& group_id) +{ + mNetService->turtleGroupRequest(group_id) ; +} +void RsGenExchange::turtleSearchRequest(const std::string& match_string) +{ + mNetService->turtleSearchRequest(match_string) ; +} diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index 491ff017b..98b1d5d30 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -287,6 +287,15 @@ public: */ bool getGroupStatistic(const uint32_t& token, GxsGroupStatistic& stats); + /*! + * \brief turtleGroupRequest + * Issues a browadcast group request using the turtle router generic search system. The request is obviously asynchroneous and will be + * handled in RsGenExchange when received. + * \param group_id + */ + void turtleGroupRequest(const RsGxsGroupId& group_id); + void turtleSearchRequest(const std::string& match_string); + protected: bool messagePublicationTest(const RsGxsMsgMetaData&) ; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 8a46fc89e..b0d1743d6 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5102,3 +5102,12 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid) return true; } + +void RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id) +{ + mGxsNetTunnel->turtleGroupRequest(group_id) ; +} +void RsGxsNetService::turtleSearchRequest(const std::string& match_string) +{ + mGxsNetTunnel->turtleSearchRequest(match_string) ; +} diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index fe1c77010..aa8f8288c 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -122,6 +122,8 @@ public: virtual void setDefaultKeepAge(uint32_t t) { mDefaultMsgStorePeriod = t ; } virtual void setDefaultSyncAge(uint32_t t) { mDefaultMsgSyncPeriod = t ; } + virtual void turtleGroupRequest(const RsGxsGroupId& group_id); + virtual void turtleSearchRequest(const std::string& match_string); /*! * pauses synchronisation of subscribed groups and request for group id * from peers diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index b81b6a7c0..964f15781 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -389,7 +389,7 @@ bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsP return mHandledHashes.find(hash) != mHandledHashes.end(); } -void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& turtle_virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) +void RsGxsNetTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *item, const RsFileHash& hash, const RsPeerId& turtle_virtual_peer_id, RsTurtleGenericTunnelItem::Direction direction) { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -424,7 +424,7 @@ void RsGxsNetTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *item,co generateEncryptionKey(group_id,turtle_virtual_peer_id,encryption_master_key); - if(!p3turtle::decryptItem(static_cast(item),encryption_master_key,data,data_size)) + if(!p3turtle::decryptItem(static_cast(item),encryption_master_key,data,data_size)) { GXS_NET_TUNNEL_ERROR() << "Cannot decrypt data!" << std::endl; @@ -852,3 +852,23 @@ RsSerialiser *RsGxsNetTunnelService::setupSerialiser() return ser ; } + +bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len) +{ + std::cerr << __PRETTY_FUNCTION__ << ": received a request. Code needed to handle it" << std::endl; + return false ; +} +void RsGxsNetTunnelService::receiveSearchResult(unsigned char *search_result_data,uint32_t search_result_data_len) +{ + std::cerr << __PRETTY_FUNCTION__ << ": received a search result. Code needed to handle it" << std::endl; +} + +void RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& group_id) +{ + std::cerr << __PRETTY_FUNCTION__ << ": handling of turtle group request not implemented yet" << std::endl; +} +void RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string) +{ + std::cerr << __PRETTY_FUNCTION__ << ": handling of turtle search request not implemented yet" << std::endl; +} + diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 46314da6f..f85b997f5 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -212,13 +212,23 @@ public: */ void dump() const; - // other methods are still missing. - // - derived from p3Config, to load/save data - // - method to respond to tunnel requests, probably using RsGxsNetService - // - method to encrypt/decrypt data and send/receive to/from turtle. - + /*! + * \brief connectToTurtleRouter + * Should be called after allocating a RsGxsNetTunnelService + * \param tr turtle router object + */ virtual void connectToTurtleRouter(p3turtle *tr) ; + void turtleGroupRequest(const RsGxsGroupId& group_id) ; + void turtleSearchRequest(const std::string& match_string) ; + + /*! + * \brief receiveSearchRequest + * See RsTurtleClientService::@ + */ + virtual bool receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len); + virtual void receiveSearchResult(unsigned char *search_result_data,uint32_t search_result_data_len) ; + // Overloaded from RsTickingThread void data_tick() ; @@ -233,7 +243,7 @@ protected: // interaction with turtle router virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ; - virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; + virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ; void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; const Bias20Bytes& locked_randomBias() ; diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index faec2577a..59d2bf209 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -81,6 +81,8 @@ public: virtual uint32_t getDefaultSyncAge() =0; virtual uint32_t getDefaultKeepAge() =0; + virtual void turtleGroupRequest(const RsGxsGroupId& group_id)=0; + virtual void turtleSearchRequest(const std::string& match_string)=0; /*! * Initiates a search through the network * This returns messages which contains the search terms set in RsGxsSearch diff --git a/libretroshare/src/gxstunnel/p3gxstunnel.cc b/libretroshare/src/gxstunnel/p3gxstunnel.cc index eef365230..a57eef36b 100644 --- a/libretroshare/src/gxstunnel/p3gxstunnel.cc +++ b/libretroshare/src/gxstunnel/p3gxstunnel.cc @@ -685,7 +685,7 @@ void p3GxsTunnelService::removeVirtualPeer(const TurtleFileHash& hash,const Turt } } -void p3GxsTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *gitem,const RsFileHash& hash, const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) +void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash& hash, const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem::Direction direction) { #ifdef DEBUG_GXS_TUNNEL std::cerr << "GxsTunnelService::receiveTurtleData(): Received turtle data. " << std::endl; @@ -697,7 +697,7 @@ void p3GxsTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *gitem,cons (void) direction; #endif - RsTurtleGenericDataItem *item = dynamic_cast(gitem) ; + const RsTurtleGenericDataItem *item = dynamic_cast(gitem) ; if(item == NULL) { diff --git a/libretroshare/src/gxstunnel/p3gxstunnel.h b/libretroshare/src/gxstunnel/p3gxstunnel.h index 32020e619..cff0fd126 100644 --- a/libretroshare/src/gxstunnel/p3gxstunnel.h +++ b/libretroshare/src/gxstunnel/p3gxstunnel.h @@ -211,7 +211,7 @@ private: // Overloaded from RsTurtleClientService virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ; - virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; + virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ; void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ; diff --git a/libretroshare/src/pqi/p3notify.cc b/libretroshare/src/pqi/p3notify.cc index 2461d5f1c..5cc525dad 100644 --- a/libretroshare/src/pqi/p3notify.cc +++ b/libretroshare/src/pqi/p3notify.cc @@ -231,7 +231,8 @@ void p3Notify::notifyChatLobbyTimeShift (int time_shift) void p3Notify::notifyCustomState (const std::string& peer_id , const std::string& status_string ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyCustomState (peer_id,status_string) ; } void p3Notify::notifyHashingInfo (uint32_t type , const std::string& fileinfo ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyHashingInfo (type,fileinfo) ; } void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list& files ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,files) ; } -void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list& groups ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,groups) ; } +#warning MISSING CODE HERE +//void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list& groups ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,groups) ; } void p3Notify::notifyPeerHasNewAvatar (std::string peer_id ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; } void p3Notify::notifyOwnAvatarChanged () { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnAvatarChanged() ; } void p3Notify::notifyOwnStatusMessageChanged() { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnStatusMessageChanged() ; } diff --git a/libretroshare/src/pqi/p3notify.h b/libretroshare/src/pqi/p3notify.h index 22e1f143d..30386c95a 100644 --- a/libretroshare/src/pqi/p3notify.h +++ b/libretroshare/src/pqi/p3notify.h @@ -106,7 +106,8 @@ class p3Notify: public RsNotify void notifyCustomState (const std::string& /* peer_id */, const std::string& /* status_string */) ; void notifyHashingInfo (uint32_t /* type */, const std::string& /* fileinfo */) ; void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* files */) ; - void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* groups */) ; +#warning MISSING CODE HERE +// void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* groups */) ; void notifyPeerHasNewAvatar (std::string /* peer_id */) ; void notifyOwnAvatarChanged () ; void notifyOwnStatusMessageChanged () ; diff --git a/libretroshare/src/retroshare/rsfiles.h b/libretroshare/src/retroshare/rsfiles.h index d7efb2937..001328956 100644 --- a/libretroshare/src/retroshare/rsfiles.h +++ b/libretroshare/src/retroshare/rsfiles.h @@ -32,6 +32,7 @@ #include #include "rstypes.h" +#include "rsturtle.h" class RsFiles; extern RsFiles *rsFiles; @@ -213,6 +214,8 @@ class RsFiles virtual uint32_t getMaxUploadSlotsPerFriend()=0; virtual void setFilePermDirectDL(uint32_t perm)=0; virtual uint32_t filePermDirectDL()=0; + virtual TurtleRequestId turtleSearch(const std::string& string_to_match) =0; + virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) =0; /*** * Control of Downloads Priority. diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index c75235f0e..175c7fc22 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -96,6 +96,9 @@ virtual bool getPostData(const uint32_t &token, std::vector &p //virtual bool createComment(uint32_t &token, RsGxsComment &comment) = 0; //virtual bool createVote(uint32_t &token, RsGxsVote &vote) = 0; + virtual void turtleGroupRequest(const RsGxsGroupId& group_id)=0; + virtual void turtleSearchRequest(const std::string& match_string)=0; + ////////////////////////////////////////////////////////////////////////////// virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0; diff --git a/libretroshare/src/retroshare/rsnotify.h b/libretroshare/src/retroshare/rsnotify.h index 6d9db3ef7..ee731a67a 100644 --- a/libretroshare/src/retroshare/rsnotify.h +++ b/libretroshare/src/retroshare/rsnotify.h @@ -228,7 +228,8 @@ class NotifyClient virtual void notifyCustomState (const std::string& /* peer_id */, const std::string& /* status_string */) {} virtual void notifyHashingInfo (uint32_t /* type */, const std::string& /* fileinfo */) {} virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* files */) {} - virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* groups */) {} +#warning MISSING CODE HERE + // virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list& /* groups */) {} virtual void notifyPeerHasNewAvatar (std::string /* peer_id */) {} virtual void notifyOwnAvatarChanged () {} virtual void notifyOwnStatusMessageChanged () {} diff --git a/libretroshare/src/retroshare/rsturtle.h b/libretroshare/src/retroshare/rsturtle.h index 0467b452d..1b3cb1b7e 100644 --- a/libretroshare/src/retroshare/rsturtle.h +++ b/libretroshare/src/retroshare/rsturtle.h @@ -54,13 +54,7 @@ struct TurtleFileInfo std::string name ; uint64_t size ; }; -struct TurtleGxsInfo -{ - RsGxsGroupId group_id ; - uint16_t service_id ; - std::string name ; - //RsTlvBinaryData meta ;// is that actually needed? Not sure. Better if it's not. -}; + struct TurtleTunnelRequestDisplayInfo { uint32_t request_id ; // Id of the request @@ -118,8 +112,7 @@ class RsTurtle // the request id, which will be further used by the gui to store results // as they come back. // - virtual TurtleRequestId turtleSearch(const std::string& match_string) = 0 ; - virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) = 0 ; + virtual TurtleRequestId turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) =0; // Initiates tunnel handling for the given file hash. tunnels. Launches // an exception if an error occurs during the initialization process. The diff --git a/libretroshare/src/rsitems/rsgxsitems.cc b/libretroshare/src/rsitems/rsgxsitems.cc index 9971514ec..141f22167 100644 --- a/libretroshare/src/rsitems/rsgxsitems.cc +++ b/libretroshare/src/rsitems/rsgxsitems.cc @@ -6,6 +6,8 @@ */ +#include "serialiser/rstypeserializer.h" +#include "serialiser/rsbaseserial.h" #include "rsgxsitems.h" #include "gxs/rsgxsdata.h" #include @@ -70,4 +72,58 @@ std::ostream &operator<<(std::ostream &out, const RsMsgMetaData &meta) return out; } +template<> uint32_t RsTypeSerializer::serial_size(const TurtleGxsInfo& i) +{ + uint32_t s = 0 ; + s += 2 ; // service_id + s += i.group_id.SIZE_IN_BYTES ; + s += GetTlvStringSize(i.name) ; + + return s; +} + +template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,TurtleGxsInfo& i) +{ + uint32_t saved_offset = offset ; + bool ok = true ; + + ok &= getRawUInt16(data, size, &offset, &i.service_id); // service_id + ok &= i.group_id.deserialise(data, size, offset); // group_id + ok &= GetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name); // group name + + if(!ok) + offset = saved_offset ; + + return ok; +} + +template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const TurtleGxsInfo& i) +{ + uint32_t saved_offset = offset ; + bool ok = true ; + + ok &= setRawUInt16(data, size, &offset, i.service_id); // service_id + ok &= i.group_id.serialise(data, size, offset); // group_id + ok &= SetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name); // group name + + if(!ok) + offset = saved_offset ; + + return ok; +} + +template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleGxsInfo& i) +{ + std::cerr << " [GXS Info ] " << n << " group_id=" << i.group_id << " service=" << std::hex << i.service_id << std::dec << ", name=" << i.name << std::endl; +} + +void RsTurtleGxsSearchResultGroupSummaryItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +{ + RsTypeSerializer::serial_process(j,ctx,result,"result") ; +} +void RsTurtleGxsSearchResultGroupDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +{ + RsTypeSerializer::TlvMemBlock_proxy prox(encrypted_nxs_group_data,encrypted_nxs_group_data_len) ; + RsTypeSerializer::serial_process(j,ctx,prox,"encrypted_nxs_data") ; +} diff --git a/libretroshare/src/rsitems/rsgxsitems.h b/libretroshare/src/rsitems/rsgxsitems.h index fa461a0e8..74f471533 100644 --- a/libretroshare/src/rsitems/rsgxsitems.h +++ b/libretroshare/src/rsitems/rsgxsitems.h @@ -57,6 +57,82 @@ public: RsMsgMetaData meta; }; +// We should make these items templates or generic classes so that each GXS service will handle them on its own. + +static const uint8_t RS_PKT_SUBTYPE_GXS_SUBSTRING_SEARCH_ITEM = 0x20 ; +static const uint8_t RS_PKT_SUBTYPE_GXS_GROUP_SEARCH_ITEM = 0x21 ; +static const uint8_t RS_PKT_SUBTYPE_GXS_GROUP_SUMMARY_ITEM = 0x22 ; +static const uint8_t RS_PKT_SUBTYPE_GXS_GROUP_DATA_ITEM = 0x23 ; + +class RsGxsTurtleSubStringSearchItem: public RsItem +{ + public: + RsGxsTurtleSubStringSearchItem(uint16_t service): RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_SUBSTRING_SEARCH_ITEM) {} + virtual ~RsGxsTurtleSubStringSearchItem() {} + + std::string match_string ; // string to match + + std::string GetKeywords() { return match_string; } + void clear() { match_string.clear() ; } + + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; + +class RsGxsTurtleGroupSearchItem: public RsItem +{ + public: + RsGxsTurtleGroupSearchItem(uint16_t service): RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_SEARCH_ITEM) {} + virtual ~RsGxsTurtleGroupSearchItem() {} + + uint16_t service_id ; // searvice to search + Sha1CheckSum hashed_group_id ; // the group ID is hashed in order to keep it private. + + std::string GetKeywords() { return std::string("Group request for [hashed] ")+hashed_group_id.toStdString() ; } + void clear() { hashed_group_id.clear() ; } + + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; + +struct TurtleGxsInfo +{ + uint16_t service_id ; + RsGxsGroupId group_id ; + RsGxsId author; + std::string name ; + std::string description ; + time_t last_post ; + uint32_t number_of_posts ; +}; + +class RsTurtleGxsSearchResultGroupSummaryItem: public RsItem +{ + public: + RsTurtleGxsSearchResultGroupSummaryItem(uint16_t service) : RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_SUMMARY_ITEM){} + virtual ~RsTurtleGxsSearchResultGroupSummaryItem() {} + + std::list result ; + + void clear() { result.clear() ; } + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; + +class RsTurtleGxsSearchResultGroupDataItem: public RsItem +{ + public: + RsTurtleGxsSearchResultGroupDataItem(uint16_t service) : RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_DATA_ITEM){} + virtual ~RsTurtleGxsSearchResultGroupDataItem() {} + + unsigned char *encrypted_nxs_group_data; // data is encrypted with group ID. Only the requester, or anyone who already know the group id can decrypt. + uint32_t encrypted_nxs_group_data_len ; + + void clear() { free(encrypted_nxs_group_data); encrypted_nxs_group_data=NULL; encrypted_nxs_group_data_len=0; } + protected: + void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); +}; + diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index d2578d470..7d36048de 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -1695,3 +1695,10 @@ void p3GxsChannels::handle_event(uint32_t event_type, const std::string &elabel) } } +void p3GxsChannels::turtleGroupRequest(const RsGxsGroupId& group_id) +{ +} +void p3GxsChannels::turtleSearchRequest(const std::string& match_string) +{ +} + diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index 4eb5f5fba..c166dbc2f 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -72,6 +72,9 @@ virtual void service_tick(); virtual bool saveList(bool &cleanup, std::list&saveList); // @see p3Config::saveList(bool &cleanup, std::list&) virtual bool loadList(std::list& loadList); // @see p3Config::loadList(std::list&) + virtual void turtleGroupRequest(const RsGxsGroupId& group_id); + virtual void turtleSearchRequest(const std::string& match_string); + // Overloaded to cache new groups. virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet); diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 5208b32d8..9914d510c 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -156,10 +156,9 @@ void p3turtle::getItemNames(std::map& names) const names.clear(); names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST ] = "Filename substring search request"; - names[RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST ] = "GXS search request"; + names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST ] = "Generic search request"; names[RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT ] = "File search result"; - names[RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY ] = "GXS group summary"; - names[RS_TURTLE_SUBTYPE_GXS_GROUP_DATA ] = "GXS group data"; + names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT ] = "Generic search result"; names[RS_TURTLE_SUBTYPE_OPEN_TUNNEL ] = "Tunnel request"; names[RS_TURTLE_SUBTYPE_TUNNEL_OK ] = "Tunnel response"; names[RS_TURTLE_SUBTYPE_FILE_REQUEST ] = "Data request"; @@ -840,11 +839,11 @@ int p3turtle::handleIncoming() switch(item->PacketSubType()) { case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST: + case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST: case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast(item)) ; break ; - case RS_TURTLE_SUBTYPE_GXS_GROUP_DATA : - case RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY : + case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT : case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : handleSearchResult(dynamic_cast(item)) ; break ; @@ -1002,65 +1001,92 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) { - RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ - // Find who actually sent the corresponding request. - // - std::map::iterator it = _search_requests_origins.find(item->request_id) ; -#ifdef P3TURTLE_DEBUG - std::cerr << "Received search result:" << std::endl ; - item->print(std::cerr,0) ; -#endif - if(it == _search_requests_origins.end()) + std::list > results_to_notify_off_mutex ; + { - // This is an error: how could we receive a search result corresponding to a search item we - // have forwarded but that it not in the list ?? + RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ + // Find who actually sent the corresponding request. + // + std::map::iterator it = _search_requests_origins.find(item->request_id) ; - std::cerr << __PRETTY_FUNCTION__ << ": search result has no peer direction!" << std::endl ; - return ; - } - - // Is this result's target actually ours ? - - if(it->second.origin == _own_id) - { - it->second.result_count += item->count() ; - returnSearchResult(item) ; // Yes, so send upward. - } - else - { // Nope, so forward it back. #ifdef P3TURTLE_DEBUG - std::cerr << " Forwarding result back to " << it->second.origin << std::endl; + std::cerr << "Received search result:" << std::endl ; + item->print(std::cerr,0) ; #endif - // We update the total count forwarded back, and chop it to TURTLE_SEARCH_RESULT_MAX_HITS. - - uint32_t n = item->count(); // not so good! - - if(it->second.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + if(it == _search_requests_origins.end()) { - std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl; + // This is an error: how could we receive a search result corresponding to a search item we + // have forwarded but that it not in the list ?? + + std::cerr << __PRETTY_FUNCTION__ << ": search result has no peer direction!" << std::endl ; return ; } - if(it->second.result_count + n > TURTLE_SEARCH_RESULT_MAX_HITS) - { - for(uint32_t i=it->second.result_count + n; i>TURTLE_SEARCH_RESULT_MAX_HITS;--i) - item->pop() ; + // Is this result's target actually ours ? - it->second.result_count = TURTLE_SEARCH_RESULT_MAX_HITS ; + if(it->second.origin == _own_id) + { + it->second.result_count += item->count() ; + + results_to_notify_off_mutex.push_back(std::make_pair(item,it->second.client)) ; } else - it->second.result_count += n ; + { // Nope, so forward it back. +#ifdef P3TURTLE_DEBUG + std::cerr << " Forwarding result back to " << it->second.origin << std::endl; +#endif + // We update the total count forwarded back, and chop it to TURTLE_SEARCH_RESULT_MAX_HITS. - RsTurtleSearchResultItem *fwd_item = item->duplicate(); + uint32_t n = item->count(); // not so good! - // Normally here, we should setup the forward adress, so that the owner's - // of the files found can be further reached by a tunnel. + if(it->second.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + { + std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl; + return ; + } - fwd_item->PeerId(it->second.origin) ; - fwd_item->depth = 0 ; // obfuscate the depth for non immediate friends. Result will always be 0. This effectively removes the information. + if(it->second.result_count + n > TURTLE_SEARCH_RESULT_MAX_HITS) + { + for(uint32_t i=it->second.result_count + n; i>TURTLE_SEARCH_RESULT_MAX_HITS;--i) + item->pop() ; - sendItem(fwd_item) ; + it->second.result_count = TURTLE_SEARCH_RESULT_MAX_HITS ; + } + else + it->second.result_count += n ; + + RsTurtleSearchResultItem *fwd_item = item->duplicate(); + + // Normally here, we should setup the forward adress, so that the owner's + // of the files found can be further reached by a tunnel. + + fwd_item->PeerId(it->second.origin) ; + fwd_item->depth = 0 ; // obfuscate the depth for non immediate friends. Result will always be 0. This effectively removes the information. + + sendItem(fwd_item) ; + } } + + // now we notify clients off-mutex. + + for(auto it(results_to_notify_off_mutex.begin());it!=results_to_notify_off_mutex.end();++it) + { + // Hack to use the old search result handling in ftServer. Normally ftServer should use the new method with serialized result. + +#warning make sure memory is correctly deleted here + RsTurtleFTSearchResultItem *ftsr = dynamic_cast(it->first) ; + + if(ftsr!=NULL) + { + RsServer::notify()->notifyTurtleSearchResult(ftsr->request_id,ftsr->result) ; + continue ; + } + + RsTurtleGenericSearchResultItem *gnsr = dynamic_cast(it->first) ; + + if(gnsr!=NULL) + (*it).second->receiveSearchResult(gnsr->result_data,gnsr->result_data_len) ; + } } // -----------------------------------------------------------------------------------// @@ -1794,17 +1820,11 @@ void RsTurtleFileSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo & } } -void RsTurtleGxsSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const +void RsTurtleGenericSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const { std::cerr << "(EE) p3turtle: Missing code to perform actual GXS search" << std::endl; } -void RsTurtleGxsGroupRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const -{ - std::cerr << "(EE) p3turtle: Missing code to perform actual retrieval of GXS group" << std::endl; -} - - void RsTurtleStringSearchRequestItem::search(std::list& result) const { /* call to core */ @@ -1939,6 +1959,34 @@ TurtleRequestId p3turtle::turtleSearch(const RsRegularExpression::LinearizedExpr return id ; } +TurtleRequestId p3turtle::turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) +{ + // generate a new search id. + + TurtleRequestId id = generateRandomRequestId() ; + + // Form a request packet that simulates a request from us. + // + RsTurtleGenericSearchRequestItem *item = new RsTurtleGenericSearchRequestItem ; + +#ifdef P3TURTLE_DEBUG + std::cerr << "performing search. OwnId = " << _own_id << std::endl ; +#endif + + item->PeerId(_own_id) ; + item->service_id = client_service->serviceId(); + item->search_data = search_bin_data ; + item->search_data_len = search_bin_data_len ; + item->request_id = id ; + item->depth = 0 ; + + // send it + + handleSearchRequest(item) ; + + return id ; +} + void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *client_service,bool allow_multi_tunnels) { { @@ -1979,36 +2027,26 @@ void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *clie IndicateConfigChanged() ; // initiates saving of handled hashes. } -void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item) -{ #ifdef P3TURTLE_DEBUG std::cerr << " Returning result for search request " << HEX_PRINT(item->request_id) << " upwards." << std::endl ; #endif - RsTurtleFTSearchResultItem *ft_sr = dynamic_cast(item) ; - if(ft_sr != NULL) - { - RsServer::notify()->notifyTurtleSearchResult(ft_sr->request_id,ft_sr->result) ; - return ; - } - - RsTurtleGxsSearchResultGroupSummaryItem *gxs_sr_gs = dynamic_cast(item) ; - - if(gxs_sr_gs != NULL) - { - RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gs->request_id,gxs_sr_gs->result) ; - return ; - } - RsTurtleGxsSearchResultGroupDataItem *gxs_sr_gd = dynamic_cast(item) ; - - if(gxs_sr_gd != NULL) - { -#warning MISSING CODE HERE TO HANDLE ENCRYPTED INCOMING GROUP DATA. - //RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gd->request_id,gxs_sr_gd->encrypted_nxs_group) ; - return ; - } -} +// RsTurtleGxsSearchResultGroupSummaryItem *gxs_sr_gs = dynamic_cast(item) ; +// +// if(gxs_sr_gs != NULL) +// { +// RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gs->request_id,gxs_sr_gs->result) ; +// return ; +// } +// RsTurtleGxsSearchResultGroupDataItem *gxs_sr_gd = dynamic_cast(item) ; +// +// if(gxs_sr_gd != NULL) +// { +//#warning MISSING CODE HERE TO HANDLE ENCRYPTED INCOMING GROUP DATA. +// //RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gd->request_id,gxs_sr_gd->encrypted_nxs_group) ; +// return ; +// } /// Warning: this function should never be called while the turtle mutex is locked. /// Otherwize this is a possible source of cross-lock with the File mutex. diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index b3c4debe4..04d90c812 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -173,6 +173,7 @@ class TurtleSearchRequestInfo int depth ; // depth of the request. Used to optimize tunnel length. uint32_t result_count; // responses to this request. Useful to avoid spamming tunnel responses. std::string keywords; + RsTurtleClientService *client;// client who issues the request. This is null if the request does not have a local origin. }; class TurtleTunnelRequestInfo { @@ -244,11 +245,16 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config // the request id, which will be further used by the gui to store results // as they come back. // - // Eventually, search requests should be handled by client services. We will therefore - // remove the specific file search packets from the turtle router. - // - virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ; - virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ; + // The first two methods are old style search requests for FT, while the 3rd one is using a generic search data type, that is only to + // be deserialized by the service. The memory ownership is kept by the calling function. Similarly, the search response will be a + // generic data type that is to be deserialized by the client service. + // + // Eventually, search requests will use the generic system + // even for FT. We need to keep the old method for a while for backward compatibility. + // + virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ; + virtual TurtleRequestId turtleSearch(const std::string& string_to_match) ; + virtual TurtleRequestId turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) ; // Initiates tunnel handling for the given file hash. tunnels. Launches // an exception if an error occurs during the initialization process. The @@ -393,9 +399,6 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config /// Performs a search calling local cache and search structure. void performLocalSearch(const std::string& match_string,std::list& result) ; - /// Returns a search result upwards (possibly to the gui) - void returnSearchResult(RsTurtleSearchResultItem *item) ; - /// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer. virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service); diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index 175996575..ce3a4450a 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -32,11 +32,8 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : return new RsTurtleOpenTunnelItem(); case RS_TURTLE_SUBTYPE_TUNNEL_OK : return new RsTurtleTunnelOkItem(); case RS_TURTLE_SUBTYPE_GENERIC_DATA : return new RsTurtleGenericDataItem(); - - case RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST : return new RsTurtleGxsSearchRequestItem(); - case RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST : return new RsTurtleGxsGroupRequestItem(); - case RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY : return new RsTurtleGxsSearchResultGroupSummaryItem(); - case RS_TURTLE_SUBTYPE_GXS_GROUP_DATA : return new RsTurtleGxsSearchResultGroupDataItem(); + case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST : return new RsTurtleGenericSearchRequestItem(); + case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT : return new RsTurtleGenericSearchResultItem(); default: break ; @@ -65,21 +62,24 @@ void RsTurtleRegExpSearchRequestItem::serial_process(RsGenericSerializer::Serial RsTypeSerializer::serial_process(j,ctx,depth,"depth") ; RsTypeSerializer::serial_process(j,ctx,expr,"expr") ; } -void RsTurtleGxsSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +void RsTurtleGenericSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; RsTypeSerializer::serial_process(j,ctx,depth,"depth") ; RsTypeSerializer::serial_process(j,ctx,service_id,"service_id") ; - RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ; -} -void RsTurtleGxsGroupRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) -{ - RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; - RsTypeSerializer::serial_process(j,ctx,depth,"depth") ; - RsTypeSerializer::serial_process(j,ctx,service_id,"service_id") ; - RsTypeSerializer::serial_process(j,ctx,hashed_group_id,"hashed_group_id") ; -} + RsTypeSerializer::TlvMemBlock_proxy prox(search_data,search_data_len) ; + RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ; +} +RsTurtleSearchRequestItem *RsTurtleGenericSearchRequestItem::clone() const +{ + RsTurtleGenericSearchRequestItem *sr = new RsTurtleGenericSearchRequestItem ; + + sr->search_data = (unsigned char*)rs_malloc(search_data_len) ; + memcpy(sr->search_data,search_data,search_data_len) ; + sr->search_data_len = search_data_len ; + return sr ; +} template<> uint32_t RsTypeSerializer::serial_size(const RsRegularExpression::LinearizedExpression& r) { uint32_t s = 0 ; @@ -164,18 +164,24 @@ void RsTurtleFTSearchResultItem::serial_process(RsGenericSerializer::SerializeJo RsTypeSerializer::serial_process(j,ctx,depth ,"depth") ; RsTypeSerializer::serial_process (j,ctx,result ,"result") ; } -void RsTurtleGxsSearchResultGroupSummaryItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +void RsTurtleGenericSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; RsTypeSerializer::serial_process(j,ctx,depth ,"depth") ; - RsTypeSerializer::serial_process (j,ctx,result ,"result") ; + + RsTypeSerializer::TlvMemBlock_proxy prox(result_data,result_data_len) ; + RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ; } -void RsTurtleGxsSearchResultGroupDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) +RsTurtleSearchResultItem *RsTurtleGenericSearchResultItem::duplicate() const { - RsTypeSerializer::serial_process (j,ctx,request_id,"request_id") ; - RsTypeSerializer::serial_process (j,ctx,depth ,"depth") ; - RsTypeSerializer::serial_process(j,ctx,encrypted_nxs_group,"encrypted_nxs_group") ; + RsTurtleGenericSearchResultItem *sr = new RsTurtleGenericSearchResultItem ; + + sr->result_data = (unsigned char*)rs_malloc(result_data_len) ; + memcpy(sr->result_data,result_data,result_data_len) ; + sr->result_data_len = result_data_len ; + return sr ; } + template<> uint32_t RsTypeSerializer::serial_size(const TurtleFileInfo& i) { uint32_t s = 0 ; @@ -222,52 +228,6 @@ template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleF std::cerr << " [FileInfo ] " << n << " size=" << i.size << " hash=" << i.hash << ", name=" << i.name << std::endl; } -template<> uint32_t RsTypeSerializer::serial_size(const TurtleGxsInfo& i) -{ - uint32_t s = 0 ; - - s += 2 ; // service_id - s += i.group_id.SIZE_IN_BYTES ; - s += GetTlvStringSize(i.name) ; - - return s; -} - -template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,TurtleGxsInfo& i) -{ - uint32_t saved_offset = offset ; - bool ok = true ; - - ok &= getRawUInt16(data, size, &offset, &i.service_id); // service_id - ok &= i.group_id.deserialise(data, size, offset); // group_id - ok &= GetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name); // group name - - if(!ok) - offset = saved_offset ; - - return ok; -} - -template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const TurtleGxsInfo& i) -{ - uint32_t saved_offset = offset ; - bool ok = true ; - - ok &= setRawUInt16(data, size, &offset, i.service_id); // service_id - ok &= i.group_id.serialise(data, size, offset); // group_id - ok &= SetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name); // group name - - if(!ok) - offset = saved_offset ; - - return ok; -} - -template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleGxsInfo& i) -{ - std::cerr << " [GXS Info ] " << n << " group_id=" << i.group_id << " service=" << std::hex << i.service_id << std::dec << ", name=" << i.name << std::endl; -} - void RsTurtleOpenTunnelItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process (j,ctx,file_hash ,"file_hash") ; diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 885ab04de..035490bc1 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -23,17 +23,15 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_REQUEST = 0x07 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA = 0x08 ; const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST = 0x09 ; const uint8_t RS_TURTLE_SUBTYPE_GENERIC_DATA = 0x0a ; -const uint8_t RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST = 0x0b ; -const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST = 0x0c ; +const uint8_t RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST = 0x0b ; +const uint8_t RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT = 0x0c ; const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ; const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ; -const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ; -const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST = 0x15 ; -const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY = 0x16 ; -const uint8_t RS_TURTLE_SUBTYPE_GXS_GROUP_DATA = 0x17 ; - // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ; // unused // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ; +const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ; +const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST = 0x15 ; + class TurtleSearchRequestInfo ; @@ -106,6 +104,8 @@ class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem virtual ~RsTurtleFileSearchRequestItem() {} virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method + + protected: virtual void search(std::list &) const =0; }; @@ -151,44 +151,29 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleFileSearchRequestItem void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; -class RsTurtleGxsSearchRequestItem: public RsTurtleSearchRequestItem +class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem { public: - RsTurtleGxsSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GXS_SEARCH_REQUEST) {} - virtual ~RsTurtleGxsSearchRequestItem() {} + RsTurtleGenericSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST) {} + virtual ~RsTurtleGenericSearchRequestItem() { clear(); } - std::string match_string ; // string to match - uint16_t service_id ; // searvice to search + uint16_t service_id ; // service to search + uint32_t search_data_len ; + unsigned char *search_data ; - std::string GetKeywords() { return match_string; } + std::string GetKeywords() { return std::string("Generic search " + RsUtil::BinToHex(search_data,search_data_len,10)); } - virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method - virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleGxsSearchRequestItem(*this) ; } - void clear() { match_string.clear() ; } + virtual RsTurtleSearchRequestItem *clone() const ; + void clear() { free(search_data); search_data=NULL; search_data_len=0; } + virtual void performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const; protected: void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); + + private: + RsTurtleGenericSearchRequestItem(const RsTurtleGenericSearchRequestItem&): RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST) {} // make the object non copi-able. + RsTurtleGenericSearchRequestItem& operator=(const RsTurtleGenericSearchRequestItem&) { return *this;} }; - -class RsTurtleGxsGroupRequestItem: public RsTurtleSearchRequestItem -{ - public: - RsTurtleGxsGroupRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GXS_GROUP_REQUEST) {} - virtual ~RsTurtleGxsGroupRequestItem() {} - - uint16_t service_id ; // searvice to search - Sha1CheckSum hashed_group_id ; // the group ID is hashed in order to keep it private. - - virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleGxsGroupRequestItem(*this) ; } - - virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method - void clear() { hashed_group_id.clear() ; } - std::string GetKeywords() { return hashed_group_id.toStdString(); } - - protected: - void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); -}; - class RsTurtleSearchResultItem: public RsTurtleItem { public: @@ -199,7 +184,6 @@ class RsTurtleSearchResultItem: public RsTurtleItem // If the actual depth is 1, this field will be 1. // If the actual depth is > 1, this field is a larger arbitrary integer. - virtual void clear() =0; virtual uint32_t count() const =0; virtual void pop() =0; @@ -222,33 +206,20 @@ class RsTurtleFTSearchResultItem: public RsTurtleSearchResultItem void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; -class RsTurtleGxsSearchResultGroupSummaryItem: public RsTurtleSearchResultItem +class RsTurtleGenericSearchResultItem: public RsTurtleSearchResultItem { public: - RsTurtleGxsSearchResultGroupSummaryItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_GROUP_SUMMARY){} - virtual ~RsTurtleGxsSearchResultGroupSummaryItem() {} + RsTurtleGenericSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT){} + virtual ~RsTurtleGenericSearchResultItem() {} - std::list result ; + uint32_t count() const { return result_data_len/50 ; } // This is a blind size estimate. We should probably use the actual size to limit search results. + virtual void pop() {} - void clear() { result.clear() ; } - uint32_t count() const { return result.size() ; } - virtual void pop() { result.pop_back() ;} - virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleGxsSearchResultGroupSummaryItem(*this) ; } - protected: - void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); -}; -class RsTurtleGxsSearchResultGroupDataItem: public RsTurtleSearchResultItem -{ - public: - RsTurtleGxsSearchResultGroupDataItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GXS_GROUP_DATA){} - virtual ~RsTurtleGxsSearchResultGroupDataItem() {} + unsigned char *result_data ; + uint32_t result_data_len ; - RsTlvBinaryData encrypted_nxs_group; // data is encrypted with group ID. - - uint32_t count() const { return 1 ; } - virtual void pop() { clear(); } - void clear() { encrypted_nxs_group.TlvClear() ; } - virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleGxsSearchResultGroupDataItem(*this) ; } + virtual RsTurtleSearchResultItem *duplicate() const ; + void clear() { free(result_data); result_data=NULL; result_data_len=0; } protected: void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; @@ -262,10 +233,10 @@ class RsTurtleOpenTunnelItem: public RsTurtleItem public: RsTurtleOpenTunnelItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_OPEN_TUNNEL), request_id(0), partial_tunnel_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_OPEN_TUNNEL) ;} - TurtleFileHash file_hash ; // hash to match - uint32_t request_id ; // randomly generated request id. + TurtleFileHash file_hash ; // hash to match + uint32_t request_id ; // randomly generated request id. uint32_t partial_tunnel_id ; // uncomplete tunnel id. Will be completed at destination. - uint16_t depth ; // Used for limiting search depth. + uint16_t depth ; // Used for limiting search depth. void clear() { file_hash.clear() ;} protected: diff --git a/libretroshare/src/turtle/turtleclientservice.h b/libretroshare/src/turtle/turtleclientservice.h index 962e57cf0..277c8d5c3 100644 --- a/libretroshare/src/turtle/turtleclientservice.h +++ b/libretroshare/src/turtle/turtleclientservice.h @@ -42,6 +42,18 @@ class p3turtle ; class RsTurtleClientService { public: + /*! + * \brief serviceId + * Returns the ID of the client service. This is used to pass the ID to search requests, from the client services + * \return + * The service ID. + */ + + virtual uint16_t serviceId() const + { + std::cerr << "!!!!!! Received request for service ID in turtle router client, but the client service is not handling it !!!!!!!" << std::endl ; + return 0 ; + } /*! * \brief handleTunnelRequest @@ -52,7 +64,6 @@ class RsTurtleClientService */ virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) { return false ; } - /*! * \brief receiveTurtleData * This method is called by the turtle router to send data that comes out of a turtle tunnel, and should @@ -74,11 +85,43 @@ class RsTurtleClientService * By default (if not overloaded), the method will just free the data, as any subclass should do as well. * Note: p3turtle stays owner of the item, so the client should not delete it! */ - virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/) + virtual void receiveTurtleData(const RsTurtleGenericTunnelItem * /* item */,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/) { std::cerr << "!!!!!! Received Data from turtle router, but the client service is not handling it !!!!!!!!!!" << std::endl ; } + /*! + * \brief receiveSearchRequest + * This method is called by the turtle router to notify the client of a search request in the form generic data. The returned + * result contains the serialised generic result returned by the client. + * + * The turtle router keeps the memory ownership over search_request_data + * + * \param search_request_data generic serialized search data + * \param search_request_data_len length of the serialized search data + * \param search_result_data generic serialized search result data + * \param search_result_data_len length of the serialized search result data + * + * \return true if the search is successful. + */ + virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,uint32_t /*search_request_data_len*/,unsigned char *& /*search_result_data*/,uint32_t& /*search_result_data_len*/) + { + std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ; + return false; + } + + /*! + * \brief receiveSearchResult + * This method is called by the turtle router to notify the client of a search result. The result is serialized for the current class to read. + * + * \param search_result_data result data. Memory ownership is owned by the turtle router. So do not delete! + * \param search_result_data length of result data + */ + virtual void receiveSearchResult(unsigned char * /*search_result_data*/,uint32_t /*search_result_data_len*/) + { + std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ; + } + /*! * \brief serializer * Method for creating specific items of the client service. The diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp index bc2f08644..f1c2e2fb6 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp @@ -792,7 +792,7 @@ void SearchDialog::advancedSearch(RsRegularExpression::Expression* expression) RsRegularExpression::LinearizedExpression e ; expression->linearize(e) ; - TurtleRequestId req_id = rsTurtle->turtleSearch(e) ; + TurtleRequestId req_id = rsFiles->turtleSearch(e) ; // This will act before turtle results come to the interface, thanks to the signals scheduling policy. initSearchResult(QString::fromStdString(e.GetStrings()),req_id, ui.FileTypeComboBox->currentIndex(), true) ; @@ -858,9 +858,9 @@ void SearchDialog::searchKeywords(const QString& keywords) if(ui._anonF2Fsearch_CB->isChecked()) { if(n==1) - req_id = rsTurtle->turtleSearch(words.front()) ; + req_id = rsFiles->turtleSearch(words.front()) ; else - req_id = rsTurtle->turtleSearch(lin_exp) ; + req_id = rsFiles->turtleSearch(lin_exp) ; } else req_id = RSRandom::random_u32() ; // generate a random 32 bits request id diff --git a/retroshare-gui/src/gui/notifyqt.h b/retroshare-gui/src/gui/notifyqt.h index 597231703..56ec5c42a 100644 --- a/retroshare-gui/src/gui/notifyqt.h +++ b/retroshare-gui/src/gui/notifyqt.h @@ -23,7 +23,9 @@ class MessengerWindow; class ToasterItem; class ToasterNotify; class SignatureEventData ; + struct TurtleFileInfo; +struct TurtleGxsInfo; class NotifyQt: public QObject, public NotifyClient { From 6fb459ce642cd04377ee3975560902b9d7f65577 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 10 Jun 2018 22:34:17 +0200 Subject: [PATCH 42/84] added logic for GXS search in RsGxsNetTunnel and Gxs client net service --- libretroshare/src/ft/ftserver.cc | 2 +- libretroshare/src/gxs/rsgxsnetservice.cc | 10 +- libretroshare/src/gxs/rsgxsnetservice.h | 8 + libretroshare/src/gxs/rsgxsnettunnel.cc | 230 ++++++++++++++++-- libretroshare/src/gxs/rsgxsnettunnel.h | 18 +- libretroshare/src/gxs/rsnxs.h | 24 ++ libretroshare/src/turtle/p3turtle.cc | 2 +- .../src/turtle/turtleclientservice.h | 2 +- 8 files changed, 273 insertions(+), 23 deletions(-) diff --git a/libretroshare/src/ft/ftserver.cc b/libretroshare/src/ft/ftserver.cc index 3172cfd89..a454be33f 100644 --- a/libretroshare/src/ft/ftserver.cc +++ b/libretroshare/src/ft/ftserver.cc @@ -1462,7 +1462,7 @@ TurtleSearchRequestId ftServer::turtleSearch(const RsRegularExpression::Lineariz return mTurtleRouter->turtleSearch(expr) ; } -#warning we should do this here, but for now it's done by turtle router. +#warning we should do this here, but for now it is done by turtle router. // // Dont delete the item. The client (p3turtle) is doing it after calling this. // // // void ftServer::receiveSearchResult(RsTurtleSearchResultItem *item) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index b0d1743d6..d83ceed54 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5105,9 +5105,15 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid) void RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id) { - mGxsNetTunnel->turtleGroupRequest(group_id) ; + mGxsNetTunnel->turtleGroupRequest(group_id,this) ; } void RsGxsNetService::turtleSearchRequest(const std::string& match_string) { - mGxsNetTunnel->turtleSearchRequest(match_string) ; + mGxsNetTunnel->turtleSearchRequest(match_string,this) ; +} + +bool RsGxsNetService::search(const std::string& substring,std::list& group_infos) +{ +#warning MISSING CODE HERE! + return true ; } diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index aa8f8288c..9643f49bf 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -122,8 +122,16 @@ public: virtual void setDefaultKeepAge(uint32_t t) { mDefaultMsgStorePeriod = t ; } virtual void setDefaultSyncAge(uint32_t t) { mDefaultMsgSyncPeriod = t ; } + /*! + * \brief Search methods. + * These four methods are used to request distant search and receive the results. + * \param group_id + */ virtual void turtleGroupRequest(const RsGxsGroupId& group_id); virtual void turtleSearchRequest(const std::string& match_string); + + virtual bool search(const std::string& substring,std::list& group_infos) ; + /*! * pauses synchronisation of subscribed groups and request for group id * from peers diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 964f15781..7cd657a55 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -27,6 +27,7 @@ #include "util/rstime.h" #include "retroshare/rspeers.h" #include "serialiser/rstypeserializer.h" +#include "gxs/rsnxs.h" #include "rsgxsnettunnel.h" #define DEBUG_RSGXSNETTUNNEL 1 @@ -52,9 +53,13 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") const uint16_t RS_SERVICE_TYPE_GXS_NET_TUNNEL = 0x2233 ; -const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; -const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ; -const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER = 0x01 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE = 0x02 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS = 0x03 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING = 0x04 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST = 0x05 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY = 0x06 ; +const uint8_t RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA = 0x07 ; class RsGxsNetTunnelItem: public RsItem { @@ -107,6 +112,57 @@ public: Bias20Bytes mRandomBias; // Cannot be a simple char[] because of serialization. }; +class RsGxsNetTunnelTurtleSearchSubstringItem: public RsGxsNetTunnelItem +{ +public: + explicit RsGxsNetTunnelTurtleSearchSubstringItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING) {} + virtual ~RsGxsNetTunnelTurtleSearchSubstringItem() {} + + uint16_t service ; + std::string substring_match ; + + virtual void clear() { substring_match.clear() ; } + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,service,"service") ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_KEY,substring_match,"substring_match") ; + } +}; + +class RsGxsNetTunnelTurtleSearchGroupRequestItem: public RsGxsNetTunnelItem +{ +public: + explicit RsGxsNetTunnelTurtleSearchGroupRequestItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST) {} + virtual ~RsGxsNetTunnelTurtleSearchGroupRequestItem() {} + + uint16_t service ; + Sha1CheckSum hashed_group_id ; + + virtual void clear() { hashed_group_id.clear() ; } + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,service,"service") ; + RsTypeSerializer::serial_process(j,ctx,hashed_group_id,"hashed_group_id") ; + } +}; + +class RsGxsNetTunnelTurtleSearchGroupSummaryItem: public RsGxsNetTunnelItem +{ +public: + explicit RsGxsNetTunnelTurtleSearchGroupSummaryItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY) {} + virtual ~RsGxsNetTunnelTurtleSearchGroupSummaryItem() {} + + uint16_t service ; + std::list group_infos; + + virtual void clear() { group_infos.clear() ; } + + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,service,"service") ; + RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ; + } +}; class RsGxsNetTunnelSerializer: public RsServiceSerializer { public: @@ -132,10 +188,32 @@ public: } }; +template<> +void RsTypeSerializer::serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, RsGxsGroupSummary& gs, const std::string& member_name ) +{ + RsTypeSerializer::serial_process(j,ctx,gs.group_id,member_name+"-group_id") ; // RsGxsGroupId group_id ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_NAME ,gs.group_name,member_name+"-group_name") ; // std::string group_name ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_COMMENT,gs.group_description,member_name+"-group_description") ; // std::string group_description ; + RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_VALUE ,gs.search_context,member_name+"-group_name") ; // std::string search_context ; + RsTypeSerializer::serial_process(j,ctx,gs.author_id ,member_name+"-author_id") ; // RsGxsId author_id ; + RsTypeSerializer::serial_process(j,ctx,gs.publish_ts ,member_name+"-publish_ts") ; // time_t publish_ts ; + RsTypeSerializer::serial_process(j,ctx,gs.number_of_messages,member_name+"-number_of_messages") ; // uint32_t number_of_messages ; + RsTypeSerializer::serial_process(j,ctx,gs.last_message_ts,member_name+"-last_message_ts") ; // time_t last_message_ts ; +} + //===========================================================================================================================================// // Interface with rest of the software // //===========================================================================================================================================// +bool RsGxsNetTunnelService::registerSearchableService(RsNetworkExchangeService *gxs_service) +{ + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + mSearchableServices[gxs_service->serviceType()] = gxs_service ; + + return true; +} + class DataAutoDelete { public: @@ -853,22 +931,144 @@ RsSerialiser *RsGxsNetTunnelService::setupSerialiser() return ser ; } -bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len) +//===========================================================================================================================================// +// Turtle Search system // +//===========================================================================================================================================// + +TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& group_id,RsNetworkExchangeService *client_service) { - std::cerr << __PRETTY_FUNCTION__ << ": received a request. Code needed to handle it" << std::endl; + Sha1CheckSum hashed_group_id = RsDirUtil::sha1sum(group_id.toByteArray(),group_id.SIZE_IN_BYTES) ; + + GXS_NET_TUNNEL_DEBUG() << ": starting a turtle group request for grp \"" << group_id << "\" hashed to \"" << hashed_group_id << "\"" << std::endl; + + RsGxsNetTunnelTurtleSearchGroupRequestItem search_item ; + search_item.hashed_group_id = hashed_group_id ; + search_item.service = client_service->serviceType() ; + + uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ; + unsigned char *mem = (unsigned char*)rs_malloc(size) ; + + if(mem == NULL) + return 0 ; + + RsGxsNetTunnelSerializer().serialise(&search_item,mem,&size); + + return mTurtle->turtleSearch(mem,size,this) ; +} + +TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) +{ + GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string\"" << match_string << "\"" << std::endl; + + RsGxsNetTunnelTurtleSearchSubstringItem search_item ; + search_item.substring_match = match_string ; + search_item.service = client_service->serviceType() ; + + uint32_t size = RsGxsNetTunnelSerializer().size(&search_item) ; + unsigned char *mem = (unsigned char*)rs_malloc(size) ; + + if(mem == NULL) + return 0 ; + + RsGxsNetTunnelSerializer().serialise(&search_item,mem,&size); + + return mTurtle->turtleSearch(mem,size,this) ; +} + +bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size) +{ + GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; + + RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ; + + RsGxsNetTunnelTurtleSearchSubstringItem *substring_sr = dynamic_cast(item) ; + + if(substring_sr != NULL) + { + auto it = mSearchableServices.find(substring_sr->service) ; + + std::list results ; + + if(it != mSearchableServices.end() && it->second->search(substring_sr->substring_match,results)) + { + RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ; + + search_result_item.service = substring_sr->service ; + search_result_item.group_infos = results ; + + search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ; + search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ; + + if(search_result_data == NULL) + return false ; + + RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size); + + return true ; + } + } + + RsGxsNetTunnelTurtleSearchGroupRequestItem *substring_gr = dynamic_cast(item) ; + + if(substring_gr != NULL) + { +#ifdef TODO + auto it = mSearchableGxsServices.find(substring_sr->service) ; + + RsNxsGrp *grp = NULL ; + + if(it != mSearchableGxsServices.end() && it->second.search(substring_sr->group_id,grp)) + { + RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ; + + search_result_item.service = substring_sr->service ; + search_result_item.group_infos = results ; + + search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ; + search_result_data = (unsigned char*)rs_malloc(size) ; + + if(search_result_data == NULL) + return false ; + + RsGxsNetTunnelSerializer().serialise(&search_result_item,search_result_data,&search_result_data_size); + + return true ; + } +#endif + } + return false ; } -void RsGxsNetTunnelService::receiveSearchResult(unsigned char *search_result_data,uint32_t search_result_data_len) + +void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len) { - std::cerr << __PRETTY_FUNCTION__ << ": received a search result. Code needed to handle it" << std::endl; + RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len); + + RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast(item) ; + + if(result_gs != NULL) + { + + } } -void RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& group_id) -{ - std::cerr << __PRETTY_FUNCTION__ << ": handling of turtle group request not implemented yet" << std::endl; -} -void RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string) -{ - std::cerr << __PRETTY_FUNCTION__ << ": handling of turtle search request not implemented yet" << std::endl; -} + + + + + + + + + + + + + + + + + + + diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index f85b997f5..22917f35b 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -105,6 +105,7 @@ typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; class RsGxsNetTunnelItem ; +class RsNetworkExchangeService ; struct RsGxsNetTunnelVirtualPeerInfo { @@ -159,6 +160,15 @@ public: RsGxsNetTunnelService() ; virtual ~RsGxsNetTunnelService() ; + /*! + * \brief registerSearchableService + * Adds the network exchange service as a possible search source. This is used to allow distant search on the corresponding + * GXS service. + * \return + * always returns true. + */ + bool registerSearchableService(RsNetworkExchangeService *) ; + /*! * \brief Manage tunnels for this group * @param group_id group for which tunnels should be released @@ -219,15 +229,15 @@ public: */ virtual void connectToTurtleRouter(p3turtle *tr) ; - void turtleGroupRequest(const RsGxsGroupId& group_id) ; - void turtleSearchRequest(const std::string& match_string) ; + TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ; + TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ; /*! * \brief receiveSearchRequest * See RsTurtleClientService::@ */ virtual bool receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len); - virtual void receiveSearchResult(unsigned char *search_result_data,uint32_t search_result_data_len) ; + virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len); // Overloaded from RsTickingThread @@ -270,6 +280,8 @@ private: std::map > > mIncomingData; // list of incoming data items + std::map mSearchableServices ; + /*! * \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to * hide the real group id. diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 59d2bf209..5dfcde9d6 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -58,6 +58,26 @@ * 2 transfers only between group * - the also group matrix settings which is by default everyone can transfer to each other */ + +/*! + * \brief The RsGxsGroupSymmary struct + * This structure is used to transport group summary information when a GXS service is searched. It contains the group information + * as well as a context string to tell where the information was found. It is more compact than a GroupMeta object, so as to make + * search responses as light as possible. + */ +struct RsGxsGroupSummary +{ + RsGxsGroupId group_id ; + + std::string group_name ; + std::string group_description ; + std::string search_context ; + RsGxsId author_id ; + time_t publish_ts ; + uint32_t number_of_messages ; + time_t last_message_ts ; +}; + class RsNetworkExchangeService { public: @@ -65,6 +85,7 @@ public: RsNetworkExchangeService(){ return;} virtual ~RsNetworkExchangeService() {} + virtual uint16_t serviceType() const =0; /*! * Use this to set how far back synchronisation of messages should take place * @param age in seconds the max age a sync/store item can to be allowed in a synchronisation @@ -83,6 +104,9 @@ public: virtual void turtleGroupRequest(const RsGxsGroupId& group_id)=0; virtual void turtleSearchRequest(const std::string& match_string)=0; + + virtual bool search(const std::string& substring,std::list& group_infos) =0; + /*! * Initiates a search through the network * This returns messages which contains the search terms set in RsGxsSearch diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 9914d510c..1232b950c 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -1085,7 +1085,7 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) RsTurtleGenericSearchResultItem *gnsr = dynamic_cast(it->first) ; if(gnsr!=NULL) - (*it).second->receiveSearchResult(gnsr->result_data,gnsr->result_data_len) ; + (*it).second->receiveSearchResult(gnsr->request_id,gnsr->result_data,gnsr->result_data_len) ; } } diff --git a/libretroshare/src/turtle/turtleclientservice.h b/libretroshare/src/turtle/turtleclientservice.h index 277c8d5c3..9bdd15ad4 100644 --- a/libretroshare/src/turtle/turtleclientservice.h +++ b/libretroshare/src/turtle/turtleclientservice.h @@ -117,7 +117,7 @@ class RsTurtleClientService * \param search_result_data result data. Memory ownership is owned by the turtle router. So do not delete! * \param search_result_data length of result data */ - virtual void receiveSearchResult(unsigned char * /*search_result_data*/,uint32_t /*search_result_data_len*/) + virtual void receiveSearchResult(TurtleSearchRequestId /* request_id */,unsigned char * /*search_result_data*/,uint32_t /*search_result_data_len*/) { std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ; } From be1e127a93e3a704c825b4c3a116d989c8c2ab35 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 11 Jun 2018 22:00:03 +0200 Subject: [PATCH 43/84] added test search functions in rsgxsnetservice --- libretroshare/src/gxs/rsgxsnetservice.cc | 32 ++++++++++++++++++++++-- libretroshare/src/gxs/rsgxsnettunnel.cc | 5 ++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index d83ceed54..8636bc6b0 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5112,8 +5112,36 @@ void RsGxsNetService::turtleSearchRequest(const std::string& match_string) mGxsNetTunnel->turtleSearchRequest(match_string,this) ; } +static bool termSearch(const std::string& src, const std::string& substring) +{ + /* always ignore case */ + return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() ); +} + bool RsGxsNetService::search(const std::string& substring,std::list& group_infos) { -#warning MISSING CODE HERE! - return true ; + RsGxsGrpMetaTemporaryMap grpMetaMap; + mDataStore->retrieveGxsGrpMetaData(grpMetaMap); + + RsGroupNetworkStats stats ; + + for(auto it(grpMetaMap.begin());it!=grpMetaMap.end();++it) + if(termSearch(it->second->mGroupName,substring)) + { + getGroupNetworkStats(it->first,stats) ; + + RsGxsGroupSummary s ; + s.group_id = it->first ; + s.group_name = it->second->mGroupName ; + s.group_description = it->second->mGroupName ; // to be filled with something better when we use the real search + s.search_context = it->second->mGroupName ; + s.author_id = it->second->mAuthorId; + s.publish_ts = it->second->mPublishTs; + s.number_of_messages = stats.mMaxVisibleCount ; + s.last_message_ts = stats.mLastGroupModificationTS ; + + group_infos.push_back(s) ; + } + + return !group_infos.empty(); } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 7cd657a55..48382754b 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -1048,7 +1048,12 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id if(result_gs != NULL) { + std::cerr << "Received group summary result for search request " << std::hex << request_id << " for service " << result_gs->service << std::dec << ": " << std::endl; + for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) + std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl; + +#warning MISSING CODE HERE - data should be passed up to UI in some way } } From 7a135c5c43750bc8ff279b3434d3dec910254d05 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 13 Jun 2018 22:46:27 +0200 Subject: [PATCH 44/84] added quick and dirty turtle search for channels in Files search tab --- libretroshare/src/gxs/rsgenexchange.h | 2 + libretroshare/src/gxs/rsgxsnetservice.cc | 8 +-- libretroshare/src/gxs/rsgxsnetservice.h | 4 +- libretroshare/src/gxs/rsgxsnettunnel.cc | 2 +- libretroshare/src/gxs/rsnxs.h | 5 +- libretroshare/src/retroshare/rsgxschannels.h | 5 +- libretroshare/src/services/p3gxschannels.cc | 6 +- libretroshare/src/services/p3gxschannels.h | 4 +- .../src/gui/FileTransfer/SearchDialog.cpp | 35 +++++++---- .../src/gui/FileTransfer/SearchDialog.ui | 59 +++++++++++++------ 10 files changed, 86 insertions(+), 44 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index 98b1d5d30..1e51d8b04 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -764,6 +764,8 @@ protected: */ int createMessage(RsNxsMsg* msg); + RsNetworkExchangeService *netService() const { return mNetService ; } + private: /*! * convenience function to create sign diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 8636bc6b0..da94cb002 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5103,13 +5103,13 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid) return true; } -void RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id) +TurtleRequestId RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id) { - mGxsNetTunnel->turtleGroupRequest(group_id,this) ; + return mGxsNetTunnel->turtleGroupRequest(group_id,this) ; } -void RsGxsNetService::turtleSearchRequest(const std::string& match_string) +TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_string) { - mGxsNetTunnel->turtleSearchRequest(match_string,this) ; + return mGxsNetTunnel->turtleSearchRequest(match_string,this) ; } static bool termSearch(const std::string& src, const std::string& substring) diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 9643f49bf..db2de16b8 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -127,8 +127,8 @@ public: * These four methods are used to request distant search and receive the results. * \param group_id */ - virtual void turtleGroupRequest(const RsGxsGroupId& group_id); - virtual void turtleSearchRequest(const std::string& match_string); + virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id); + virtual TurtleRequestId turtleSearchRequest(const std::string& match_string); virtual bool search(const std::string& substring,std::list& group_infos) ; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 48382754b..de67547d6 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -958,7 +958,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleGroupRequest(const RsGxsGroupId& gr TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) { - GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string\"" << match_string << "\"" << std::endl; + GXS_NET_TUNNEL_DEBUG() << ": starting a turtle search request for string \"" << match_string << "\"" << std::endl; RsGxsNetTunnelTurtleSearchSubstringItem search_item ; search_item.substring_match = match_string ; diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 5dfcde9d6..9d3673f1c 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -36,6 +36,7 @@ #include "services/p3service.h" #include "retroshare/rsreputations.h" #include "retroshare/rsidentity.h" +#include "retroshare/rsturtle.h" #include "rsgds.h" /*! @@ -102,8 +103,8 @@ public: virtual uint32_t getDefaultSyncAge() =0; virtual uint32_t getDefaultKeepAge() =0; - virtual void turtleGroupRequest(const RsGxsGroupId& group_id)=0; - virtual void turtleSearchRequest(const std::string& match_string)=0; + virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; + virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; virtual bool search(const std::string& substring,std::list& group_infos) =0; diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index 175c7fc22..6bad3ba21 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -33,6 +33,7 @@ #include "retroshare/rstokenservice.h" #include "retroshare/rsgxsifacehelper.h" #include "retroshare/rsgxscommon.h" +#include "retroshare/rsturtle.h" @@ -96,8 +97,8 @@ virtual bool getPostData(const uint32_t &token, std::vector &p //virtual bool createComment(uint32_t &token, RsGxsComment &comment) = 0; //virtual bool createVote(uint32_t &token, RsGxsVote &vote) = 0; - virtual void turtleGroupRequest(const RsGxsGroupId& group_id)=0; - virtual void turtleSearchRequest(const std::string& match_string)=0; + virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; + virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; ////////////////////////////////////////////////////////////////////////////// virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0; diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index 7d36048de..493779a8c 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -1695,10 +1695,12 @@ void p3GxsChannels::handle_event(uint32_t event_type, const std::string &elabel) } } -void p3GxsChannels::turtleGroupRequest(const RsGxsGroupId& group_id) +TurtleRequestId p3GxsChannels::turtleGroupRequest(const RsGxsGroupId& group_id) { + return netService()->turtleGroupRequest(group_id) ; } -void p3GxsChannels::turtleSearchRequest(const std::string& match_string) +TurtleRequestId p3GxsChannels::turtleSearchRequest(const std::string& match_string) { + return netService()->turtleSearchRequest(match_string) ; } diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index c166dbc2f..685439211 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -72,8 +72,8 @@ virtual void service_tick(); virtual bool saveList(bool &cleanup, std::list&saveList); // @see p3Config::saveList(bool &cleanup, std::list&) virtual bool loadList(std::list& loadList); // @see p3Config::loadList(std::list&) - virtual void turtleGroupRequest(const RsGxsGroupId& group_id); - virtual void turtleSearchRequest(const std::string& match_string); + virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id); + virtual TurtleRequestId turtleSearchRequest(const std::string& match_string); // Overloaded to cache new groups. virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet); diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp index f1c2e2fb6..6e2bc4223 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp @@ -38,6 +38,7 @@ #include "gui/common/RSTreeWidgetItem.h" #include "util/QtVersion.h" +#include "retroshare/rsgxschannels.h" #include #include #include @@ -165,8 +166,10 @@ SearchDialog::SearchDialog(QWidget *parent) QHeaderView_setSectionResizeModeColumn(_smheader, SS_KEYWORDS_COL, QHeaderView::Interactive); QHeaderView_setSectionResizeModeColumn(_smheader, SS_RESULTS_COL, QHeaderView::Interactive); - _smheader->resizeSection ( SS_KEYWORDS_COL, 160 ); - _smheader->resizeSection ( SS_RESULTS_COL, 50 ); + float f = QFontMetricsF(font()).height()/14.0 ; + + _smheader->resizeSection ( SS_KEYWORDS_COL, 160*f ); + _smheader->resizeSection ( SS_RESULTS_COL, 50*f ); ui.searchResultWidget->setColumnCount(SR_COL_COUNT); _smheader = ui.searchResultWidget->header () ; @@ -174,12 +177,12 @@ SearchDialog::SearchDialog(QWidget *parent) QHeaderView_setSectionResizeModeColumn(_smheader, SR_SIZE_COL, QHeaderView::Interactive); QHeaderView_setSectionResizeModeColumn(_smheader, SR_SOURCES_COL, QHeaderView::Interactive); - _smheader->resizeSection ( SR_NAME_COL, 240 ); - _smheader->resizeSection ( SR_SIZE_COL, 75 ); - _smheader->resizeSection ( SR_SOURCES_COL, 75 ); - _smheader->resizeSection ( SR_TYPE_COL, 75 ); - _smheader->resizeSection ( SR_AGE_COL, 90 ); - _smheader->resizeSection ( SR_HASH_COL, 240 ); + _smheader->resizeSection ( SR_NAME_COL, 240*f ); + _smheader->resizeSection ( SR_SIZE_COL, 75*f ); + _smheader->resizeSection ( SR_SOURCES_COL, 75*f ); + _smheader->resizeSection ( SR_TYPE_COL, 75*f ); + _smheader->resizeSection ( SR_AGE_COL, 90*f ); + _smheader->resizeSection ( SR_HASH_COL, 240*f ); // set header text aligment QTreeWidgetItem * headerItem = ui.searchResultWidget->headerItem(); @@ -201,10 +204,11 @@ SearchDialog::SearchDialog(QWidget *parent) // load settings processSettings(true); - ui._ownFiles_CB->setMinimumWidth(20); - ui._friendListsearch_SB->setMinimumWidth(20); - ui._anonF2Fsearch_CB->setMinimumWidth(20); - ui.label->setMinimumWidth(20); + ui._channels_CB->setMinimumWidth(20 * f); + ui._ownFiles_CB->setMinimumWidth(20*f); + ui._friendListsearch_SB->setMinimumWidth(20*f); + ui._anonF2Fsearch_CB->setMinimumWidth(20*f); + ui.label->setMinimumWidth(20*f); // workaround for Qt bug, be solved in next Qt release 4.7.0 // https://bugreports.qt-project.org/browse/QTBUG-8270 @@ -862,6 +866,13 @@ void SearchDialog::searchKeywords(const QString& keywords) else req_id = rsFiles->turtleSearch(lin_exp) ; } + else if(ui._channels_CB->isChecked()) + { + if(n==1) + req_id = rsGxsChannels->turtleSearchRequest(words.front()) ; + else + QMessageBox::critical(this,"Cannot search multiple words yet.","Search for multiple words is not implemented yet.") ; + } else req_id = RSRandom::random_u32() ; // generate a random 32 bits request id diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.ui b/retroshare-gui/src/gui/FileTransfer/SearchDialog.ui index 5411dd3c8..e6f6ded74 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.ui +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.ui @@ -6,8 +6,8 @@ 0 0 - 758 - 339 + 1531 + 889 @@ -32,7 +32,16 @@ QFrame::Sunken - + + 2 + + + 2 + + + 2 + + 2 @@ -55,15 +64,24 @@ + + 0 + + + 0 + + + 0 + + + 0 + 0 1 - - 0 - @@ -99,6 +117,13 @@ + + + + Channels + + + @@ -307,7 +332,7 @@ Any - + :/images/FileTypeAny.png:/images/FileTypeAny.png @@ -316,7 +341,7 @@ Archive - + :/images/FileTypeArchive.png:/images/FileTypeArchive.png @@ -325,7 +350,7 @@ Audio - + :/images/FileTypeAudio.png:/images/FileTypeAudio.png @@ -334,7 +359,7 @@ CD-Image - + :/images/FileTypeCDImage.png:/images/FileTypeCDImage.png @@ -343,7 +368,7 @@ Document - + :/images/FileTypeDocument.png:/images/FileTypeDocument.png @@ -352,7 +377,7 @@ Picture - + :/images/FileTypePicture.png:/images/FileTypePicture.png @@ -361,7 +386,7 @@ Program - + :/images/FileTypeProgram.png:/images/FileTypeProgram.png @@ -370,7 +395,7 @@ Video - + :/images/FileTypeVideo.png:/images/FileTypeVideo.png @@ -379,7 +404,7 @@ Directory - + :/images/folder16.png:/images/folder16.png @@ -406,7 +431,7 @@ Download selected - + :/images/download16.png:/images/download16.png @@ -432,7 +457,7 @@ - + From 84194b6234c9156f03fb43b193a9aa4b94ef4a84 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 16 Jun 2018 22:39:35 +0200 Subject: [PATCH 45/84] sereral fixes to GXS distant search --- libretroshare/src/ft/ftserver.h | 2 + libretroshare/src/grouter/p3grouter.h | 1 + libretroshare/src/gxs/rsgxsnetservice.cc | 9 +- libretroshare/src/gxs/rsgxsnettunnel.cc | 30 +++- libretroshare/src/gxs/rsgxsnettunnel.h | 8 + libretroshare/src/gxstunnel/p3gxstunnel.h | 2 + libretroshare/src/rsserver/rsinit.cc | 5 + libretroshare/src/services/p3idservice.cc | 1 - libretroshare/src/turtle/p3turtle.cc | 176 +++++++++++++--------- libretroshare/src/turtle/p3turtle.h | 8 +- libretroshare/src/turtle/rsturtleitem.cc | 4 +- libretroshare/src/turtle/rsturtleitem.h | 33 ++-- 12 files changed, 172 insertions(+), 107 deletions(-) diff --git a/libretroshare/src/ft/ftserver.h b/libretroshare/src/ft/ftserver.h index 0833127cf..cd0101933 100644 --- a/libretroshare/src/ft/ftserver.h +++ b/libretroshare/src/ft/ftserver.h @@ -95,6 +95,8 @@ public: // Implements RsTurtleClientService // + + uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; } virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ; virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ; //virtual void receiveSearchResult(RsTurtleSearchResultItem *item);// TODO diff --git a/libretroshare/src/grouter/p3grouter.h b/libretroshare/src/grouter/p3grouter.h index 8a3a54000..a4bc280ff 100644 --- a/libretroshare/src/grouter/p3grouter.h +++ b/libretroshare/src/grouter/p3grouter.h @@ -208,6 +208,7 @@ protected: // Interaction with turtle router // //===================================================// + uint16_t serviceId() const { return RS_SERVICE_TYPE_GROUTER; } virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) ; virtual void receiveTurtleData(const RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/); virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) ; diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index da94cb002..3a5c9ecb2 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -272,12 +272,12 @@ NXS_NET_DEBUG_8 gxs distant sync ***/ -#define NXS_NET_DEBUG_0 1 -#define NXS_NET_DEBUG_1 1 +//#define NXS_NET_DEBUG_0 1 +//#define NXS_NET_DEBUG_1 1 //#define NXS_NET_DEBUG_2 1 //#define NXS_NET_DEBUG_3 1 //#define NXS_NET_DEBUG_4 1 -#define NXS_NET_DEBUG_5 1 +//#define NXS_NET_DEBUG_5 1 //#define NXS_NET_DEBUG_6 1 //#define NXS_NET_DEBUG_7 1 #define NXS_NET_DEBUG_8 1 @@ -5143,5 +5143,8 @@ bool RsGxsNetService::search(const std::string& substring,std::listturtleSearch(mem,size,this) ; } bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size) { - GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; + GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_request_data,&search_request_data_len) ; @@ -985,14 +997,20 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d if(substring_sr != NULL) { - auto it = mSearchableServices.find(substring_sr->service) ; + GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl; std::list results ; + RS_STACK_MUTEX(mGxsNetTunnelMtx); + + auto it = mSearchableServices.find(substring_sr->service) ; + if(it != mSearchableServices.end() && it->second->search(substring_sr->substring_match,results)) { RsGxsNetTunnelTurtleSearchGroupSummaryItem search_result_item ; + GXS_NET_TUNNEL_DEBUG() << " : " << results.size() << " result found. Sending back." << std::endl; + search_result_item.service = substring_sr->service ; search_result_item.group_infos = results ; @@ -1044,11 +1062,13 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id { RsItem *item = RsGxsNetTunnelSerializer().deserialise(search_result_data,&search_result_data_len); + GXS_NET_TUNNEL_DEBUG() << " : received search result for search request " << std::hex << request_id << "" << std::endl; + RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast(item) ; if(result_gs != NULL) { - std::cerr << "Received group summary result for search request " << std::hex << request_id << " for service " << result_gs->service << std::dec << ": " << std::endl; + GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 22917f35b..ade780fe8 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -187,6 +187,14 @@ public: */ bool getVirtualPeers(std::list& peers) ; // returns the virtual peers for this service + /*! + * \brief serviceId + * Overloads the method in RsTurtleClientService. + * \return + * The service id for RsGxsNetTunnel. + */ + uint16_t serviceId() const ; + /*! * \brief sendData * send data to this virtual peer, and takes memory ownership (deletes the item) diff --git a/libretroshare/src/gxstunnel/p3gxstunnel.h b/libretroshare/src/gxstunnel/p3gxstunnel.h index cff0fd126..a5a53b437 100644 --- a/libretroshare/src/gxstunnel/p3gxstunnel.h +++ b/libretroshare/src/gxstunnel/p3gxstunnel.h @@ -127,6 +127,8 @@ public: explicit p3GxsTunnelService(RsGixs *pids) ; virtual void connectToTurtleRouter(p3turtle *) ; + uint16_t serviceId() const { return RS_SERVICE_TYPE_GXS_TUNNEL ; } + // Creates the invite if the public key of the distant peer is available. // Om success, stores the invite in the map above, so that we can respond to tunnel requests. // diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 7fd67fa27..34b68ef11 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1621,6 +1621,11 @@ int RsServer::StartupRetroShare() serviceCtrl->registerServiceMonitor(mBwCtrl, mBwCtrl->getServiceInfo().mServiceType); /**************************************************************************/ + // Turtle search for GXS services + + mGxsNetTunnel->registerSearchableService(gxschannels_ns) ; + + /**************************************************************************/ //mConfigMgr->addConfiguration("ftserver.cfg", ftserver); // diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 5618568fa..eff0f5cb5 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -47,7 +47,6 @@ #include #include -#define DEBUG_IDS 1 /**** * #define DEBUG_IDS 1 * #define DEBUG_RECOGN 1 diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 1232b950c..233703492 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -930,7 +930,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) #endif std::list search_results ; - item->performLocalSearch(req,search_results) ; + locked_performLocalSearch(item,req,search_results) ; for(auto it(search_results.begin());it!=search_results.end();++it) sendItem(*it) ; @@ -999,6 +999,97 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) #endif } +// This function should be removed in the future, when file search will also use generic search items. + +void p3turtle::locked_performLocalSearch(RsTurtleSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& search_results) +{ + RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast(item) ; + + if(ftsearch != NULL) + { + locked_performLocalSearch_files(ftsearch,req,search_results) ; + return ; + } + + RsTurtleGenericSearchRequestItem *gnsearch = dynamic_cast(item) ; + + if(gnsearch != NULL) + { + locked_performLocalSearch_generic(gnsearch,req,search_results) ; + return ; + } +} + +void p3turtle::locked_performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) +{ + unsigned char *search_result_data = NULL ; + uint32_t search_result_data_len = 0 ; + + auto it = _registered_services.find(item->service_id) ; + + if(it == _registered_services.end()) + return ; + + if(it->second->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len)) + { + RsTurtleGenericSearchResultItem *result_item = new RsTurtleGenericSearchResultItem ; + + result_item->result_data = search_result_data ; + result_item->result_data_len = search_result_data_len ; + + result.push_back(result_item) ; + } +} + +void p3turtle::locked_performLocalSearch_files(RsTurtleFileSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) +{ +#ifdef P3TURTLE_DEBUG + std::cerr << "Performing rsFiles->search()" << std::endl ; +#endif + // now, search! + std::list initialResults ; + item->search(initialResults) ; + +#ifdef P3TURTLE_DEBUG + std::cerr << initialResults.size() << " matches found." << std::endl ; +#endif + result.clear() ; + RsTurtleFTSearchResultItem *res_item = NULL ; + uint32_t item_size = 0 ; + + static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ; + + for(auto it(initialResults.begin());it!=initialResults.end();++it) + { + if(res_item == NULL) + { + res_item = new RsTurtleFTSearchResultItem ; + item_size = 0 ; + + res_item->depth = 0 ; + res_item->request_id = item->request_id ; + res_item->PeerId(item->PeerId()) ; // send back to the same guy + + result.push_back(res_item) ; + } + res_item->result.push_back(*it); + + // Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity. + // + ++req.result_count ; // increase hit number for this particular search request. + + item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ; + + if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + { +#ifdef P3TURTLE_DEBUG + std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ; +#endif + res_item = NULL ; // forces creation of a new item. + } + } +} + void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) { std::list > results_to_notify_off_mutex ; @@ -1771,59 +1862,7 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item) // ------------------------------ IO with libretroshare ----------------------------// // -----------------------------------------------------------------------------------// // -void RsTurtleFileSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const -{ -#ifdef P3TURTLE_DEBUG - std::cerr << "Performing rsFiles->search()" << std::endl ; -#endif - // now, search! - std::list initialResults ; - search(initialResults) ; -#ifdef P3TURTLE_DEBUG - std::cerr << initialResults.size() << " matches found." << std::endl ; -#endif - result.clear() ; - RsTurtleFTSearchResultItem *res_item = NULL ; - uint32_t item_size = 0 ; - - static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ; - - for(auto it(initialResults.begin());it!=initialResults.end();++it) - { - if(res_item == NULL) - { - res_item = new RsTurtleFTSearchResultItem ; - item_size = 0 ; - - res_item->depth = 0 ; - res_item->request_id = request_id ; - res_item->PeerId(PeerId()) ; // send back to the same guy - - result.push_back(res_item) ; - } - res_item->result.push_back(*it); - - // Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity. - // - ++req.result_count ; // increase hit number for this particular search request. - - item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ; - - if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) - { -#ifdef P3TURTLE_DEBUG - std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ; -#endif - res_item = NULL ; // forces creation of a new item. - } - } -} - -void RsTurtleGenericSearchRequestItem::performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const -{ - std::cerr << "(EE) p3turtle: Missing code to perform actual GXS search" << std::endl; -} void RsTurtleStringSearchRequestItem::search(std::list& result) const { @@ -1967,22 +2006,22 @@ TurtleRequestId p3turtle::turtleSearch(unsigned char *search_bin_data,uint32_t s // Form a request packet that simulates a request from us. // - RsTurtleGenericSearchRequestItem *item = new RsTurtleGenericSearchRequestItem ; + RsTurtleGenericSearchRequestItem item ; #ifdef P3TURTLE_DEBUG std::cerr << "performing search. OwnId = " << _own_id << std::endl ; #endif - item->PeerId(_own_id) ; - item->service_id = client_service->serviceId(); - item->search_data = search_bin_data ; - item->search_data_len = search_bin_data_len ; - item->request_id = id ; - item->depth = 0 ; + item.PeerId(_own_id) ; + item.service_id = client_service->serviceId(); + item.search_data = search_bin_data ; + item.search_data_len = search_bin_data_len ; + item.request_id = id ; + item.depth = 0 ; // send it - handleSearchRequest(item) ; + handleSearchRequest(&item) ; return id ; } @@ -2056,10 +2095,10 @@ bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& if(_registered_services.empty()) std::cerr << "Turtle router has no services registered. Tunnel requests cannot be handled." << std::endl; - for(std::list::const_iterator it(_registered_services.begin());it!=_registered_services.end();++it) - if( (*it)->handleTunnelRequest(hash,peer_id)) + for(auto it(_registered_services.begin());it!=_registered_services.end();++it) + if( (*it).second->handleTunnelRequest(hash,peer_id)) { - service = *it ; + service = it->second ; return true ; } @@ -2069,14 +2108,9 @@ bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& void p3turtle::registerTunnelService(RsTurtleClientService *service) { -#ifdef P3TURTLE_DEBUG - for(std::list::const_iterator it(_registered_services.begin());it!=_registered_services.end();++it) - if(service == *it) - throw std::runtime_error("p3turtle::registerTunnelService(): Cannot register the same service twice. Please fix the code!") ; -#endif - std::cerr << "p3turtle: registered new tunnel service " << (void*)service << std::endl; + std::cerr << "p3turtle: registered new tunnel service with ID=" << std::hex << service->serviceId() << std::dec << " and pointer " << (void*)service << std::endl; - _registered_services.push_back(service) ; + _registered_services[service->serviceId()] = service ; _serialiser->registerClientService(service) ; } diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index 04d90c812..0c30653f3 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -397,7 +397,9 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config //------ Functions connecting the turtle router to other components.----------// /// Performs a search calling local cache and search structure. - void performLocalSearch(const std::string& match_string,std::list& result) ; + void locked_performLocalSearch (RsTurtleSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; + void locked_performLocalSearch_files (RsTurtleFileSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; + void locked_performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; /// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer. virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service); @@ -419,7 +421,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config std::map _tunnel_requests_origins ; /// stores adequate tunnels for each file hash locally managed - std::map _incoming_file_hashes ; + std::map _incoming_file_hashes ; /// stores file info for each file we provide. std::map _outgoing_tunnel_client_services ; @@ -434,7 +436,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config std::set _hashes_to_remove ; /// List of client services that have regitered. - std::list _registered_services ; + std::map _registered_services ; time_t _last_clean_time ; time_t _last_tunnel_management_time ; diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index ce3a4450a..d600b4877 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -75,9 +75,11 @@ RsTurtleSearchRequestItem *RsTurtleGenericSearchRequestItem::clone() const { RsTurtleGenericSearchRequestItem *sr = new RsTurtleGenericSearchRequestItem ; + memcpy(sr,this,sizeof(RsTurtleGenericSearchRequestItem)) ; + sr->search_data = (unsigned char*)rs_malloc(search_data_len) ; memcpy(sr->search_data,search_data,search_data_len) ; - sr->search_data_len = search_data_len ; + return sr ; } template<> uint32_t RsTypeSerializer::serial_size(const RsRegularExpression::LinearizedExpression& r) diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index 035490bc1..c5e3bc727 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -57,26 +57,18 @@ class RsTurtleItem: public RsItem // +---- RsTurtleSearchRequestItem // | | // | +---- RsTurtleFileSearchRequestItem -// | | | -// | | +---- RsTurtleFileSearchRequestItem -// | | | -// | | +---- RsTurtleStringSearchRequestItem -// | | | -// | | +---- RsTurtleReqExpSearchRequestItem +// | | | +// | | +---- RsTurtleStringSearchRequestItem +// | | | +// | | +---- RsTurtleReqExpSearchRequestItem // | | -// | +---- RsTurtleGxsSearchRequestItem -// | | -// | +---- RsTurtleGxsGroupRequestItem +// | +---- RsTurtleGenericSearchRequestItem // | // +---- RsTurtleSearchResultItem // | // +---- RsTurtleFTSearchResultItem // | -// +---- RsTurtleGxsSearchResultItem -// | -// +---- RsTurtleGxsSearchResultGroupSummaryItem -// | -// +---- RsTurtleGxsSearchResultGroupDataItem +// +---- RsTurtleGenericSearchResultItem // class RsTurtleSearchResultItem ; @@ -89,8 +81,6 @@ class RsTurtleSearchRequestItem: public RsTurtleItem virtual RsTurtleSearchRequestItem *clone() const = 0 ; // used for cloning in routing methods - virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const = 0 ; // abstracts the search method - virtual std::string GetKeywords() = 0; uint32_t request_id ; // randomly generated request id. @@ -103,9 +93,6 @@ class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {} virtual ~RsTurtleFileSearchRequestItem() {} - virtual void performLocalSearch(TurtleSearchRequestInfo& req,std::list&) const ; // abstracts the search method - - protected: virtual void search(std::list &) const =0; }; @@ -115,8 +102,9 @@ class RsTurtleStringSearchRequestItem: public RsTurtleFileSearchRequestItem RsTurtleStringSearchRequestItem() : RsTurtleFileSearchRequestItem(RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST) {} virtual ~RsTurtleStringSearchRequestItem() {} - std::string match_string ; // string to match + virtual void search(std::list &) const ; + std::string match_string ; // string to match std::string GetKeywords() { return match_string; } virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleStringSearchRequestItem(*this) ; } @@ -124,7 +112,6 @@ class RsTurtleStringSearchRequestItem: public RsTurtleFileSearchRequestItem void clear() { match_string.clear() ; } protected: - virtual void search(std::list &) const ; void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; @@ -144,10 +131,11 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleFileSearchRequestItem return exs; } + virtual void search(std::list &) const ; + virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; } void clear() { expr = RsRegularExpression::LinearizedExpression(); } protected: - virtual void search(std::list &) const ; void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); }; @@ -166,7 +154,6 @@ class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem virtual RsTurtleSearchRequestItem *clone() const ; void clear() { free(search_data); search_data=NULL; search_data_len=0; } - virtual void performLocalSearch(TurtleSearchRequestInfo &req, std::list& result) const; protected: void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); From c67084b7dea39b4707f6ce2d9bd9a2608f9422d3 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 17 Jun 2018 21:23:16 +0200 Subject: [PATCH 46/84] fixed mutex problem in turtle-GXS search --- libretroshare/src/turtle/p3turtle.cc | 125 +++++++++++++++--------- libretroshare/src/turtle/p3turtle.h | 16 +-- libretroshare/src/turtle/rsturtleitem.h | 3 + 3 files changed, 89 insertions(+), 55 deletions(-) diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 233703492..898491cf2 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -870,8 +870,6 @@ int p3turtle::handleIncoming() // void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) { - RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ - // take a look at the item and test against inconsistent values // - If the item destimation is @@ -891,26 +889,55 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) return ; } - if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE) { + RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ + + if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE) + { #ifdef P3TURTLE_DEBUG - std::cerr << " Dropping, because the search request cache is full." << std::endl ; + std::cerr << " Dropping, because the search request cache is full." << std::endl ; #endif - std::cerr << " More than " << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. A peer is probably trying to flood your network See the depth charts to find him." << std::endl; - return ; + std::cerr << " More than " << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. A peer is probably trying to flood your network See the depth charts to find him." << std::endl; + return ; + } + + // If the item contains an already handled search request, give up. This + // happens when the same search request gets relayed by different peers + // + if(_search_requests_origins.find(item->request_id) != _search_requests_origins.end()) + { +#ifdef P3TURTLE_DEBUG + std::cerr << " This is a bouncing request. Ignoring and deleting it." << std::endl ; +#endif + return ; + } } - // If the item contains an already handled search request, give up. This - // happens when the same search request gets relayed by different peers - // - if(_search_requests_origins.find(item->request_id) != _search_requests_origins.end()) + // Perform local search off-mutex,because this might call some services that are above turtle in the mutex chain. + + uint32_t search_result_count = 0; + + if(item->PeerId() != _own_id) // is the request not coming from us? { #ifdef P3TURTLE_DEBUG - std::cerr << " This is a bouncing request. Ignoring and deleting it." << std::endl ; + std::cerr << " Request not from us. Performing local search" << std::endl ; #endif - return ; + std::list search_results ; + + performLocalSearch(item,search_result_count,search_results) ; + + for(auto it(search_results.begin());it!=search_results.end();++it) + { + (*it)->request_id = item->request_id ; + (*it)->depth = 0 ; + (*it)->PeerId(item->PeerId()) ; + + sendItem(*it) ; + } } + RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ + // This is a new request. Let's add it to the request map, and forward it to // open peers. @@ -918,23 +945,9 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) req.origin = item->PeerId() ; req.time_stamp = time(NULL) ; req.depth = item->depth ; - req.result_count = 0; + req.result_count = search_result_count; req.keywords = item->GetKeywords() ; - - // If it's not for us, perform a local search. If something found, forward the search result back. - - if(item->PeerId() != _own_id) - { -#ifdef P3TURTLE_DEBUG - std::cerr << " Request not from us. Performing local search" << std::endl ; -#endif - std::list search_results ; - - locked_performLocalSearch(item,req,search_results) ; - - for(auto it(search_results.begin());it!=search_results.end();++it) - sendItem(*it) ; - } + req.service_id = item->serviceId() ; // if enough has been sent back already, do not sarch further @@ -1001,13 +1014,13 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) // This function should be removed in the future, when file search will also use generic search items. -void p3turtle::locked_performLocalSearch(RsTurtleSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& search_results) +void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list& search_results) { RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast(item) ; if(ftsearch != NULL) { - locked_performLocalSearch_files(ftsearch,req,search_results) ; + performLocalSearch_files(ftsearch,req_result_count,search_results) ; return ; } @@ -1015,22 +1028,29 @@ void p3turtle::locked_performLocalSearch(RsTurtleSearchRequestItem *item,TurtleS if(gnsearch != NULL) { - locked_performLocalSearch_generic(gnsearch,req,search_results) ; + performLocalSearch_generic(gnsearch,req_result_count,search_results) ; return ; } } -void p3turtle::locked_performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) +void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list& result) { unsigned char *search_result_data = NULL ; uint32_t search_result_data_len = 0 ; - auto it = _registered_services.find(item->service_id) ; + RsTurtleClientService *client = NULL ; - if(it == _registered_services.end()) - return ; + { + RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ + auto it = _registered_services.find(item->service_id) ; - if(it->second->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len)) + if(it == _registered_services.end()) + return ; + + client = it->second ; + } + + if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len)) { RsTurtleGenericSearchResultItem *result_item = new RsTurtleGenericSearchResultItem ; @@ -1041,7 +1061,7 @@ void p3turtle::locked_performLocalSearch_generic(RsTurtleGenericSearchRequestIte } } -void p3turtle::locked_performLocalSearch_files(RsTurtleFileSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) +void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list& result) { #ifdef P3TURTLE_DEBUG std::cerr << "Performing rsFiles->search()" << std::endl ; @@ -1066,21 +1086,17 @@ void p3turtle::locked_performLocalSearch_files(RsTurtleFileSearchRequestItem *it res_item = new RsTurtleFTSearchResultItem ; item_size = 0 ; - res_item->depth = 0 ; - res_item->request_id = item->request_id ; - res_item->PeerId(item->PeerId()) ; // send back to the same guy - result.push_back(res_item) ; } res_item->result.push_back(*it); // Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity. // - ++req.result_count ; // increase hit number for this particular search request. + ++req_result_count ; // increase hit number for this particular search request. item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ; - if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) { #ifdef P3TURTLE_DEBUG std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ; @@ -1119,7 +1135,12 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) { it->second.result_count += item->count() ; - results_to_notify_off_mutex.push_back(std::make_pair(item,it->second.client)) ; + auto it2 = _registered_services.find(it->second.service_id) ; + + if(it2 != _registered_services.end()) + results_to_notify_off_mutex.push_back(std::make_pair(item,it2->second)) ; + else + std::cerr << "(EE) cannot find client service for ID " << std::hex << it->second.service_id << std::dec << ": search result item will be dropped." << std::endl; } else { // Nope, so forward it back. @@ -2092,10 +2113,20 @@ void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *clie // bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& peer_id,RsTurtleClientService *& service) { - if(_registered_services.empty()) - std::cerr << "Turtle router has no services registered. Tunnel requests cannot be handled." << std::endl; + std::map client_map ; + { + RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/ - for(auto it(_registered_services.begin());it!=_registered_services.end();++it) + if(_registered_services.empty()) + { + std::cerr << "Turtle router has no services registered. Tunnel requests cannot be handled." << std::endl; + return false ; + } + + client_map = _registered_services ; + } + + for(auto it(client_map.begin());it!=client_map.end();++it) if( (*it).second->handleTunnelRequest(hash,peer_id)) { service = it->second ; diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index 0c30653f3..be34a56a7 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -169,11 +169,11 @@ class TurtleSearchRequestInfo { public: TurtlePeerId origin ; // where the request came from. - uint32_t time_stamp ; // last time the tunnel was actually used. Used for cleaning old tunnels. - int depth ; // depth of the request. Used to optimize tunnel length. - uint32_t result_count; // responses to this request. Useful to avoid spamming tunnel responses. - std::string keywords; - RsTurtleClientService *client;// client who issues the request. This is null if the request does not have a local origin. + uint32_t time_stamp ; // last time the tunnel was actually used. Used for cleaning old tunnels. + int depth ; // depth of the request. Used to optimize tunnel length. + uint32_t result_count; // responses to this request. Useful to avoid spamming tunnel responses. + std::string keywords; + uint16_t service_id; // ID of the client service who issues the request. This is null if the request does not have a local origin. }; class TurtleTunnelRequestInfo { @@ -397,9 +397,9 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config //------ Functions connecting the turtle router to other components.----------// /// Performs a search calling local cache and search structure. - void locked_performLocalSearch (RsTurtleSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; - void locked_performLocalSearch_files (RsTurtleFileSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; - void locked_performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item,TurtleSearchRequestInfo& req,std::list& result) ; + void performLocalSearch (RsTurtleSearchRequestItem *item, uint32_t& req_result_count,std::list& result) ; + void performLocalSearch_files (RsTurtleFileSearchRequestItem *item, uint32_t& req_result_count,std::list& result) ; + void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count,std::list& result) ; /// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer. virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service); diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index c5e3bc727..d35cbb8fd 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -82,6 +82,7 @@ class RsTurtleSearchRequestItem: public RsTurtleItem virtual RsTurtleSearchRequestItem *clone() const = 0 ; // used for cloning in routing methods virtual std::string GetKeywords() = 0; + virtual uint16_t serviceId() = 0 ; uint32_t request_id ; // randomly generated request id. uint16_t depth ; // Used for limiting search depth. @@ -93,6 +94,7 @@ class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {} virtual ~RsTurtleFileSearchRequestItem() {} + virtual uint16_t serviceId() { return RS_SERVICE_TYPE_FILE_TRANSFER ; } virtual void search(std::list &) const =0; }; @@ -150,6 +152,7 @@ class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem unsigned char *search_data ; std::string GetKeywords() { return std::string("Generic search " + RsUtil::BinToHex(search_data,search_data_len,10)); } + virtual uint16_t serviceId() { return service_id ; } virtual RsTurtleSearchRequestItem *clone() const ; void clear() { free(search_data); search_data=NULL; search_data_len=0; } From 91fd38d46fbdbaa1c72a454941f4d229879391ac Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Jun 2018 22:30:44 +0200 Subject: [PATCH 47/84] started GUI part for distant network search of groups --- .../src/gui/common/GroupTreeWidget.cpp | 13 +++++++++++++ .../src/gui/common/GroupTreeWidget.h | 2 ++ .../src/gui/common/GroupTreeWidget.ui | 3 +++ .../src/gui/gxs/GxsGroupFrameDialog.cpp | 18 ++++++++++++++++++ .../src/gui/gxs/GxsGroupFrameDialog.h | 5 +++++ .../src/gui/gxschannels/GxsChannelDialog.cpp | 7 +++++++ .../src/gui/gxschannels/GxsChannelDialog.h | 1 + 7 files changed, 49 insertions(+) diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index fc1cfae19..578d3844a 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -18,6 +18,8 @@ * Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. ****************************************************************/ +#include + #include "GroupTreeWidget.h" #include "ui_GroupTreeWidget.h" @@ -119,6 +121,10 @@ GroupTreeWidget::GroupTreeWidget(QWidget *parent) : ui->filterLineEdit->addFilter(QIcon(), tr("Description"), FILTER_DESC_INDEX , tr("Search Description")); ui->filterLineEdit->setCurrentFilter(FILTER_NAME_INDEX); + ui->distantSearchLineEdit->setPlaceholderText(tr("Search entire network...")) ; + + connect(ui->distantSearchLineEdit,SIGNAL(returnPressed()),this,SLOT(distantSearch())) ; + /* Initialize display button */ initDisplayMenu(ui->displayButton); @@ -749,6 +755,13 @@ void GroupTreeWidget::resort(QTreeWidgetItem *categoryItem) } } +void GroupTreeWidget::distantSearch() +{ + emit distantSearchRequested(ui->distantSearchLineEdit->text()); + + ui->distantSearchLineEdit->clear(); +} + void GroupTreeWidget::sort() { resort(NULL); diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.h b/retroshare-gui/src/gui/common/GroupTreeWidget.h index d1a708258..e025f78e0 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.h +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.h @@ -110,6 +110,7 @@ signals: void treeCustomContextMenuRequested(const QPoint &pos); void treeCurrentItemChanged(const QString &id); void treeItemActivated(const QString &id); + void distantSearchRequested(const QString&) ; protected: void changeEvent(QEvent *e); @@ -119,6 +120,7 @@ private slots: void currentItemChanged(QTreeWidgetItem *current, QTreeWidgetItem *previous); void itemActivated(QTreeWidgetItem *item, int column); void filterChanged(); + void distantSearch(); void sort(); diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.ui b/retroshare-gui/src/gui/common/GroupTreeWidget.ui index c4d3948b1..7591c6a1c 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.ui +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.ui @@ -124,6 +124,9 @@ + + + diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index fe2ce45be..bce728e7a 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -94,6 +94,7 @@ GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *p connect(ui->groupTreeWidget, SIGNAL(treeCustomContextMenuRequested(QPoint)), this, SLOT(groupTreeCustomPopupMenu(QPoint))); connect(ui->groupTreeWidget, SIGNAL(treeCurrentItemChanged(QString)), this, SLOT(changedGroup(QString))); connect(ui->groupTreeWidget->treeWidget(), SIGNAL(signalMouseMiddleButtonClicked(QTreeWidgetItem*)), this, SLOT(groupTreeMiddleButtonClicked(QTreeWidgetItem*))); + connect(ui->groupTreeWidget, SIGNAL(distantSearchRequested(const QString&)), this, SLOT(searchNetwork(const QString&))); connect(ui->messageTabWidget, SIGNAL(tabCloseRequested(int)), this, SLOT(messageTabCloseRequested(int))); connect(ui->messageTabWidget, SIGNAL(currentChanged(int)), this, SLOT(messageTabChanged(int))); @@ -1078,3 +1079,20 @@ void GxsGroupFrameDialog::loadRequest(const TokenQueue *queue, const TokenReques } } } + +TurtleRequestId GxsGroupFrameDialog::distantSearch(const QString& search_string) // this should be overloaded in the child class +{ + std::cerr << "Searching for \"" << search_string.toStdString() << "\". Function is not implemented yet." << std::endl; + return 0; +} + +void GxsGroupFrameDialog::searchNetwork(const QString& search_string) +{ + uint32_t request_id = distantSearch(search_string); + + if(request_id == 0) + return ; + + mSearchGroups[request_id] = ui->groupTreeWidget->addCategoryItem(tr("Search for")+ " \"" + search_string + "\"", QIcon(icon(ICON_SEARCH)), true); +} + diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 2c0dca594..69058382b 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -67,6 +67,7 @@ public: ICON_SUBSCRIBED_GROUP, ICON_POPULAR_GROUP, ICON_OTHER_GROUP, + ICON_SEARCH, ICON_DEFAULT }; @@ -129,11 +130,13 @@ private slots: void sharePublishKey(); void loadComment(const RsGxsGroupId &grpId, const QVector& msg_versions,const RsGxsMessageId &most_recent_msgId, const QString &title); + void searchNetwork(const QString &search_string) ; private: virtual QString text(TextType type) = 0; virtual QString icon(IconType type) = 0; virtual QString settingsGroupName() = 0; + virtual TurtleRequestId distantSearch(const QString& search_string) ; virtual GxsGroupDialog *createNewGroupDialog(TokenQueue *tokenQueue) = 0; virtual GxsGroupDialog *createGroupDialog(TokenQueue *tokenQueue, RsTokenService *tokenService, GxsGroupDialog::Mode mode, RsGxsGroupId groupId) = 0; @@ -201,6 +204,8 @@ private: Ui::GxsGroupFrameDialog *ui; std::list mCachedGroupMetas; + + std::map mSearchGroups ; }; #endif diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index 4bff7528b..acd9dae0e 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -137,6 +137,8 @@ QString GxsChannelDialog::icon(IconType type) return ":/images/folder_green.png"; case ICON_OTHER_GROUP: return ":/images/folder_yellow.png"; + case ICON_SEARCH: + return ":/images/find.png"; case ICON_DEFAULT: return ":/images/channels.png"; } @@ -334,3 +336,8 @@ void GxsChannelDialog::groupInfoToGroupItemInfo(const RsGroupMetaData &groupInfo groupItemInfo.icon = iconIt.value(); } } + +TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string) +{ + return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ; +} diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h index e1f3c75d0..3864fba13 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h @@ -51,6 +51,7 @@ protected: virtual QString getHelpString() const ; virtual void groupInfoToGroupItemInfo(const RsGroupMetaData &groupInfo, GroupItemInfo &groupItemInfo, const RsUserdata *userdata); + virtual TurtleRequestId distantSearch(const QString& search_string) ; private slots: void toggleAutoDownload(); void setDefaultDirectory(); From c79ceba4eed727a72f9425706628ee97b8b8babb Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Jun 2018 23:26:37 +0200 Subject: [PATCH 48/84] added remove buttons for ongoing search entries --- .../src/gui/common/GroupTreeWidget.cpp | 28 +++++++++++ .../src/gui/common/GroupTreeWidget.h | 6 +++ .../src/gui/gxs/GxsGroupFrameDialog.cpp | 49 +++++++++++++++++-- .../src/gui/gxs/GxsGroupFrameDialog.h | 3 ++ 4 files changed, 83 insertions(+), 3 deletions(-) diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index 578d3844a..f5fd77daf 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -59,6 +59,8 @@ #define ROLE_SUBSCRIBE_FLAGS Qt::UserRole + 8 #define ROLE_COLOR Qt::UserRole + 9 #define ROLE_SAVED_ICON Qt::UserRole + 10 +#define ROLE_SEARCH_STRING Qt::UserRole + 11 +#define ROLE_REQUEST_ID Qt::UserRole + 12 #define FILTER_NAME_INDEX 0 #define FILTER_DESC_INDEX 1 @@ -397,6 +399,32 @@ QTreeWidgetItem *GroupTreeWidget::addCategoryItem(const QString &name, const QIc return item; } +void GroupTreeWidget::removeSearchItem(QTreeWidgetItem *item) +{ + ui->treeWidget->takeTopLevelItem(ui->treeWidget->indexOfTopLevelItem(item)) ; +} + +QTreeWidgetItem *GroupTreeWidget::addSearchItem(const QString& search_string, uint32_t id, const QIcon& icon) +{ + QTreeWidgetItem *item = addCategoryItem(search_string,icon,true); + + item->setData(COLUMN_DATA,ROLE_SEARCH_STRING,search_string) ; + item->setData(COLUMN_DATA,ROLE_REQUEST_ID ,id) ; + + return item; +} + +bool GroupTreeWidget::isSearchRequestItem(QPoint &point,uint32_t& search_req_id) +{ + QTreeWidgetItem *item = ui->treeWidget->itemAt(point); + if (item == NULL) + return false; + + search_req_id = item->data(COLUMN_DATA, ROLE_REQUEST_ID).toUInt(); + + return search_req_id > 0; +} + QString GroupTreeWidget::itemId(QTreeWidgetItem *item) { if (item == NULL) { diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.h b/retroshare-gui/src/gui/common/GroupTreeWidget.h index e025f78e0..6b782cb7c 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.h +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.h @@ -82,6 +82,10 @@ public: // Add a new category item QTreeWidgetItem *addCategoryItem(const QString &name, const QIcon &icon, bool expand); + // Add a new search item + QTreeWidgetItem *addSearchItem(const QString& search_string, uint32_t id, const QIcon &icon) ; + void removeSearchItem(QTreeWidgetItem *item); + // Get id of item QString itemId(QTreeWidgetItem *item); QString itemIdAt(QPoint &point); @@ -90,6 +94,8 @@ public: // Set the unread count of an item void setUnreadCount(QTreeWidgetItem *item, int unreadCount); + bool isSearchRequestItem(QPoint &point,uint32_t& search_req_id); + QTreeWidgetItem *getItemFromId(const QString &id); QTreeWidgetItem *activateId(const QString &id, bool focus); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index bce728e7a..d9c2c0bdd 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -46,6 +46,7 @@ #define IMAGE_EDIT ":/images/edit_16.png" #define IMAGE_SHARE ":/images/share-icon-16.png" #define IMAGE_TABNEW ":/images/tab-new.png" +#define IMAGE_DELETE ":/images/delete.png" #define IMAGE_COMMENT "" #define TOKEN_TYPE_GROUP_SUMMARY 1 @@ -252,16 +253,55 @@ void GxsGroupFrameDialog::todo() QMessageBox::information(this, "Todo", text(TEXT_TODO)); } +void GxsGroupFrameDialog::removeCurrentSearch() +{ + QAction *action = dynamic_cast(sender()) ; + + if(!action) + return ; + + TurtleRequestId search_request_id = action->data().toUInt(); + + auto it = mSearchGroups.find(search_request_id) ; + + if(it == mSearchGroups.end()) + return ; + + ui->groupTreeWidget->removeSearchItem(it->second) ; + mSearchGroups.erase(it); +} + +void GxsGroupFrameDialog::removeAllSearches() +{ + for(auto it(mSearchGroups.begin());it!=mSearchGroups.end();++it) + ui->groupTreeWidget->removeSearchItem(it->second) ; + + mSearchGroups.clear(); +} void GxsGroupFrameDialog::groupTreeCustomPopupMenu(QPoint point) { + // First separately handle the case of search top level items + + TurtleRequestId search_request_id = 0 ; + + if(ui->groupTreeWidget->isSearchRequestItem(point,search_request_id)) + { + QMenu contextMnu(this); + + contextMnu.addAction(QIcon(IMAGE_DELETE), tr("Remove this search"), this, SLOT(removeCurrentSearch()))->setData(search_request_id); + contextMnu.addAction(QIcon(IMAGE_DELETE), tr("Remove all searches"), this, SLOT(removeAllSearches())); + contextMnu.exec(QCursor::pos()); + return ; + } + QString id = ui->groupTreeWidget->itemIdAt(point); if (id.isEmpty()) return; mGroupId = RsGxsGroupId(id.toStdString()); int subscribeFlags = ui->groupTreeWidget->subscribeFlags(QString::fromStdString(mGroupId.toStdString())); - bool isAdmin = IS_GROUP_ADMIN(subscribeFlags); - bool isPublisher = IS_GROUP_PUBLISHER(subscribeFlags); + bool isAdmin = IS_GROUP_ADMIN(subscribeFlags); + bool isPublisher = IS_GROUP_PUBLISHER(subscribeFlags); bool isSubscribed = IS_GROUP_SUBSCRIBED(subscribeFlags); QMenu contextMnu(this); @@ -1088,11 +1128,14 @@ TurtleRequestId GxsGroupFrameDialog::distantSearch(const QString& search_string) void GxsGroupFrameDialog::searchNetwork(const QString& search_string) { + if(search_string.isNull()) + return ; + uint32_t request_id = distantSearch(search_string); if(request_id == 0) return ; - mSearchGroups[request_id] = ui->groupTreeWidget->addCategoryItem(tr("Search for")+ " \"" + search_string + "\"", QIcon(icon(ICON_SEARCH)), true); + mSearchGroups[request_id] = ui->groupTreeWidget->addSearchItem(tr("Search for")+ " \"" + search_string + "\"",(uint32_t)request_id,QIcon(icon(ICON_SEARCH))); } diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 69058382b..4d56ca76e 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -130,7 +130,10 @@ private slots: void sharePublishKey(); void loadComment(const RsGxsGroupId &grpId, const QVector& msg_versions,const RsGxsMessageId &most_recent_msgId, const QString &title); + void searchNetwork(const QString &search_string) ; + void removeAllSearches(); + void removeCurrentSearch(); private: virtual QString text(TextType type) = 0; From 5cb48c27de3592ab8a95a577dad8b25805377867 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 21 Jun 2018 09:26:03 +0200 Subject: [PATCH 49/84] renamed a few constants of GXS notify system into more consistent values --- libretroshare/src/gxs/rsgenexchange.cc | 14 +++++++------- libretroshare/src/retroshare/rsgxsservice.h | 2 +- libretroshare/src/services/p3gxschannels.cc | 8 ++++---- libretroshare/src/services/p3gxscircles.cc | 4 ++-- libretroshare/src/services/p3gxsforums.cc | 6 +++--- libretroshare/src/services/p3postbase.cc | 4 ++-- retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp | 6 +++--- retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h | 2 +- 8 files changed, 23 insertions(+), 23 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index ba330e334..735a09b99 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1643,7 +1643,7 @@ void RsGenExchange::notifyReceivePublishKey(const RsGxsGroupId &grpId) { RS_STACK_MUTEX(mGenMtx); - RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISHKEY, true); + RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY, true); gc->mGrpIdList.push_back(grpId); mNotifications.push_back(gc); } @@ -2314,7 +2314,7 @@ void RsGenExchange::publishMsgs() if(!msgChangeMap.empty()) { - RsGxsMsgChange* ch = new RsGxsMsgChange(RsGxsNotify::TYPE_PUBLISH, false); + RsGxsMsgChange* ch = new RsGxsMsgChange(RsGxsNotify::TYPE_PUBLISHED, false); ch->msgChangeMap = msgChangeMap; mNotifications.push_back(ch); } @@ -2451,7 +2451,7 @@ void RsGenExchange::processGroupDelete() if(!grpDeleted.empty()) { - RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISH, false); + RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISHED, false); gc->mGrpIdList = grpDeleted; mNotifications.push_back(gc); } @@ -2760,7 +2760,7 @@ void RsGenExchange::publishGrps() if(!grpChanged.empty()) { - RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, true); + RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, true); gc->mGrpIdList = grpChanged; mNotifications.push_back(gc); #ifdef GEN_EXCH_DEBUG @@ -3026,7 +3026,7 @@ void RsGenExchange::processRecvdMessages() #endif mDataStore->storeMessage(msgs_to_store); - RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_RECEIVE, false); + RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_RECEIVED_NEW, false); c->msgChangeMap = msgIds; mNotifications.push_back(c); } @@ -3159,7 +3159,7 @@ void RsGenExchange::processRecvdGroups() if(!grpIds.empty()) { - RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, false); + RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, false); c->mGrpIdList = grpIds; mNotifications.push_back(c); mDataStore->storeGroup(grps_to_store); @@ -3243,7 +3243,7 @@ void RsGenExchange::performUpdateValidation() } // notify the client - RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, true); + RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, true); for(uint32_t i=0;i > GxsMsgRelatedMe struct RsGxsNotify { enum NotifyType - { TYPE_PUBLISH, TYPE_RECEIVE, TYPE_PROCESSED, TYPE_PUBLISHKEY }; + { TYPE_PUBLISHED, TYPE_RECEIVED_NEW, TYPE_PROCESSED, TYPE_RECEIVED_PUBLISHKEY }; virtual ~RsGxsNotify() {} virtual NotifyType getType() = 0; diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index 493779a8c..a9c54b789 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -237,7 +237,7 @@ void p3GxsChannels::notifyChanges(std::vector &changes) RsGxsMsgChange *msgChange = dynamic_cast(*it); if (msgChange) { - if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVE) + if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) { /* message received */ if (notify) @@ -296,10 +296,10 @@ void p3GxsChannels::notifyChanges(std::vector &changes) switch (grpChange->getType()) { case RsGxsNotify::TYPE_PROCESSED: - case RsGxsNotify::TYPE_PUBLISH: + case RsGxsNotify::TYPE_PUBLISHED: break; - case RsGxsNotify::TYPE_RECEIVE: + case RsGxsNotify::TYPE_RECEIVED_NEW: { /* group received */ std::list &grpList = grpChange->mGrpIdList; @@ -317,7 +317,7 @@ void p3GxsChannels::notifyChanges(std::vector &changes) break; } - case RsGxsNotify::TYPE_PUBLISHKEY: + case RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY: { /* group received */ std::list &grpList = grpChange->mGrpIdList; diff --git a/libretroshare/src/services/p3gxscircles.cc b/libretroshare/src/services/p3gxscircles.cc index 5e27e378d..ac8e3a3a5 100644 --- a/libretroshare/src/services/p3gxscircles.cc +++ b/libretroshare/src/services/p3gxscircles.cc @@ -218,7 +218,7 @@ void p3GxsCircles::notifyChanges(std::vector &changes) std::cerr << " Msgs for Group: " << mit->first << std::endl; #endif force_cache_reload(RsGxsCircleId(mit->first)); - if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVE) ) + if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) ) for (std::vector::const_iterator msgIdIt(mit->second.begin()), end(mit->second.end()); msgIdIt != end; ++msgIdIt) { const RsGxsMessageId& msgId = *msgIdIt; @@ -261,7 +261,7 @@ void p3GxsCircles::notifyChanges(std::vector &changes) std::cerr << " forcing cache loading for circle " << *git << " in order to trigger subscribe update." << std::endl; #endif force_cache_reload(RsGxsCircleId(*git)) ; - if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVE) ) + if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) ) notify->AddFeedItem(RS_FEED_ITEM_CIRCLE_INVIT_REC,RsGxsCircleId(*git).toStdString(),""); } diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index dd98f88d1..69d33441e 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -197,10 +197,10 @@ void p3GxsForums::notifyChanges(std::vector &changes) switch (c->getType()) { case RsGxsNotify::TYPE_PROCESSED: - case RsGxsNotify::TYPE_PUBLISH: + case RsGxsNotify::TYPE_PUBLISHED: break; - case RsGxsNotify::TYPE_RECEIVE: + case RsGxsNotify::TYPE_RECEIVED_NEW: { RsGxsMsgChange *msgChange = dynamic_cast(c); if (msgChange) @@ -242,7 +242,7 @@ void p3GxsForums::notifyChanges(std::vector &changes) break; } - case RsGxsNotify::TYPE_PUBLISHKEY: + case RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY: { RsGxsGroupChange *grpChange = dynamic_cast(*it); if (grpChange) diff --git a/libretroshare/src/services/p3postbase.cc b/libretroshare/src/services/p3postbase.cc index 0c37db1f1..5aaa6d88f 100644 --- a/libretroshare/src/services/p3postbase.cc +++ b/libretroshare/src/services/p3postbase.cc @@ -123,7 +123,7 @@ void p3PostBase::notifyChanges(std::vector &changes) // It could be taken a step further and directly request these msgs for an update. addGroupForProcessing(mit->first); - if (notify && msgChange->getType() == RsGxsNotify::TYPE_RECEIVE) + if (notify && msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) { std::vector::iterator mit1; for (mit1 = mit->second.begin(); mit1 != mit->second.end(); ++mit1) @@ -151,7 +151,7 @@ void p3PostBase::notifyChanges(std::vector &changes) std::cerr << std::endl; #endif - if (notify && groupChange->getType() == RsGxsNotify::TYPE_RECEIVE) + if (notify && groupChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) { notify->AddFeedItem(RS_FEED_ITEM_POSTED_NEW, git->toStdString()); } diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index d9c2c0bdd..e143278d1 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -93,7 +93,7 @@ GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *p mStateHelper->addWidget(TOKEN_TYPE_GROUP_SUMMARY, ui->loadingLabel, UISTATE_LOADING_VISIBLE); connect(ui->groupTreeWidget, SIGNAL(treeCustomContextMenuRequested(QPoint)), this, SLOT(groupTreeCustomPopupMenu(QPoint))); - connect(ui->groupTreeWidget, SIGNAL(treeCurrentItemChanged(QString)), this, SLOT(changedGroup(QString))); + connect(ui->groupTreeWidget, SIGNAL(treeCurrentItemChanged(QString)), this, SLOT(changedCurrentGroup(QString))); connect(ui->groupTreeWidget->treeWidget(), SIGNAL(signalMouseMiddleButtonClicked(QTreeWidgetItem*)), this, SLOT(groupTreeMiddleButtonClicked(QTreeWidgetItem*))); connect(ui->groupTreeWidget, SIGNAL(distantSearchRequested(const QString&)), this, SLOT(searchNetwork(const QString&))); connect(ui->messageTabWidget, SIGNAL(tabCloseRequested(int)), this, SLOT(messageTabCloseRequested(int))); @@ -630,7 +630,7 @@ bool GxsGroupFrameDialog::navigate(const RsGxsGroupId &groupId, const RsGxsMessa return false; } - changedGroup(groupIdString); + changedCurrentGroup(groupIdString); /* search exisiting tab */ GxsMessageFrameWidget *msgWidget = messageWidget(mGroupId, false); @@ -691,7 +691,7 @@ GxsCommentDialog *GxsGroupFrameDialog::commentWidget(const RsGxsMessageId& msgId return NULL; } -void GxsGroupFrameDialog::changedGroup(const QString &groupId) +void GxsGroupFrameDialog::changedCurrentGroup(const QString &groupId) { if (mInFill) { return; diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 4d56ca76e..3e5091857 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -108,7 +108,7 @@ private slots: void restoreGroupKeys(); void newGroup(); - void changedGroup(const QString &groupId); + void changedCurrentGroup(const QString &groupId); void groupTreeMiddleButtonClicked(QTreeWidgetItem *item); void openInNewTab(); void messageTabCloseRequested(int index); From 3981bc8e3b1d3b82bcb3650859e4ade612401214 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 21 Jun 2018 13:48:57 +0200 Subject: [PATCH 50/84] extended notification system to add distant search result notification --- libretroshare/src/gxs/rsgenexchange.cc | 20 +++++++- libretroshare/src/gxs/rsgenexchange.h | 13 ++++- libretroshare/src/gxs/rsgxsnetservice.cc | 50 ++++++++++++++++++- libretroshare/src/gxs/rsgxsnetservice.h | 14 ++++++ libretroshare/src/gxs/rsgxsnettunnel.cc | 25 +++++++--- libretroshare/src/gxs/rsnxs.h | 35 +++++++++++++ libretroshare/src/gxs/rsnxsobserver.h | 15 +++++- libretroshare/src/gxstrans/p3gxstrans.cc | 2 +- libretroshare/src/retroshare/rsgxsservice.h | 14 +++++- libretroshare/src/services/p3gxschannels.cc | 1 + libretroshare/src/services/p3gxsforums.cc | 1 + .../src/gui/gxs/GxsGroupFrameDialog.cpp | 2 +- 12 files changed, 174 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 735a09b99..f7609a8f9 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1571,7 +1571,7 @@ bool RsGenExchange::setAuthenPolicyFlag(const uint8_t &msgFlag, uint32_t& authen return true; } -void RsGenExchange::notifyNewGroups(std::vector &groups) +void RsGenExchange::receiveNewGroups(std::vector &groups) { RS_STACK_MUTEX(mGenMtx) ; @@ -1603,7 +1603,7 @@ void RsGenExchange::notifyNewGroups(std::vector &groups) } -void RsGenExchange::notifyNewMessages(std::vector& messages) +void RsGenExchange::receiveNewMessages(std::vector& messages) { RS_STACK_MUTEX(mGenMtx) ; @@ -1639,6 +1639,13 @@ void RsGenExchange::notifyNewMessages(std::vector& messages) } } +void RsGenExchange::receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId) +{ + RS_STACK_MUTEX(mGenMtx); + + RsGxsDistantSearchResultChange* gc = new RsGxsDistantSearchResultChange(id,grpId); + mNotifications.push_back(gc); +} void RsGenExchange::notifyReceivePublishKey(const RsGxsGroupId &grpId) { RS_STACK_MUTEX(mGenMtx); @@ -1833,6 +1840,15 @@ uint32_t RsGenExchange::getSyncPeriod(const RsGxsGroupId& grpId) return RS_GXS_DEFAULT_MSG_REQ_PERIOD; } +bool RsGenExchange::getDistantSearchResults(const TurtleRequestId& id,std::list& group_infos) +{ + return (mNetService!=NULL) && mNetService->getDistantSearchResults(id,group_infos) ; +} +bool RsGenExchange::clearDistantSearchResults(const TurtleRequestId& id) +{ + return (mNetService!=NULL) && mNetService->clearDistantSearchResults(id) ; +} + bool RsGenExchange::getGroupNetworkStats(const RsGxsGroupId& grpId,RsGroupNetworkStats& stats) { return (!mNetService) || mNetService->getGroupNetworkStats(grpId,stats) ; diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index 1e51d8b04..e08ba80bd 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -132,18 +132,24 @@ public: /*! * @param messages messages are deleted after function returns */ - virtual void notifyNewMessages(std::vector& messages); + virtual void receiveNewMessages(std::vector& messages); /*! * @param groups groups are deleted after function returns */ - virtual void notifyNewGroups(std::vector& groups); + virtual void receiveNewGroups(std::vector& groups); /*! * @param grpId group id */ virtual void notifyReceivePublishKey(const RsGxsGroupId &grpId); + /*! + * \brief notifyReceiveDistantSearchResults + * Should be called when new search results arrive. + * \param grpId + */ + virtual void receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId); /*! * @param grpId group id */ @@ -682,6 +688,9 @@ public: virtual void setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs) ; virtual bool getGroupNetworkStats(const RsGxsGroupId& grpId,RsGroupNetworkStats& stats); + virtual bool getDistantSearchResults(const TurtleRequestId& id,std::list& group_infos) ; + virtual bool clearDistantSearchResults(const TurtleRequestId& id); + uint16_t serviceType() const { return mServType ; } uint32_t serviceFullType() const { return ((uint32_t)mServType << 8) + (((uint32_t) RS_PKT_VERSION_SERVICE) << 24); } diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 3a5c9ecb2..e5dfbd05a 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -498,8 +498,8 @@ void RsGxsNetService::processObserverNotifications() mNewPublishKeysToNotify.clear() ; } - if(!grps_copy.empty()) mObserver->notifyNewGroups (grps_copy); - if(!msgs_copy.empty()) mObserver->notifyNewMessages(msgs_copy); + if(!grps_copy.empty()) mObserver->receiveNewGroups (grps_copy); + if(!msgs_copy.empty()) mObserver->receiveNewMessages(msgs_copy); for(std::set::const_iterator it(keys_copy.begin());it!=keys_copy.end();++it) mObserver->notifyReceivePublishKey(*it); @@ -5117,9 +5117,36 @@ static bool termSearch(const std::string& src, const std::string& substring) /* always ignore case */ return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() ); } +void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std::list& group_infos) +{ + RS_STACK_MUTEX(mNxsMutex) ; +#warning We should use some central way to do that. This might be very costly if done often. + + RsGxsGrpMetaTemporaryMap grpMeta; + std::map& search_results_map(mDistantSearchResults[req]) ; + + for(auto it(group_infos.begin());it!=group_infos.end();++it) + if(search_results_map.find((*it).group_id) == search_results_map.end()) + grpMeta[(*it).group_id] = NULL; + + mDataStore->retrieveGxsGrpMetaData(grpMeta); + + std::list filtered_results ; + + // only keep groups that are not locally known, and groups that are not already in the mDistantSearchResults structure + + for(auto it(group_infos.begin());it!=group_infos.end();++it) + if(grpMeta[(*it).group_id] == NULL) + { + filtered_results.push_back(*it) ; + search_results_map[(*it).group_id] = *it; + mObserver->receiveDistantSearchResults(req,(*it).group_id) ; + } +} bool RsGxsNetService::search(const std::string& substring,std::list& group_infos) { + RS_STACK_MUTEX(mNxsMutex) ; RsGxsGrpMetaTemporaryMap grpMetaMap; mDataStore->retrieveGxsGrpMetaData(grpMetaMap); @@ -5148,3 +5175,22 @@ bool RsGxsNetService::search(const std::string& substring,std::list& group_infos) +{ + RS_STACK_MUTEX(mNxsMutex) ; + auto it = mDistantSearchResults.find(id) ; + + if(it == mDistantSearchResults.end()) + return false ; + + for(auto it2(it->second.begin());it2!=it->second.end();++it2) + group_infos.push_back(it2->second); + return true; +} +bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id) +{ + RS_STACK_MUTEX(mNxsMutex) ; + mDistantSearchResults.erase(id); + return true ; +} diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index db2de16b8..799a2b503 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -131,6 +131,7 @@ public: virtual TurtleRequestId turtleSearchRequest(const std::string& match_string); virtual bool search(const std::string& substring,std::list& group_infos) ; + virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos); /*! * pauses synchronisation of subscribed groups and request for group id @@ -169,6 +170,16 @@ public: */ virtual bool getGroupNetworkStats(const RsGxsGroupId& id,RsGroupNetworkStats& stats) ; + /*! + * \brief getDistantSearchResults + * \param id Id of the search request previously issued + * \param group_infos Groups currently known for this search request. + * \return + * false if the id does not correspond to an ongoing distant search request. + */ + virtual bool getDistantSearchResults(const TurtleRequestId& id,std::list& group_infos) ; + virtual bool clearDistantSearchResults(const TurtleRequestId& id); + /*! * Used to inform the net service that we changed subscription status. That helps * optimising data transfer when e.g. unsubsribed groups are updated less often, etc @@ -598,6 +609,9 @@ private: std::set mNewStatsToNotify ; std::set mNewPublishKeysToNotify ; + // Distant search result map + std::map > mDistantSearchResults ; + void debugDump(); uint32_t mDefaultMsgStorePeriod ; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index a52b2f020..eabc32e3a 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -1066,15 +1066,26 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast(item) ; - if(result_gs != NULL) + if(result_gs == NULL) { - GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; - - for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) - std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl; - -#warning MISSING CODE HERE - data should be passed up to UI in some way + GXS_NET_TUNNEL_ERROR() << ": deserialized item is not a GroupSummary Item. Smething's wrong here." << std::endl; + return ; } + + GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; + + for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) + std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl; + + auto it = mSearchableServices.find(result_gs->service) ; + + if(it == mSearchableServices.end()) + { + GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is in the searchable services list." << std::endl; + return ; + } + + it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ; } diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 9d3673f1c..3f1cc9e25 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -103,9 +103,44 @@ public: virtual uint32_t getDefaultSyncAge() =0; virtual uint32_t getDefaultKeepAge() =0; + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// DISTANT SEARCH FUNCTIONS /// + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + /*! + * \brief turtleGroupRequest + * Requests a particular group meta data. The request protects the group ID. + * \param group_id + * \return + * returns the turtle request ID that might be associated to some results. + */ virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; + + /*! + * \brief turtleSearchRequest + * Uses distant search to match the substring to the group meta data. + * \param match_string + * \return + * returns the turtle request ID that might be associated to some results. + */ virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; + /*! + * \brief receiveTurtleSearchResults + * Called by turtle (through RsGxsNetTunnel) when new results are received + * \param req Turtle search request ID associated with this result + * \param group_infos Group summary information for the groups returned by the search + */ + virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos)=0; + /*! + * \brief getDistantSearchResults + * \param id + * \param group_infos + * \return + */ + virtual bool getDistantSearchResults(const TurtleRequestId& id,std::list& group_infos)=0 ; + virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0; + virtual bool search(const std::string& substring,std::list& group_infos) =0; /*! diff --git a/libretroshare/src/gxs/rsnxsobserver.h b/libretroshare/src/gxs/rsnxsobserver.h index 087842ef0..a16bc66af 100644 --- a/libretroshare/src/gxs/rsnxsobserver.h +++ b/libretroshare/src/gxs/rsnxsobserver.h @@ -29,6 +29,7 @@ #include #include "rsitems/rsnxsitems.h" +typedef uint32_t TurtleRequestId ; class RsNxsObserver { @@ -42,12 +43,22 @@ public: /*! * @param messages messages are deleted after function returns */ - virtual void notifyNewMessages(std::vector& messages) = 0; + virtual void receiveNewMessages(std::vector& messages) = 0; /*! * @param groups groups are deleted after function returns */ - virtual void notifyNewGroups(std::vector& groups) = 0; + virtual void receiveNewGroups(std::vector& groups) = 0; + + /*! + * \brief receiveDistantSearchResults + * Called when new distant search result arrive. + * \param grpId + */ + virtual void receiveDistantSearchResults(TurtleRequestId& /*id*/,const RsGxsGroupId& /*grpId*/) + { + std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl; + } /*! * @param grpId group id diff --git a/libretroshare/src/gxstrans/p3gxstrans.cc b/libretroshare/src/gxstrans/p3gxstrans.cc index d0dbb61b0..05c37cdb5 100644 --- a/libretroshare/src/gxstrans/p3gxstrans.cc +++ b/libretroshare/src/gxstrans/p3gxstrans.cc @@ -839,7 +839,7 @@ bool p3GxsTrans::dispatchDecryptedMail( const RsGxsId& authorId, #endif std::vector rcct; rcct.push_back(receipt); - RsGenExchange::notifyNewMessages(rcct); + RsGenExchange::receiveNewMessages(rcct); GxsTransClient* recipientService = NULL; { diff --git a/libretroshare/src/retroshare/rsgxsservice.h b/libretroshare/src/retroshare/rsgxsservice.h index 14f02ed64..83f73887b 100644 --- a/libretroshare/src/retroshare/rsgxsservice.h +++ b/libretroshare/src/retroshare/rsgxsservice.h @@ -6,6 +6,7 @@ #include "retroshare/rstokenservice.h" struct RsMsgMetaData ; +typedef uint32_t TurtleRequestId; typedef std::map > GxsMsgMetaMap; typedef std::map > GxsMsgRelatedMetaMap; @@ -18,7 +19,7 @@ typedef std::map > GxsMsgRelatedMe struct RsGxsNotify { enum NotifyType - { TYPE_PUBLISHED, TYPE_RECEIVED_NEW, TYPE_PROCESSED, TYPE_RECEIVED_PUBLISHKEY }; + { TYPE_PUBLISHED, TYPE_RECEIVED_NEW, TYPE_PROCESSED, TYPE_RECEIVED_PUBLISHKEY, TYPE_RECEIVED_DISTANT_SEARCH_RESULTS }; virtual ~RsGxsNotify() {} virtual NotifyType getType() = 0; @@ -39,6 +40,17 @@ private: bool mMetaChange; }; +class RsGxsDistantSearchResultChange: public RsGxsNotify +{ +public: + RsGxsDistantSearchResultChange(TurtleRequestId id,const RsGxsGroupId& group_id) : mRequestId(id),mGroupId(group_id){} + + NotifyType getType() { return TYPE_RECEIVED_DISTANT_SEARCH_RESULTS ; } +private: + TurtleRequestId mRequestId ; + RsGxsGroupId mGroupId; +}; + /*! * Relevant to message changes */ diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index a9c54b789..221825e9b 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -295,6 +295,7 @@ void p3GxsChannels::notifyChanges(std::vector &changes) { switch (grpChange->getType()) { + default: case RsGxsNotify::TYPE_PROCESSED: case RsGxsNotify::TYPE_PUBLISHED: break; diff --git a/libretroshare/src/services/p3gxsforums.cc b/libretroshare/src/services/p3gxsforums.cc index 69d33441e..2fa829c1c 100644 --- a/libretroshare/src/services/p3gxsforums.cc +++ b/libretroshare/src/services/p3gxsforums.cc @@ -196,6 +196,7 @@ void p3GxsForums::notifyChanges(std::vector &changes) switch (c->getType()) { + default: case RsGxsNotify::TYPE_PROCESSED: case RsGxsNotify::TYPE_PUBLISHED: break; diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index e143278d1..04e64258b 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -1122,7 +1122,7 @@ void GxsGroupFrameDialog::loadRequest(const TokenQueue *queue, const TokenReques TurtleRequestId GxsGroupFrameDialog::distantSearch(const QString& search_string) // this should be overloaded in the child class { - std::cerr << "Searching for \"" << search_string.toStdString() << "\". Function is not implemented yet." << std::endl; + std::cerr << "Searching for \"" << search_string.toStdString() << "\". Function is not overloaded, so nothing will happen." << std::endl; return 0; } From 6ccc7654d688c7eff72e57c247ffec585ecd34b3 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 23 Jun 2018 22:25:36 +0200 Subject: [PATCH 51/84] added code to notify GxsBroadcast system with new distant search results --- libretroshare/src/gxs/rsgenexchange.cc | 8 ++++++++ libretroshare/src/gxs/rsgxsnetservice.cc | 9 ++++++--- libretroshare/src/gxs/rsnxsobserver.h | 2 +- libretroshare/src/retroshare/rsgxsiface.h | 1 + libretroshare/src/retroshare/rsgxsservice.h | 2 +- retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp | 10 ++++++++++ retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h | 1 + .../src/gui/gxs/RsGxsUpdateBroadcastBase.cpp | 12 +++++++++++- .../src/gui/gxs/RsGxsUpdateBroadcastBase.h | 5 +++++ .../src/gui/gxs/RsGxsUpdateBroadcastPage.cpp | 5 +++++ .../src/gui/gxs/RsGxsUpdateBroadcastPage.h | 2 ++ .../src/gui/gxs/RsGxsUpdateBroadcastWidget.cpp | 4 ++++ .../src/gui/gxs/RsGxsUpdateBroadcastWidget.h | 2 ++ retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp | 3 +++ retroshare-gui/src/util/RsGxsUpdateBroadcast.h | 3 +++ 15 files changed, 63 insertions(+), 6 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 8da9e606a..43e515732 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1110,6 +1110,8 @@ void RsGenExchange::receiveChanges(std::vector& changes) RsGxsNotify* n = *vit; RsGxsGroupChange* gc; RsGxsMsgChange* mc; + RsGxsDistantSearchResultChange *gt; + if((mc = dynamic_cast(n)) != NULL) { if (mc->metaChange()) @@ -1132,6 +1134,10 @@ void RsGenExchange::receiveChanges(std::vector& changes) out.mGrps.splice(out.mGrps.end(), gc->mGrpIdList); } } + else if((gt = dynamic_cast(n)) != NULL) + { + out.mDistantSearchReqs.push_back(gt->mRequestId); + } else std::cerr << "(EE) Unknown changes type!!" << std::endl; @@ -1641,6 +1647,8 @@ void RsGenExchange::receiveDistantSearchResults(TurtleRequestId id,const RsGxsGr RsGxsDistantSearchResultChange* gc = new RsGxsDistantSearchResultChange(id,grpId); mNotifications.push_back(gc); + + std::cerr << "RsGenExchange::receiveDistantSearchResults(): received result for request " << std::hex << id << std::dec << std::endl; } void RsGenExchange::notifyReceivePublishKey(const RsGxsGroupId &grpId) { diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index f456f4d7e..2ac77b279 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5136,7 +5136,8 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std: // only keep groups that are not locally known, and groups that are not already in the mDistantSearchResults structure for(auto it(group_infos.begin());it!=group_infos.end();++it) - if(grpMeta[(*it).group_id] == NULL) +#warning Uncomment when done with testing! +// if(grpMeta[(*it).group_id] == NULL) { filtered_results.push_back(*it) ; search_results_map[(*it).group_id] = *it; @@ -5146,9 +5147,11 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std: bool RsGxsNetService::search(const std::string& substring,std::list& group_infos) { - RS_STACK_MUTEX(mNxsMutex) ; RsGxsGrpMetaTemporaryMap grpMetaMap; - mDataStore->retrieveGxsGrpMetaData(grpMetaMap); + { + RS_STACK_MUTEX(mNxsMutex) ; + mDataStore->retrieveGxsGrpMetaData(grpMetaMap); + } RsGroupNetworkStats stats ; diff --git a/libretroshare/src/gxs/rsnxsobserver.h b/libretroshare/src/gxs/rsnxsobserver.h index a16bc66af..b52d69d65 100644 --- a/libretroshare/src/gxs/rsnxsobserver.h +++ b/libretroshare/src/gxs/rsnxsobserver.h @@ -55,7 +55,7 @@ public: * Called when new distant search result arrive. * \param grpId */ - virtual void receiveDistantSearchResults(TurtleRequestId& /*id*/,const RsGxsGroupId& /*grpId*/) + virtual void receiveDistantSearchResults(TurtleRequestId /*id*/,const RsGxsGroupId& /*grpId*/) { std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl; } diff --git a/libretroshare/src/retroshare/rsgxsiface.h b/libretroshare/src/retroshare/rsgxsiface.h index 36c660ef8..b9c974f9c 100644 --- a/libretroshare/src/retroshare/rsgxsiface.h +++ b/libretroshare/src/retroshare/rsgxsiface.h @@ -44,6 +44,7 @@ public: std::map > mMsgsMeta; std::list mGrps; std::list mGrpsMeta; + std::list mDistantSearchReqs; }; /*! diff --git a/libretroshare/src/retroshare/rsgxsservice.h b/libretroshare/src/retroshare/rsgxsservice.h index affbeea50..17ce356ec 100644 --- a/libretroshare/src/retroshare/rsgxsservice.h +++ b/libretroshare/src/retroshare/rsgxsservice.h @@ -46,7 +46,7 @@ public: RsGxsDistantSearchResultChange(TurtleRequestId id,const RsGxsGroupId& group_id) : mRequestId(id),mGroupId(group_id){} NotifyType getType() { return TYPE_RECEIVED_DISTANT_SEARCH_RESULTS ; } -private: + TurtleRequestId mRequestId ; RsGxsGroupId mGroupId; }; diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 470ffbaab..a8b6e6344 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -245,6 +245,16 @@ void GxsGroupFrameDialog::updateDisplay(bool complete) updateMessageSummaryList(msgIt->first); } } + + updateSearchResults() ; +} + +void GxsGroupFrameDialog::updateSearchResults() +{ + const std::set& reqs = getSearchResults(); + + for(auto it(reqs.begin());it!=reqs.end();++it) + std::cerr << "updating search ID " << std::hex << *it << std::dec << std::endl; } void GxsGroupFrameDialog::todo() diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 3e5091857..f8db16a10 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -152,6 +152,7 @@ private: void initUi(); void updateMessageSummaryList(RsGxsGroupId groupId); + void updateSearchResults(); void openGroupInNewTab(const RsGxsGroupId &groupId); void groupSubscribe(bool subscribe); diff --git a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.cpp b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.cpp index ffbd6147e..76274b27a 100644 --- a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.cpp +++ b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.cpp @@ -16,12 +16,19 @@ RsGxsUpdateBroadcastBase::RsGxsUpdateBroadcastBase(RsGxsIfaceHelper *ifaceImpl, connect(mUpdateBroadcast, SIGNAL(changed()), this, SLOT(updateBroadcastChanged())); connect(mUpdateBroadcast, SIGNAL(grpsChanged(std::list, std::list)), this, SLOT(updateBroadcastGrpsChanged(std::list,std::list))); connect(mUpdateBroadcast, SIGNAL(msgsChanged(std::map >, std::map >)), this, SLOT(updateBroadcastMsgsChanged(std::map >,std::map >))); + connect(mUpdateBroadcast, SIGNAL(distantSearchResultsChanged(const std::list&)), this, SLOT(updateBroadcastDistantSearchResultsChanged(const std::list&))); } RsGxsUpdateBroadcastBase::~RsGxsUpdateBroadcastBase() { } +void RsGxsUpdateBroadcastBase::updateBroadcastDistantSearchResultsChanged(const std::list& ids) +{ + for(auto it(ids.begin());it!=ids.end();++it) + mTurtleResults.insert(*it); +} + void RsGxsUpdateBroadcastBase::fillComplete() { mFillComplete = true; @@ -37,6 +44,9 @@ void RsGxsUpdateBroadcastBase::securedUpdateDisplay() return; } + // This is *bad* because if the connection is done asynchronously the client will call mGrpIds, mGrpIdsMeta, etc without the guarranty that the + // the structed havnt' been cleared in the mean time. + emit fillDisplay(mFillComplete); mFillComplete = false; @@ -75,7 +85,7 @@ void RsGxsUpdateBroadcastBase::updateBroadcastChanged() // The question to whether we should re=load when mGrpIds is not empty is still open. It's not harmful anyway. // This should probably be decided by the service itself. - if (!mGrpIds.empty() || !mGrpIdsMeta.empty() /*|| !mMsgIds.empty()*/ || !mMsgIdsMeta.empty()) + if (!mGrpIds.empty() || !mGrpIdsMeta.empty() /*|| !mMsgIds.empty()*/ || !mMsgIdsMeta.empty() || !mTurtleResults.empty()) mFillComplete = true ; securedUpdateDisplay(); diff --git a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.h b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.h index 90dd3f507..704ff2157 100644 --- a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.h +++ b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastBase.h @@ -7,6 +7,8 @@ class QShowEvent; class RsGxsIfaceHelper; class RsGxsUpdateBroadcast; +typedef uint32_t TurtleRequestId ; + class RsGxsUpdateBroadcastBase : public QObject { friend class RsGxsUpdateBroadcastPage; @@ -25,6 +27,7 @@ public: const std::map > &getMsgIds() { return mMsgIds; } const std::map > &getMsgIdsMeta() { return mMsgIdsMeta; } void getAllMsgIds(std::map > &msgIds); + const std::set& getSearchResults() { return mTurtleResults ; } protected: void fillComplete(); @@ -39,6 +42,7 @@ private slots: void updateBroadcastChanged(); void updateBroadcastGrpsChanged(const std::list& grpIds, const std::list &grpIdsMeta); void updateBroadcastMsgsChanged(const std::map >& msgIds, const std::map >& msgIdsMeta); + void updateBroadcastDistantSearchResultsChanged(const std::list& ids); void securedUpdateDisplay(); private: @@ -49,4 +53,5 @@ private: std::set mGrpIdsMeta; std::map > mMsgIds; std::map > mMsgIdsMeta; + std::set mTurtleResults; }; diff --git a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.cpp b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.cpp index 629148688..c0d19c56f 100644 --- a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.cpp +++ b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.cpp @@ -22,6 +22,11 @@ void RsGxsUpdateBroadcastPage::setUpdateWhenInvisible(bool update) mBase->setUpdateWhenInvisible(update); } +const std::set& RsGxsUpdateBroadcastPage::getSearchResults() +{ + return mBase->getSearchResults(); +} + const std::set &RsGxsUpdateBroadcastPage::getGrpIdsMeta() { return mBase->getGrpIdsMeta(); diff --git a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.h b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.h index 0c67f58ff..e3e9a4563 100644 --- a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.h +++ b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastPage.h @@ -13,6 +13,7 @@ class RsGxsIfaceHelper; class RsGxsUpdateBroadcastBase; +typedef uint32_t TurtleRequestId ; class RsGxsUpdateBroadcastPage : public MainPage { @@ -30,6 +31,7 @@ public: const std::map > &getMsgIds(); const std::map > &getMsgIdsMeta(); void getAllMsgIds(std::map > &msgIds); + const std::set& getSearchResults(); protected: virtual void showEvent(QShowEvent *event); diff --git a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.cpp b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.cpp index a6e3e5705..bade31287 100644 --- a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.cpp +++ b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.cpp @@ -29,6 +29,10 @@ const std::set &RsGxsUpdateBroadcastWidget::getGrpIds() return mBase->getGrpIds(); } +const std::set& RsGxsUpdateBroadcastWidget::getSearchResults() +{ + return mBase->getSearchResults(); +} const std::set &RsGxsUpdateBroadcastWidget::getGrpIdsMeta() { return mBase->getGrpIdsMeta(); diff --git a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.h b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.h index f0ab07742..f99a34d04 100644 --- a/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.h +++ b/retroshare-gui/src/gui/gxs/RsGxsUpdateBroadcastWidget.h @@ -13,6 +13,7 @@ class RsGxsIfaceHelper; class RsGxsUpdateBroadcastBase; +typedef uint32_t TurtleRequestId; class RsGxsUpdateBroadcastWidget : public QWidget { @@ -30,6 +31,7 @@ public: const std::map > &getMsgIds(); const std::map > &getMsgIdsMeta(); void getAllMsgIds(std::map > &msgIds); + const std::set& getSearchResults() ; RsGxsIfaceHelper *interfaceHelper() { return mInterfaceHelper; } diff --git a/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp b/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp index 0fdf63d65..6b6295e2e 100644 --- a/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp +++ b/retroshare-gui/src/util/RsGxsUpdateBroadcast.cpp @@ -79,5 +79,8 @@ void RsGxsUpdateBroadcast::onChangesReceived(const RsGxsChanges& changes) emit grpsChanged(changes.mGrps, changes.mGrpsMeta); } + if(!changes.mDistantSearchReqs.empty()) + emit distantSearchResultsChanged(changes.mDistantSearchReqs) ; + emit changed(); } diff --git a/retroshare-gui/src/util/RsGxsUpdateBroadcast.h b/retroshare-gui/src/util/RsGxsUpdateBroadcast.h index 1622ca606..e08a866c7 100644 --- a/retroshare-gui/src/util/RsGxsUpdateBroadcast.h +++ b/retroshare-gui/src/util/RsGxsUpdateBroadcast.h @@ -8,6 +8,8 @@ class RsGxsIfaceHelper; class RsGxsChanges; +typedef uint32_t TurtleRequestId ; + class RsGxsUpdateBroadcast : public QObject { Q_OBJECT @@ -21,6 +23,7 @@ signals: void changed(); void msgsChanged(const std::map >& msgIds, const std::map >& msgIdsMeta); void grpsChanged(const std::list& grpIds, const std::list& grpIdsMeta); + void distantSearchResultsChanged(const std::list& reqs); private slots: void onChangesReceived(const RsGxsChanges& changes); From e351d7257ebf1489ef491589617c2df71ce3df23 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 24 Jun 2018 16:55:38 +0200 Subject: [PATCH 52/84] added retrieval of search results in UI --- libretroshare/src/gxs/rsgenexchange.cc | 9 ----- libretroshare/src/gxs/rsgenexchange.h | 11 +++--- libretroshare/src/gxs/rsgxsnetservice.cc | 36 ++++++++++--------- libretroshare/src/gxs/rsgxsnetservice.h | 12 ++----- libretroshare/src/gxs/rsnxs.h | 30 ++++++---------- libretroshare/src/retroshare/rsgxschannels.h | 4 +-- libretroshare/src/retroshare/rsgxsiface.h | 20 +++++++++++ libretroshare/src/services/p3gxschannels.cc | 10 ++++++ libretroshare/src/services/p3gxschannels.h | 2 ++ .../src/gui/gxs/GxsGroupFrameDialog.cpp | 11 ++++++ .../src/gui/gxs/GxsGroupFrameDialog.h | 1 + .../src/gui/gxschannels/GxsChannelDialog.cpp | 5 +++ .../src/gui/gxschannels/GxsChannelDialog.h | 1 + 13 files changed, 87 insertions(+), 65 deletions(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index 43e515732..8a199331c 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -1844,15 +1844,6 @@ uint32_t RsGenExchange::getSyncPeriod(const RsGxsGroupId& grpId) return RS_GXS_DEFAULT_MSG_REQ_PERIOD; } -bool RsGenExchange::getDistantSearchResults(const TurtleRequestId& id,std::list& group_infos) -{ - return (mNetService!=NULL) && mNetService->getDistantSearchResults(id,group_infos) ; -} -bool RsGenExchange::clearDistantSearchResults(const TurtleRequestId& id) -{ - return (mNetService!=NULL) && mNetService->clearDistantSearchResults(id) ; -} - bool RsGenExchange::getGroupNetworkStats(const RsGxsGroupId& grpId,RsGroupNetworkStats& stats) { return (!mNetService) || mNetService->getGroupNetworkStats(grpId,stats) ; diff --git a/libretroshare/src/gxs/rsgenexchange.h b/libretroshare/src/gxs/rsgenexchange.h index e08ba80bd..805ab5c26 100644 --- a/libretroshare/src/gxs/rsgenexchange.h +++ b/libretroshare/src/gxs/rsgenexchange.h @@ -121,9 +121,9 @@ public: virtual ~RsGenExchange(); - // Convention that this is implemented here. + // Convention that this is implemented here. // and passes to network service. - virtual RsServiceInfo getServiceInfo() = 0; + virtual RsServiceInfo getServiceInfo() = 0; void setNetworkExchangeService(RsNetworkExchangeService *ns) ; @@ -581,7 +581,7 @@ public: * @param msgs */ void deleteMsgs(uint32_t& token, const GxsMsgReq& msgs); - + protected: /*! * This represents the group before its signature is calculated @@ -665,7 +665,7 @@ public: */ void shareGroupPublishKey(const RsGxsGroupId& grpId,const std::set& peers) ; - + /*! * Returns the local TS of the group as known by the network service. * This is useful to allow various network services to sync their update TS @@ -688,9 +688,6 @@ public: virtual void setSyncPeriod(const RsGxsGroupId& grpId,uint32_t age_in_secs) ; virtual bool getGroupNetworkStats(const RsGxsGroupId& grpId,RsGroupNetworkStats& stats); - virtual bool getDistantSearchResults(const TurtleRequestId& id,std::list& group_infos) ; - virtual bool clearDistantSearchResults(const TurtleRequestId& id); - uint16_t serviceType() const { return mServType ; } uint32_t serviceFullType() const { return ((uint32_t)mServType << 8) + (((uint32_t) RS_PKT_VERSION_SERVICE) << 24); } diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 2ac77b279..480e9dc2e 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5117,6 +5117,25 @@ static bool termSearch(const std::string& src, const std::string& substring) /* always ignore case */ return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() ); } + +bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map& group_infos) +{ + RS_STACK_MUTEX(mNxsMutex) ; + + auto it = mDistantSearchResults.find(req) ; + + if(it == mDistantSearchResults.end()) + return false ; + + group_infos = it->second; + return true ; +} +bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id) +{ + RS_STACK_MUTEX(mNxsMutex) ; + mDistantSearchResults.erase(id); + return true ; +} void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std::list& group_infos) { RS_STACK_MUTEX(mNxsMutex) ; @@ -5179,21 +5198,4 @@ bool RsGxsNetService::search(const std::string& substring,std::list& group_infos) -{ - RS_STACK_MUTEX(mNxsMutex) ; - auto it = mDistantSearchResults.find(id) ; - if(it == mDistantSearchResults.end()) - return false ; - - for(auto it2(it->second.begin());it2!=it->second.end();++it2) - group_infos.push_back(it2->second); - return true; -} -bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id) -{ - RS_STACK_MUTEX(mNxsMutex) ; - mDistantSearchResults.erase(id); - return true ; -} diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 799a2b503..05031feb3 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -132,6 +132,8 @@ public: virtual bool search(const std::string& substring,std::list& group_infos) ; virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos); + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos); + virtual bool clearDistantSearchResults(const TurtleRequestId& id); /*! * pauses synchronisation of subscribed groups and request for group id @@ -170,16 +172,6 @@ public: */ virtual bool getGroupNetworkStats(const RsGxsGroupId& id,RsGroupNetworkStats& stats) ; - /*! - * \brief getDistantSearchResults - * \param id Id of the search request previously issued - * \param group_infos Groups currently known for this search request. - * \return - * false if the id does not correspond to an ongoing distant search request. - */ - virtual bool getDistantSearchResults(const TurtleRequestId& id,std::list& group_infos) ; - virtual bool clearDistantSearchResults(const TurtleRequestId& id); - /*! * Used to inform the net service that we changed subscription status. That helps * optimising data transfer when e.g. unsubsribed groups are updated less often, etc diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 3f1cc9e25..b483e5748 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -60,25 +60,6 @@ * - the also group matrix settings which is by default everyone can transfer to each other */ -/*! - * \brief The RsGxsGroupSymmary struct - * This structure is used to transport group summary information when a GXS service is searched. It contains the group information - * as well as a context string to tell where the information was found. It is more compact than a GroupMeta object, so as to make - * search responses as light as possible. - */ -struct RsGxsGroupSummary -{ - RsGxsGroupId group_id ; - - std::string group_name ; - std::string group_description ; - std::string search_context ; - RsGxsId author_id ; - time_t publish_ts ; - uint32_t number_of_messages ; - time_t last_message_ts ; -}; - class RsNetworkExchangeService { public: @@ -132,13 +113,22 @@ public: * \param group_infos Group summary information for the groups returned by the search */ virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos)=0; + + /*! + * \brief retrieveTurtleSearchResults + * To be used to retrieve the search results that have been notified (or not) + * \param req request that match the results to retrieve + * \param group_infos results to retrieve. + * \return + * false when the request is unknown. + */ + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos)=0; /*! * \brief getDistantSearchResults * \param id * \param group_infos * \return */ - virtual bool getDistantSearchResults(const TurtleRequestId& id,std::list& group_infos)=0 ; virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0; virtual bool search(const std::string& substring,std::list& group_infos) =0; diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index 6bad3ba21..af08f4bb4 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -35,8 +35,6 @@ #include "retroshare/rsgxscommon.h" #include "retroshare/rsturtle.h" - - /* The Main Interface Class - for information about your Peers */ class RsGxsChannels; extern RsGxsChannels *rsGxsChannels; @@ -99,6 +97,8 @@ virtual bool getPostData(const uint32_t &token, std::vector &p virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) =0; + virtual bool clearDistantSearchResults(TurtleRequestId req)=0; ////////////////////////////////////////////////////////////////////////////// virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0; diff --git a/libretroshare/src/retroshare/rsgxsiface.h b/libretroshare/src/retroshare/rsgxsiface.h index b9c974f9c..e822d70d4 100644 --- a/libretroshare/src/retroshare/rsgxsiface.h +++ b/libretroshare/src/retroshare/rsgxsiface.h @@ -32,6 +32,26 @@ #include "gxs/rsgxsdata.h" #include "retroshare/rsgxsifacetypes.h" +/*! + * \brief The RsGxsGroupSymmary struct + * This structure is used to transport group summary information when a GXS service is searched. It contains the group information + * as well as a context string to tell where the information was found. It is more compact than a GroupMeta object, so as to make + * search responses as light as possible. + */ +struct RsGxsGroupSummary +{ + RsGxsGroupId group_id ; + + std::string group_name ; + std::string group_description ; + std::string search_context ; + RsGxsId author_id ; + time_t publish_ts ; + uint32_t number_of_messages ; + time_t last_message_ts ; +}; + + /*! * Stores ids of changed gxs groups and messages. It is used to notify the GUI about changes. */ diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index de4bb1fe6..e4f89861e 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -1696,3 +1696,13 @@ TurtleRequestId p3GxsChannels::turtleSearchRequest(const std::string& match_stri return netService()->turtleSearchRequest(match_string) ; } +bool p3GxsChannels::clearDistantSearchResults(TurtleRequestId req) +{ + return netService()->clearDistantSearchResults(req); +} +bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map& results) +{ + return netService()->retrieveDistantSearchResults(req,results); +} + + diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index 961c05fa6..2243e57ee 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -74,6 +74,8 @@ virtual void service_tick(); virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id); virtual TurtleRequestId turtleSearchRequest(const std::string& match_string); + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) ; + virtual bool clearDistantSearchResults(TurtleRequestId req); // Overloaded to cache new groups. virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index a8b6e6344..598bd35fc 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -254,7 +254,18 @@ void GxsGroupFrameDialog::updateSearchResults() const std::set& reqs = getSearchResults(); for(auto it(reqs.begin());it!=reqs.end();++it) + { std::cerr << "updating search ID " << std::hex << *it << std::dec << std::endl; + + std::map group_infos; + + getDistantSearchResults(*it,group_infos) ; + + std::cerr << "retrieved " << std::endl; + + for(auto it2(group_infos.begin());it2!=group_infos.end();++it2) + std::cerr << " " << it2->first << " " << it2->second.group_id << " \"" << it2->second.group_name << "\"" << std::endl; + } } void GxsGroupFrameDialog::todo() diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index f8db16a10..ac04edf59 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -148,6 +148,7 @@ private: virtual void groupTreeCustomActions(RsGxsGroupId /*grpId*/, int /*subscribeFlags*/, QList &/*actions*/) {} virtual RsGxsCommentService *getCommentService() { return NULL; } virtual QWidget *createCommentHeaderWidget(const RsGxsGroupId &/*grpId*/, const RsGxsMessageId &/*msgId*/) { return NULL; } + virtual bool getDistantSearchResults(TurtleRequestId id, std::map& group_infos){ return false ;} void initUi(); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index acd9dae0e..160119ee2 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -341,3 +341,8 @@ TurtleRequestId GxsChannelDialog::distantSearch(const QString& search_string) { return rsGxsChannels->turtleSearchRequest(search_string.toStdString()) ; } + +bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::map& group_infos) +{ + return rsGxsChannels->retrieveDistantSearchResults(id,group_infos); +} diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h index 3864fba13..38b27d805 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h @@ -50,6 +50,7 @@ protected: virtual GroupFrameSettings::Type groupFrameSettingsType() { return GroupFrameSettings::Channel; } virtual QString getHelpString() const ; virtual void groupInfoToGroupItemInfo(const RsGroupMetaData &groupInfo, GroupItemInfo &groupItemInfo, const RsUserdata *userdata); + virtual bool getDistantSearchResults(TurtleRequestId id, std::map& group_infos); virtual TurtleRequestId distantSearch(const QString& search_string) ; private slots: From 08b436e5f47eb1d3e4215cbdc13f60ae1be03c3b Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 24 Jun 2018 23:15:22 +0200 Subject: [PATCH 53/84] added display of searched groups --- .../src/gui/gxs/GxsGroupFrameDialog.cpp | 51 ++++++++++++++++--- .../src/gui/gxs/GxsGroupFrameDialog.h | 3 +- 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 598bd35fc..1f2d2fc00 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -263,8 +263,40 @@ void GxsGroupFrameDialog::updateSearchResults() std::cerr << "retrieved " << std::endl; - for(auto it2(group_infos.begin());it2!=group_infos.end();++it2) - std::cerr << " " << it2->first << " " << it2->second.group_id << " \"" << it2->second.group_name << "\"" << std::endl; + auto it2 = mSearchGroupsItems.find(*it); + + std::set& known_groups(mKnownGroups[*it]) ; + + if(mSearchGroupsItems.end() == it2) + { + std::cerr << "GxsGroupFrameDialog::updateSearchResults(): received result notification for req " << std::hex << *it << std::dec << " but no item present!" << std::endl; + continue ; // we could create the item just as well but since this situation is not supposed to happen, I prefer to make this a failure case. + } + + QList group_items ; + + for(auto it3(group_infos.begin());it3!=group_infos.end();++it3) + if(known_groups.end() == known_groups.find(it3->first)) + { + std::cerr << " adding new group " << it3->first << " " << it3->second.group_id << " \"" << it3->second.group_name << "\"" << std::endl; + + known_groups.insert(it3->first) ; + + GroupItemInfo i ; + i.id = QString(it3->second.group_id.toStdString().c_str()) ; + i.name = QString::fromUtf8(it3->second.group_name.c_str()) ; + i.description = QString::fromUtf8(it3->second.group_description.c_str()) ; + i.popularity = 0; // could be set to the number of hits + i.lastpost = QDateTime::fromTime_t(it3->second.last_message_ts); + i.subscribeFlags = 0; // irrelevant here + i.publishKey = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags) ; + i.adminKey = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags) ; + i.max_visible_posts = it3->second.number_of_messages ; + + group_items.push_back(i); + } + + ui->groupTreeWidget->fillGroupItems(it2->second, group_items); } } @@ -282,21 +314,24 @@ void GxsGroupFrameDialog::removeCurrentSearch() TurtleRequestId search_request_id = action->data().toUInt(); - auto it = mSearchGroups.find(search_request_id) ; + auto it = mSearchGroupsItems.find(search_request_id) ; - if(it == mSearchGroups.end()) + if(it == mSearchGroupsItems.end()) return ; ui->groupTreeWidget->removeSearchItem(it->second) ; - mSearchGroups.erase(it); + mSearchGroupsItems.erase(it); + + mKnownGroups.erase(search_request_id); } void GxsGroupFrameDialog::removeAllSearches() { - for(auto it(mSearchGroups.begin());it!=mSearchGroups.end();++it) + for(auto it(mSearchGroupsItems.begin());it!=mSearchGroupsItems.end();++it) ui->groupTreeWidget->removeSearchItem(it->second) ; - mSearchGroups.clear(); + mSearchGroupsItems.clear(); + mKnownGroups.clear(); } void GxsGroupFrameDialog::groupTreeCustomPopupMenu(QPoint point) { @@ -1158,6 +1193,6 @@ void GxsGroupFrameDialog::searchNetwork(const QString& search_string) if(request_id == 0) return ; - mSearchGroups[request_id] = ui->groupTreeWidget->addSearchItem(tr("Search for")+ " \"" + search_string + "\"",(uint32_t)request_id,QIcon(icon(ICON_SEARCH))); + mSearchGroupsItems[request_id] = ui->groupTreeWidget->addSearchItem(tr("Search for")+ " \"" + search_string + "\"",(uint32_t)request_id,QIcon(icon(ICON_SEARCH))); } diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index ac04edf59..bcfbabe78 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -210,7 +210,8 @@ private: std::list mCachedGroupMetas; - std::map mSearchGroups ; + std::map mSearchGroupsItems ; + std::map > mKnownGroups; }; #endif From a5d1a154a473b0cd1c1550fa31777412ba453ad9 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 25 Jun 2018 23:08:10 +0200 Subject: [PATCH 54/84] remove channel/posted/forum info when no group is selected --- .../src/gui/Posted/PostedListWidget.cpp | 4 +++ .../src/gui/Posted/PostedListWidget.h | 1 + .../src/gui/gxs/GxsMessageFramePostWidget.h | 1 + .../src/gui/gxs/GxsMessageFrameWidget.cpp | 31 +++++++++++-------- .../src/gui/gxs/GxsMessageFrameWidget.h | 1 + .../gui/gxschannels/GxsChannelPostsWidget.cpp | 14 +++++++++ .../gui/gxschannels/GxsChannelPostsWidget.h | 1 + .../gui/gxsforums/GxsForumThreadWidget.cpp | 22 +++++++++++++ .../src/gui/gxsforums/GxsForumThreadWidget.h | 1 + 9 files changed, 63 insertions(+), 13 deletions(-) diff --git a/retroshare-gui/src/gui/Posted/PostedListWidget.cpp b/retroshare-gui/src/gui/Posted/PostedListWidget.cpp index d321933eb..f7e47863f 100644 --- a/retroshare-gui/src/gui/Posted/PostedListWidget.cpp +++ b/retroshare-gui/src/gui/Posted/PostedListWidget.cpp @@ -454,6 +454,10 @@ void PostedListWidget::applyRanking() ui->scrollAreaWidgetContents->update(); } +void PostedListWidget::blank() +{ + clearPosts(); +} void PostedListWidget::clearPosts() { /* clear all messages */ diff --git a/retroshare-gui/src/gui/Posted/PostedListWidget.h b/retroshare-gui/src/gui/Posted/PostedListWidget.h index 89f95e995..b9102cabb 100644 --- a/retroshare-gui/src/gui/Posted/PostedListWidget.h +++ b/retroshare-gui/src/gui/Posted/PostedListWidget.h @@ -63,6 +63,7 @@ protected: virtual void insertAllPosts(const uint32_t &token, GxsMessageFramePostThread *thread); virtual void insertPosts(const uint32_t &token); virtual void clearPosts(); + virtual void blank(); virtual bool navigatePostItem(const RsGxsMessageId& msgId); /* GxsMessageFrameWidget */ diff --git a/retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.h b/retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.h index 2a42cb772..bebdae7cc 100644 --- a/retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.h +++ b/retroshare-gui/src/gui/gxs/GxsMessageFramePostWidget.h @@ -57,6 +57,7 @@ protected: virtual void groupNameChanged(const QString &/*name*/) {} virtual void clearPosts() = 0; + virtual void blank() = 0; virtual bool navigatePostItem(const RsGxsMessageId& msgId) = 0; /* Thread functions */ diff --git a/retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.cpp b/retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.cpp index e3b4f3cfe..18db60fa2 100644 --- a/retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.cpp +++ b/retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.cpp @@ -72,21 +72,26 @@ bool GxsMessageFrameWidget::isWaiting() void GxsMessageFrameWidget::setGroupId(const RsGxsGroupId &groupId) { - if (mGroupId == groupId) { - if (!groupId.isNull()) { - return; + if (mGroupId == groupId && !groupId.isNull()) + return; + + if(!groupId.isNull()) + { + mAcknowledgeReadStatusToken = 0; + if (mStateHelper->isLoading(mTokenTypeAcknowledgeReadStatus)) { + mStateHelper->setLoading(mTokenTypeAcknowledgeReadStatus, false); + + emit waitingChanged(this); } + + mGroupId = groupId; + groupIdChanged(); } - - mAcknowledgeReadStatusToken = 0; - if (mStateHelper->isLoading(mTokenTypeAcknowledgeReadStatus)) { - mStateHelper->setLoading(mTokenTypeAcknowledgeReadStatus, false); - - emit waitingChanged(this); - } - - mGroupId = groupId; - groupIdChanged(); + else + { + mGroupId.clear(); + blank(); // clear the displayed data, because no group is selected. + } } void GxsMessageFrameWidget::setAllMessagesRead(bool read) diff --git a/retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.h b/retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.h index 71d0ae2a1..a99aada3c 100644 --- a/retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.h +++ b/retroshare-gui/src/gui/gxs/GxsMessageFrameWidget.h @@ -43,6 +43,7 @@ public: virtual void groupIdChanged() = 0; virtual QString groupName(bool withUnreadCount) = 0; virtual QIcon groupIcon() = 0; + virtual void blank() =0; virtual bool navigate(const RsGxsMessageId& msgId) = 0; virtual bool isLoading(); virtual bool isWaiting(); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp index b3f5662b4..379b20823 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp @@ -596,6 +596,20 @@ void GxsChannelPostsWidget::clearPosts() ui->fileWidget->clear(); } +void GxsChannelPostsWidget::blank() +{ + mStateHelper->setWidgetEnabled(ui->postButton, false); + mStateHelper->setWidgetEnabled(ui->subscribeToolButton, false); + + clearPosts(); + + groupNameChanged(QString()); + + ui->infoWidget->hide(); + ui->feedWidget->show(); + ui->fileWidget->hide(); +} + bool GxsChannelPostsWidget::navigatePostItem(const RsGxsMessageId &msgId) { FeedItem *feedItem = ui->feedWidget->findGxsFeedItem(groupId(), msgId); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.h b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.h index ddeb474a0..cfd442973 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.h +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.h @@ -73,6 +73,7 @@ protected: virtual bool useThread() { return mUseThread; } virtual void fillThreadCreatePost(const QVariant &post, bool related, int current, int count); virtual bool navigatePostItem(const RsGxsMessageId& msgId); + virtual void blank() ; /* GxsMessageFrameWidget */ virtual void setAllMessagesReadDo(bool read, uint32_t &token); diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp index 7b9ff9183..a621872d4 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.cpp @@ -299,6 +299,28 @@ GxsForumThreadWidget::GxsForumThreadWidget(const RsGxsGroupId &forumId, QWidget } +void GxsForumThreadWidget::blank() +{ + ui->progressBar->hide(); + ui->progressText->hide(); + ui->postText->clear() ; + ui->by_label->setId(RsGxsId()) ; + ui->time_label->clear(); + ui->lineRight->hide(); + ui->lineLeft->hide(); + ui->by_text_label->hide(); + ui->by_label->hide(); + ui->postText->setImageBlockWidget(ui->imageBlockWidget) ; + ui->postText->resetImagesStatus(Settings->getForumLoadEmbeddedImages()); + ui->threadTreeWidget->clear(); + ui->forumName->setText(""); + + mStateHelper->setWidgetEnabled(ui->newthreadButton, false); + mStateHelper->setWidgetEnabled(ui->previousButton, false); + mStateHelper->setWidgetEnabled(ui->nextButton, false); + ui->versions_CB->hide(); +} + GxsForumThreadWidget::~GxsForumThreadWidget() { if (mFillThread) { diff --git a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.h b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.h index 71103507e..cbec31440 100644 --- a/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.h +++ b/retroshare-gui/src/gui/gxsforums/GxsForumThreadWidget.h @@ -58,6 +58,7 @@ public: // Callback for all Loads. virtual void loadRequest(const TokenQueue *queue, const TokenRequest &req); + virtual void blank(); protected: bool eventFilter(QObject *obj, QEvent *ev); From 00dfa0f3c2c2a4130f2a1c2458f54413aad24860 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 26 Jun 2018 22:20:02 +0200 Subject: [PATCH 55/84] added fallback for GXS GroupMessage UI to look into cached distant group data --- libretroshare/src/gxs/rsgxsnetservice.cc | 36 +++++++++++++++++-- libretroshare/src/gxs/rsgxsnetservice.h | 1 + libretroshare/src/gxs/rsgxsnettunnel.cc | 3 ++ libretroshare/src/gxs/rsnxs.h | 1 + libretroshare/src/retroshare/rsgxschannels.h | 1 + libretroshare/src/retroshare/rsgxsiface.h | 4 +++ .../src/retroshare/rsgxsifacetypes.h | 12 +++---- libretroshare/src/services/p3gxschannels.cc | 35 ++++++++++++++++++ libretroshare/src/services/p3gxschannels.h | 1 + .../gui/gxschannels/GxsChannelPostsWidget.cpp | 12 ++++++- 10 files changed, 96 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 480e9dc2e..3c97cf95d 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5130,6 +5130,20 @@ bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map< group_infos = it->second; return true ; } +bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSummary& gs) +{ + for(auto it(mDistantSearchResults.begin());it!=mDistantSearchResults.end();++it) + { + auto it2 = it->second.find(group_id) ; + + if(it2 != it->second.end()) + { + gs = it2->second; + return true ; + } + } + return false ; +} bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id) { RS_STACK_MUTEX(mNxsMutex) ; @@ -5139,7 +5153,6 @@ bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id) void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std::list& group_infos) { RS_STACK_MUTEX(mNxsMutex) ; -#warning We should use some central way to do that. This might be very costly if done often. RsGxsGrpMetaTemporaryMap grpMeta; std::map& search_results_map(mDistantSearchResults[req]) ; @@ -5159,7 +5172,22 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std: // if(grpMeta[(*it).group_id] == NULL) { filtered_results.push_back(*it) ; - search_results_map[(*it).group_id] = *it; + + auto it2 = search_results_map.find((*it).group_id) ; + + if(it2 != search_results_map.end()) + { + // update existing data + + it2->second.popularity++ ; + it2->second.number_of_messages = std::max(it2->second.number_of_messages,(*it).number_of_messages) ; + } + else + { + search_results_map[(*it).group_id] = *it; + search_results_map[(*it).group_id].popularity = 1; // number of results so far + } + mObserver->receiveDistantSearchResults(req,(*it).group_id) ; } } @@ -5184,10 +5212,12 @@ bool RsGxsNetService::search(const std::string& substring,std::listsecond->mGroupName ; s.group_description = it->second->mGroupName ; // to be filled with something better when we use the real search s.search_context = it->second->mGroupName ; - s.author_id = it->second->mAuthorId; + s.sign_flags = it->second->mSignFlags; s.publish_ts = it->second->mPublishTs; + s.author_id = it->second->mAuthorId; s.number_of_messages = stats.mMaxVisibleCount ; s.last_message_ts = stats.mLastGroupModificationTS ; + s.popularity = it->second->mPop; group_infos.push_back(s) ; } diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 05031feb3..d6496e795 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -134,6 +134,7 @@ public: virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos); virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos); virtual bool clearDistantSearchResults(const TurtleRequestId& id); + virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&); /*! * pauses synchronisation of subscribed groups and request for group id diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index eabc32e3a..cce855165 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -203,8 +203,11 @@ void RsTypeSerializer::serial_process( RsGenericSerializer::SerializeJob j, RsGe RsTypeSerializer::serial_process(j,ctx,gs.publish_ts ,member_name+"-publish_ts") ; // time_t publish_ts ; RsTypeSerializer::serial_process(j,ctx,gs.number_of_messages,member_name+"-number_of_messages") ; // uint32_t number_of_messages ; RsTypeSerializer::serial_process(j,ctx,gs.last_message_ts,member_name+"-last_message_ts") ; // time_t last_message_ts ; + RsTypeSerializer::serial_process(j,ctx,gs.sign_flags,member_name+"-sign_flags") ; // uint32_t sign_flags ; + RsTypeSerializer::serial_process(j,ctx,gs.popularity,member_name+"-popularity") ; // uint32_t popularity ; } + //===========================================================================================================================================// // Interface with rest of the software // //===========================================================================================================================================// diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index b483e5748..046b2e80f 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -130,6 +130,7 @@ public: * \return */ virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0; + virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&)=0; virtual bool search(const std::string& substring,std::list& group_infos) =0; diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index af08f4bb4..a23275b3b 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -99,6 +99,7 @@ virtual bool getPostData(const uint32_t &token, std::vector &p virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) =0; virtual bool clearDistantSearchResults(TurtleRequestId req)=0; + virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0; ////////////////////////////////////////////////////////////////////////////// virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0; diff --git a/libretroshare/src/retroshare/rsgxsiface.h b/libretroshare/src/retroshare/rsgxsiface.h index e822d70d4..bdf65c115 100644 --- a/libretroshare/src/retroshare/rsgxsiface.h +++ b/libretroshare/src/retroshare/rsgxsiface.h @@ -40,6 +40,8 @@ */ struct RsGxsGroupSummary { + RsGxsGroupSummary() : publish_ts(0), number_of_messages(0),last_message_ts(0),sign_flags(0),popularity(0) {} + RsGxsGroupId group_id ; std::string group_name ; @@ -49,6 +51,8 @@ struct RsGxsGroupSummary time_t publish_ts ; uint32_t number_of_messages ; time_t last_message_ts ; + uint32_t sign_flags ; + uint32_t popularity ; }; diff --git a/libretroshare/src/retroshare/rsgxsifacetypes.h b/libretroshare/src/retroshare/rsgxsifacetypes.h index 071b6bd46..0b0bb3322 100644 --- a/libretroshare/src/retroshare/rsgxsifacetypes.h +++ b/libretroshare/src/retroshare/rsgxsifacetypes.h @@ -69,18 +69,18 @@ struct RsGroupMetaData : RsSerializable RsGxsGroupId mGroupId; std::string mGroupName; - uint32_t mGroupFlags; // Combination of FLAG_PRIVACY_PRIVATE | FLAG_PRIVACY_RESTRICTED | FLAG_PRIVACY_PUBLIC - uint32_t mSignFlags; // Combination of RSGXS_GROUP_SIGN_PUBLISH_MASK & RSGXS_GROUP_SIGN_AUTHOR_MASK. + uint32_t mGroupFlags; // Combination of FLAG_PRIVACY_PRIVATE | FLAG_PRIVACY_RESTRICTED | FLAG_PRIVACY_PUBLIC: diffusion + uint32_t mSignFlags; // Combination of RSGXS_GROUP_SIGN_PUBLISH_MASK & RSGXS_GROUP_SIGN_AUTHOR_MASK, i.e. what signatures are required for parent and child msgs time_t mPublishTs; // Mandatory. - RsGxsId mAuthorId; // Optional. + RsGxsId mAuthorId; // Author of the group. Left to "000....0" if anonymous // for circles - RsGxsCircleId mCircleId; - uint32_t mCircleType; + RsGxsCircleId mCircleId; // Id of the circle to which the group is restricted + uint32_t mCircleType; // combination of CIRCLE_TYPE_{ PUBLIC,EXTERNAL,YOUR_FRIENDS_ONLY,LOCAL,EXT_SELF,YOUR_EYES_ONLY } // other stuff. - uint32_t mAuthenFlags; + uint32_t mAuthenFlags; // Actually not used yet. RsGxsGroupId mParentGrpId; // BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG. diff --git a/libretroshare/src/services/p3gxschannels.cc b/libretroshare/src/services/p3gxschannels.cc index e4f89861e..ee4c89f2f 100644 --- a/libretroshare/src/services/p3gxschannels.cc +++ b/libretroshare/src/services/p3gxschannels.cc @@ -1705,4 +1705,39 @@ bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::mapretrieveDistantSearchResults(req,results); } +bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group) +{ + RsGxsGroupSummary gs ; + + if(netService()->retrieveDistantGroupSummary(group_id,gs)) + { + // This is a placeholder information by the time we receive the full group meta data. + + distant_group.mDescription = gs.group_description; + + distant_group.mMeta.mGroupId = gs.group_id ; + distant_group.mMeta.mGroupName = gs.group_name; + distant_group.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC ; + distant_group.mMeta.mSignFlags = gs.sign_flags; + + distant_group.mMeta.mPublishTs = gs.publish_ts; + distant_group.mMeta.mAuthorId = gs.author_id; + + distant_group.mMeta.mCircleType = GXS_CIRCLE_TYPE_PUBLIC ;// guessed, otherwise the group would not be search-able. + + // other stuff. + distant_group.mMeta.mAuthenFlags = 0; // wild guess... + + distant_group.mMeta.mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED ; + + distant_group.mMeta.mPop = gs.popularity; // Popularity = number of friend subscribers + distant_group.mMeta.mVisibleMsgCount = gs.number_of_messages; // Max messages reported by friends + distant_group.mMeta.mLastPost = gs.last_message_ts; // Timestamp for last message. Not used yet. + + return true ; + } + else + return false ; +} + diff --git a/libretroshare/src/services/p3gxschannels.h b/libretroshare/src/services/p3gxschannels.h index 2243e57ee..7a5e19423 100644 --- a/libretroshare/src/services/p3gxschannels.h +++ b/libretroshare/src/services/p3gxschannels.h @@ -76,6 +76,7 @@ virtual void service_tick(); virtual TurtleRequestId turtleSearchRequest(const std::string& match_string); virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) ; virtual bool clearDistantSearchResults(TurtleRequestId req); + virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group); // Overloaded to cache new groups. virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp index 379b20823..d238c7007 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelPostsWidget.cpp @@ -657,12 +657,22 @@ bool GxsChannelPostsWidget::insertGroupData(const uint32_t &token, RsGroupMetaDa std::vector groups; rsGxsChannels->getGroupData(token, groups); - if (groups.size() == 1) + if(groups.size() == 1) { insertChannelDetails(groups[0]); metaData = groups[0].mMeta; return true; } + else + { + RsGxsChannelGroup distant_group; + if(rsGxsChannels->retrieveDistantGroup(groupId(),distant_group)) + { + insertChannelDetails(distant_group); + metaData = distant_group.mMeta; + return true ; + } + } return false; } From a6edf47e00cbb13c50cf25db7c3797900f52d963 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 26 Jun 2018 22:25:06 +0200 Subject: [PATCH 56/84] removed debug/experimental channel search from files tab --- retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp | 9 --------- retroshare-gui/src/gui/FileTransfer/SearchDialog.ui | 9 +-------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp index 6e2bc4223..be61a1dcc 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.cpp @@ -38,7 +38,6 @@ #include "gui/common/RSTreeWidgetItem.h" #include "util/QtVersion.h" -#include "retroshare/rsgxschannels.h" #include #include #include @@ -204,7 +203,6 @@ SearchDialog::SearchDialog(QWidget *parent) // load settings processSettings(true); - ui._channels_CB->setMinimumWidth(20 * f); ui._ownFiles_CB->setMinimumWidth(20*f); ui._friendListsearch_SB->setMinimumWidth(20*f); ui._anonF2Fsearch_CB->setMinimumWidth(20*f); @@ -866,13 +864,6 @@ void SearchDialog::searchKeywords(const QString& keywords) else req_id = rsFiles->turtleSearch(lin_exp) ; } - else if(ui._channels_CB->isChecked()) - { - if(n==1) - req_id = rsGxsChannels->turtleSearchRequest(words.front()) ; - else - QMessageBox::critical(this,"Cannot search multiple words yet.","Search for multiple words is not implemented yet.") ; - } else req_id = RSRandom::random_u32() ; // generate a random 32 bits request id diff --git a/retroshare-gui/src/gui/FileTransfer/SearchDialog.ui b/retroshare-gui/src/gui/FileTransfer/SearchDialog.ui index e6f6ded74..b3df60f05 100644 --- a/retroshare-gui/src/gui/FileTransfer/SearchDialog.ui +++ b/retroshare-gui/src/gui/FileTransfer/SearchDialog.ui @@ -117,13 +117,6 @@ - - - - Channels - - - @@ -448,7 +441,7 @@ LineEditClear QLineEdit -
gui/common/LineEditClear.h
+
gui/common/LineEditClear.h
 SearchTreeWidget From db06c32e806a8c29f3a571e0f9c125a68172a897 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 28 Jun 2018 10:01:57 +0200 Subject: [PATCH 57/84] turned turtle encryption routine into a generic authenticated encryption method in librs::crypto --- libretroshare/src/crypto/rscrypto.cpp | 212 ++++++++++++++++++++++++++ libretroshare/src/crypto/rscrypto.h | 59 +++++++ libretroshare/src/libretroshare.pro | 6 +- libretroshare/src/turtle/p3turtle.cc | 175 ++------------------- 4 files changed, 287 insertions(+), 165 deletions(-) create mode 100644 libretroshare/src/crypto/rscrypto.cpp create mode 100644 libretroshare/src/crypto/rscrypto.h diff --git a/libretroshare/src/crypto/rscrypto.cpp b/libretroshare/src/crypto/rscrypto.cpp new file mode 100644 index 000000000..0a296673e --- /dev/null +++ b/libretroshare/src/crypto/rscrypto.cpp @@ -0,0 +1,212 @@ +/******************************************************************************* + * libretroshare/src/crypto: crypto.cc * + * * + * libretroshare: retroshare core library * + * * + * Copyright 2018 by Cyril Soler * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include +#include +#include + +#include "rscrypto.h" +#include "util/rsrandom.h" + +//#define CRYPTO_DEBUG 1 + +namespace librs { +namespace crypto { + +#define RSCRYPTO_DEBUG() std::cerr << time(NULL) << " : RSCRYPTO : " << __FUNCTION__ << " : " +#define RSCRYPTO_ERROR() std::cerr << "(EE) RSCRYPTO ERROR : " + +static const uint32_t ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE = 12 ; +static const uint32_t ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE = 16 ; +static const uint32_t ENCRYPTED_MEMORY_HEADER_SIZE = 4 ; +static const uint32_t ENCRYPTED_MEMORY_EDATA_SIZE = 4 ; + +static const uint8_t ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305 = 0x01 ; +static const uint8_t ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256 = 0x02 ; + +bool encryptAuthenticateData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,unsigned char *& encrypted_data,uint32_t& encrypted_data_len) +{ + uint8_t initialization_vector[ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE] ; + + RSRandom::random_bytes(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) ; + +#ifdef CRYPTO_DEBUG + RSCRYPTO_DEBUG() << "ftServer::Encrypting ft item." << std::endl; + RSCRYPTO_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) << std::endl; +#endif + + uint32_t item_serialized_size = clear_data_size;//RsGenericSerializer().size(clear_item) ; + uint32_t total_data_size = ENCRYPTED_MEMORY_HEADER_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE + item_serialized_size + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE ; + +#ifdef CRYPTO_DEBUG + RSCRYPTO_DEBUG() << " clear part size : " << size(clear_item) << std::endl; + RSCRYPTO_DEBUG() << " total item size : " << total_data_size << std::endl; +#endif + + encrypted_data = (unsigned char*)rs_malloc( total_data_size ) ; + encrypted_data_len = total_data_size ; + + if(encrypted_data == NULL) + return false ; + + uint8_t *edata = (uint8_t*)encrypted_data; + uint32_t edata_size = item_serialized_size; + uint32_t offset = 0; + + edata[0] = 0xae ; + edata[1] = 0xad ; + edata[2] = ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256 ; // means AEAD_chacha20_sha256 + edata[3] = 0x01 ; + + offset += ENCRYPTED_MEMORY_HEADER_SIZE; + uint32_t aad_offset = offset ; + uint32_t aad_size = ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE ; + + memcpy(&edata[offset], initialization_vector, ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) ; + offset += ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ; + + edata[offset+0] = (edata_size >> 0) & 0xff ; + edata[offset+1] = (edata_size >> 8) & 0xff ; + edata[offset+2] = (edata_size >> 16) & 0xff ; + edata[offset+3] = (edata_size >> 24) & 0xff ; + + offset += ENCRYPTED_MEMORY_EDATA_SIZE ; + + memcpy(&edata[offset],clear_data,clear_data_size); + +#ifdef CRYPTO_DEBUG + RSCRYPTO_DEBUG() << " clear item : " << RsUtil::BinToHex(&edata[offset],std::min(50,(int)total_data_size-(int)offset)) << "(...)" << std::endl; +#endif + + uint32_t clear_item_offset = offset ; + offset += edata_size ; + + uint32_t authentication_tag_offset = offset ; + assert(ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + offset == total_data_size) ; + + //uint8_t encryption_key[32] ; + //deriveEncryptionKey(hash,encryption_key) ; + + if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305) + librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; + else if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256) + librs::crypto::AEAD_chacha20_sha256 (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; + else + return false ; + +#ifdef CRYPTO_DEBUG + RSCRYPTO_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; + RSCRYPTO_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE) << std::endl; + RSCRYPTO_DEBUG() << " final item : " << RsUtil::BinToHex(&edata[0],std::min(50u,total_data_size)) << "(...)" << std::endl; +#endif + + return true ; +} + +// Decrypts the given item using aead-chacha20-poly1305 +bool decryptAuthenticateData(const unsigned char *encrypted_data,uint32_t encrypted_data_len,uint8_t *encryption_master_key, unsigned char *& decrypted_data, uint32_t& decrypted_data_size) +{ + //uint8_t encryption_key[32] ; + //deriveEncryptionKey(hash,encryption_key) ; + + uint8_t *edata = (uint8_t*)encrypted_data; + uint32_t offset = 0; + + if(encrypted_data_len < ENCRYPTED_MEMORY_HEADER_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE) return false ; + + if(edata[0] != 0xae) return false ; + if(edata[1] != 0xad) return false ; + if(edata[2] != ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305 && edata[2] != ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256) return false ; + if(edata[3] != 0x01) return false ; + + offset += ENCRYPTED_MEMORY_HEADER_SIZE ; + uint32_t aad_offset = offset ; + uint32_t aad_size = ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ; + + uint8_t *initialization_vector = &edata[offset] ; + +#ifdef CRYPTO_DEBUG + RSCRYPTO_DEBUG() << "ftServer::decrypting ft item." << std::endl; + RSCRYPTO_DEBUG() << " item data : " << RsUtil::BinToHex(edata,std::min(50u,encrypted_data_len) << "(...)" << std::endl; + RSCRYPTO_DEBUG() << " hash : " << hash << std::endl; + RSCRYPTO_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; + RSCRYPTO_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) << std::endl; +#endif + + offset += ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ; + + uint32_t edata_size = 0 ; + edata_size += ((uint32_t)edata[offset+0]) << 0 ; + edata_size += ((uint32_t)edata[offset+1]) << 8 ; + edata_size += ((uint32_t)edata[offset+2]) << 16 ; + edata_size += ((uint32_t)edata[offset+3]) << 24 ; + + if(edata_size + ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_HEADER_SIZE != encrypted_data_len) + { + RSCRYPTO_ERROR() << " ERROR: encrypted data size is " << edata_size << ", should be " << encrypted_data_len - (ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_HEADER_SIZE ) << std::endl; + return false ; + } + + offset += ENCRYPTED_MEMORY_EDATA_SIZE ; + uint32_t clear_item_offset = offset ; + + uint32_t authentication_tag_offset = offset + edata_size ; +#ifdef CRYPTO_DEBUG + RSCRYPTO_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE) << std::endl; +#endif + + bool result ; + + if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305) + result = librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; + else if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256) + result = librs::crypto::AEAD_chacha20_sha256 (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; + else + return false ; + +#ifdef CRYPTO_DEBUG + RSCRYPTO_DEBUG() << " authen. result : " << result << std::endl; + RSCRYPTO_DEBUG() << " decrypted daya : " << RsUtil::BinToHex(&edata[clear_item_offset],std::min(50u,edata_size)) << "(...)" << std::endl; +#endif + + if(!result) + { + RSCRYPTO_ERROR() << "(EE) decryption/authentication went wrong." << std::endl; + return false ; + } + + decrypted_data_size = edata_size ; + decrypted_data = (unsigned char*)rs_malloc(edata_size) ; + + if(decrypted_data == NULL) + { + std::cerr << "Failed to allocate memory for decrypted data chunk of size " << edata_size << std::endl; + return false ; + } + memcpy(decrypted_data,&edata[clear_item_offset],edata_size) ; + + return true ; +} + +} +} + diff --git a/libretroshare/src/crypto/rscrypto.h b/libretroshare/src/crypto/rscrypto.h new file mode 100644 index 000000000..9666aed28 --- /dev/null +++ b/libretroshare/src/crypto/rscrypto.h @@ -0,0 +1,59 @@ + +/******************************************************************************* + * libretroshare/src/crypto: crypto.h * + * * + * libretroshare: retroshare core library * + * * + * Copyright 2018 by Cyril Soler * + * * + * This program is free software: you can redistribute it and/or modify * + * it under the terms of the GNU Lesser General Public License as * + * published by the Free Software Foundation, either version 3 of the * + * License, or (at your option) any later version. * + * * + * This program is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * GNU Lesser General Public License for more details. * + * * + * You should have received a copy of the GNU Lesser General Public License * + * along with this program. If not, see . * + * * + *******************************************************************************/ + +#include "crypto/chacha20.h" + +namespace librs +{ +namespace crypto +{ +/*! + * \brief encryptAuthenticateData + Encrypts/decrypts data, using a autenticated construction + chacha20, based on a given 32 bytes master key. The actual encryption using a randomized key + based on a 96 bits IV. Input values are not touched (memory is not released). Memory ownership of outputs is left to the client. + * \param clear_data input data to encrypt + * \param clear_data_size length of input data + * \param encryption_master_key encryption master key of length 32 bytes. + * \param encrypted_data encrypted data, allocated using malloc + * \param encrypted_data_len length of encrypted data + * \return + * true if everything went well. + */ +bool encryptAuthenticateData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,unsigned char *& encrypted_data,uint32_t& encrypted_data_size); + +/*! + * \brief decryptAuthenticateData + Encrypts/decrypts data, using a autenticated construction + chacha20, based on a given 32 bytes master key. The actual encryption using a randomized key + based on a 96 bits IV. Input values are not touched (memory is not released). Memory ownership of outputs is left to the client. + * \param encrypted_data input encrypted data + * \param encrypted_data_size input encrypted data length + * \param encryption_master_key encryption master key of length 32 bytes. + * \param decrypted_data decrypted data, allocated using malloc. + * \param decrypted_data_size length of allocated decrypted data. + * \return + * true if decryption + authentication are ok. + */ +bool decryptAuthenticateData(const unsigned char *encrypted_data,uint32_t encrypted_data_size, uint8_t* encryption_master_key, unsigned char *& decrypted_data,uint32_t& decrypted_data_size); +} +} + diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index e34410ed2..ca85b7d83 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -356,7 +356,8 @@ HEADERS += ft/ftchunkmap.h \ ft/ftturtlefiletransferitem.h HEADERS += crypto/chacha20.h \ - crypto/hashstream.h + crypto/hashstream.h \ + crypto/rscrypto.h HEADERS += directory_updater.h \ directory_list.h \ @@ -523,7 +524,8 @@ SOURCES += ft/ftchunkmap.cc \ ft/ftturtlefiletransferitem.cc SOURCES += crypto/chacha20.cpp \ - crypto/hashstream.cc + crypto/hashstream.cc \ + crypto/rscrypto.cpp SOURCES += chat/distantchat.cc \ chat/p3chatservice.cc \ diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 898491cf2..31286de8f 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -34,7 +34,7 @@ #endif #include "rsserver/p3face.h" -#include "crypto/chacha20.h" +#include "crypto/rscrypto.h" #include "pqi/authssl.h" #include "pqi/p3linkmgr.h" @@ -2235,177 +2235,26 @@ std::string p3turtle::getPeerNameForVirtualPeerId(const RsPeerId& virtual_peer_i return name; } -static const uint32_t ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE = 12 ; -static const uint32_t ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE = 16 ; -static const uint32_t ENCRYPTED_TURTLE_HEADER_SIZE = 4 ; -static const uint32_t ENCRYPTED_TURTLE_EDATA_SIZE = 4 ; - -static const uint8_t ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_POLY1305 = 0x01 ; -static const uint8_t ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256 = 0x02 ; - bool p3turtle::encryptData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item) { - uint8_t initialization_vector[ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE] ; - - RSRandom::random_bytes(initialization_vector,ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) ; - -#ifdef SERVER_DEBUG - TURTLE_DEBUG() << "ftServer::Encrypting ft item." << std::endl; - TURTLE_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) << std::endl; -#endif - - uint32_t item_serialized_size = clear_data_size;//RsGenericSerializer().size(clear_item) ; - uint32_t total_data_size = ENCRYPTED_TURTLE_HEADER_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_EDATA_SIZE + item_serialized_size + ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE ; - -#ifdef SERVER_DEBUG - TURTLE_DEBUG() << " clear part size : " << size(clear_item) << std::endl; - TURTLE_DEBUG() << " total item size : " << total_data_size << std::endl; -#endif + unsigned char *encrypted_data = NULL ; + uint32_t encrypted_data_len = 0 ; + if(!librs::crypto::encryptAuthenticateData(clear_data,clear_data_size,encryption_master_key,encrypted_data,encrypted_data_len)) + { + delete encrypted_item ; + return false ; + } encrypted_item = new RsTurtleGenericDataItem ; - encrypted_item->data_bytes = rs_malloc( total_data_size ) ; - encrypted_item->data_size = total_data_size ; - if(encrypted_item->data_bytes == NULL) - return false ; - - uint8_t *edata = (uint8_t*)encrypted_item->data_bytes ; - uint32_t edata_size = item_serialized_size; - uint32_t offset = 0; - - edata[0] = 0xae ; - edata[1] = 0xad ; - edata[2] = ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256 ; // means AEAD_chacha20_sha256 - edata[3] = 0x01 ; - - offset += ENCRYPTED_TURTLE_HEADER_SIZE; - uint32_t aad_offset = offset ; - uint32_t aad_size = ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_EDATA_SIZE ; - - memcpy(&edata[offset], initialization_vector, ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) ; - offset += ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE ; - - edata[offset+0] = (edata_size >> 0) & 0xff ; - edata[offset+1] = (edata_size >> 8) & 0xff ; - edata[offset+2] = (edata_size >> 16) & 0xff ; - edata[offset+3] = (edata_size >> 24) & 0xff ; - - offset += ENCRYPTED_TURTLE_EDATA_SIZE ; - - memcpy(&edata[offset],clear_data,clear_data_size); - -#ifdef SERVER_DEBUG - TURTLE_DEBUG() << " clear item : " << RsUtil::BinToHex(&edata[offset],std::min(50,(int)total_data_size-(int)offset)) << "(...)" << std::endl; -#endif - - uint32_t clear_item_offset = offset ; - offset += edata_size ; - - uint32_t authentication_tag_offset = offset ; - assert(ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE + offset == total_data_size) ; - - //uint8_t encryption_key[32] ; - //deriveEncryptionKey(hash,encryption_key) ; - - if(edata[2] == ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_POLY1305) - librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; - else if(edata[2] == ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256) - librs::crypto::AEAD_chacha20_sha256 (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ; - else - return false ; - -#ifdef SERVER_DEBUG - TURTLE_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; - TURTLE_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE) << std::endl; - TURTLE_DEBUG() << " final item : " << RsUtil::BinToHex(&edata[0],std::min(50u,total_data_size)) << "(...)" << std::endl; -#endif - - return true ; + encrypted_item->data_bytes = encrypted_data ; + encrypted_item->data_size = encrypted_data_len ; + return true; } -// Decrypts the given item using aead-chacha20-poly1305 bool p3turtle::decryptItem(const RsTurtleGenericDataItem* encrypted_item, uint8_t *encryption_master_key, unsigned char *& decrypted_data, uint32_t& decrypted_data_size) { - //uint8_t encryption_key[32] ; - //deriveEncryptionKey(hash,encryption_key) ; - - uint8_t *edata = (uint8_t*)encrypted_item->data_bytes ; - uint32_t offset = 0; - - if(encrypted_item->data_size < ENCRYPTED_TURTLE_HEADER_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_EDATA_SIZE) return false ; - - if(edata[0] != 0xae) return false ; - if(edata[1] != 0xad) return false ; - if(edata[2] != ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_POLY1305 && edata[2] != ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256) return false ; - if(edata[3] != 0x01) return false ; - - offset += ENCRYPTED_TURTLE_HEADER_SIZE ; - uint32_t aad_offset = offset ; - uint32_t aad_size = ENCRYPTED_TURTLE_EDATA_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE ; - - uint8_t *initialization_vector = &edata[offset] ; - -#ifdef SERVER_DEBUG - TURTLE_DEBUG() << "ftServer::decrypting ft item." << std::endl; - TURTLE_DEBUG() << " item data : " << RsUtil::BinToHex(edata,std::min(50u,encrypted_item->data_size)) << "(...)" << std::endl; - TURTLE_DEBUG() << " hash : " << hash << std::endl; - TURTLE_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl; - TURTLE_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE) << std::endl; -#endif - - offset += ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE ; - - uint32_t edata_size = 0 ; - edata_size += ((uint32_t)edata[offset+0]) << 0 ; - edata_size += ((uint32_t)edata[offset+1]) << 8 ; - edata_size += ((uint32_t)edata[offset+2]) << 16 ; - edata_size += ((uint32_t)edata[offset+3]) << 24 ; - - if(edata_size + ENCRYPTED_TURTLE_EDATA_SIZE + ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_HEADER_SIZE != encrypted_item->data_size) - { - TURTLE_ERROR() << " ERROR: encrypted data size is " << edata_size << ", should be " << encrypted_item->data_size - (ENCRYPTED_TURTLE_EDATA_SIZE + ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE + ENCRYPTED_TURTLE_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_TURTLE_HEADER_SIZE ) << std::endl; - return false ; - } - - offset += ENCRYPTED_TURTLE_EDATA_SIZE ; - uint32_t clear_item_offset = offset ; - - uint32_t authentication_tag_offset = offset + edata_size ; -#ifdef SERVER_DEBUG - TURTLE_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_TURTLE_AUTHENTICATION_TAG_SIZE) << std::endl; -#endif - - bool result ; - - if(edata[2] == ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_POLY1305) - result = librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; - else if(edata[2] == ENCRYPTED_TURTLE_FORMAT_AEAD_CHACHA20_SHA256) - result = librs::crypto::AEAD_chacha20_sha256 (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ; - else - return false ; - -#ifdef SERVER_DEBUG - TURTLE_DEBUG() << " authen. result : " << result << std::endl; - TURTLE_DEBUG() << " decrypted daya : " << RsUtil::BinToHex(&edata[clear_item_offset],std::min(50u,edata_size)) << "(...)" << std::endl; -#endif - - if(!result) - { - TURTLE_ERROR() << "(EE) decryption/authentication went wrong." << std::endl; - return false ; - } - - decrypted_data_size = edata_size ; - decrypted_data = (unsigned char*)rs_malloc(edata_size) ; - - if(decrypted_data == NULL) - { - std::cerr << "Failed to allocate memory for decrypted data chunk of size " << edata_size << std::endl; - return false ; - } - memcpy(decrypted_data,&edata[clear_item_offset],edata_size) ; - - return true ; + return librs::crypto::decryptAuthenticateData((unsigned char*)encrypted_item->data_bytes,encrypted_item->data_size,encryption_master_key,decrypted_data,decrypted_data_size); } void p3turtle::getInfo( std::vector >& hashes_info, From 80a43fe3d59163b7f93092a5589bf1ca4f5897b0 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 30 Jun 2018 21:52:25 +0200 Subject: [PATCH 58/84] added secure retrieval of distant groups --- libretroshare/src/gxs/rsgxsdataaccess.cc | 17 +++ libretroshare/src/gxs/rsgxsdataaccess.h | 8 ++ libretroshare/src/gxs/rsgxsnetservice.cc | 134 ++++++++++++++++++++++- libretroshare/src/gxs/rsgxsnetservice.h | 7 ++ libretroshare/src/gxs/rsgxsnettunnel.cc | 38 +++++-- libretroshare/src/gxs/rsnxs.h | 4 + 6 files changed, 199 insertions(+), 9 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsdataaccess.cc b/libretroshare/src/gxs/rsgxsdataaccess.cc index 77c0d05d9..2ba317920 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.cc +++ b/libretroshare/src/gxs/rsgxsdataaccess.cc @@ -1807,6 +1807,23 @@ bool RsGxsDataAccess::updateGroupData(RsNxsGrp* grp) { return mDataStore->updateGroup(grpM); } +bool RsGxsDataAccess::getGroupData(const RsGxsGroupId& grpId, RsNxsGrp *& grp_data) +{ + RsStackMutex stack(mDataMutex); + + std::map grps ; + + grps[grpId] = NULL ; + + if(mDataStore->retrieveNxsGrps(grps, false, true)) // the false here is very important: it removes the private key parts. + { + grp_data = grps.begin()->second; + return true; + } + else + return false ; +} + bool RsGxsDataAccess::addMsgData(RsNxsMsg* msg) { RsStackMutex stack(mDataMutex); diff --git a/libretroshare/src/gxs/rsgxsdataaccess.h b/libretroshare/src/gxs/rsgxsdataaccess.h index 77f94137e..da5da3ca6 100644 --- a/libretroshare/src/gxs/rsgxsdataaccess.h +++ b/libretroshare/src/gxs/rsgxsdataaccess.h @@ -156,6 +156,14 @@ public: */ bool addMsgData(RsNxsMsg* msg); + /*! + * This retrieves a group from the gxs data base, this is a blocking call \n + * @param grp the group to add, memory ownership passed to the callee + * @return false if group cound not be retrieved + */ + bool getGroupData(const RsGxsGroupId& grpId,RsNxsGrp *& grp_data); + + public: diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 3c97cf95d..13d2341be 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -254,6 +254,7 @@ #include "retroshare/rsgxscircles.h" #include "retroshare/rspeers.h" #include "pgp/pgpauxutils.h" +#include "crypto/rscrypto.h" #include "util/rsdir.h" #include "util/rstime.h" #include "util/rsmemory.h" @@ -5105,7 +5106,10 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid) TurtleRequestId RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id) { - return mGxsNetTunnel->turtleGroupRequest(group_id,this) ; + TurtleRequestId req = mGxsNetTunnel->turtleGroupRequest(group_id,this) ; + mSearchRequests[req] = group_id; + + return req; } TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_string) { @@ -5192,6 +5196,58 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std: } } +void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len) +{ +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl; +#endif + auto it = mSearchRequests.find(req); + + if(mSearchRequests.end() == it) + { + std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ; + return; + } + RsGxsGroupId grpId = it->second; + + uint8_t encryption_master_key[32]; + Sha256CheckSum s = RsDirUtil::sha256sum(grpId.toByteArray(),grpId.SIZE_IN_BYTES); + memcpy(encryption_master_key,s.toByteArray(),32); + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___ << " attempting data decryption with master key " << RsUtil::BinToHex(encryption_master_key,32) << std::endl; +#endif + unsigned char *clear_group_data = NULL; + uint32_t clear_group_data_len ; + + if(!librs::crypto::decryptAuthenticateData(encrypted_group_data,encrypted_group_data_len,encryption_master_key,clear_group_data,clear_group_data_len)) + { + std::cerr << "(EE) Could not decrypt data. Something went wrong. Wrong key??" << std::endl; + return ; + } + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___ << " successfuly decrypted data : " << RsUtil::BinToHex(clear_group_data,clear_group_data_len,50) << std::endl; +#endif + RsItem *item = RsNxsSerialiser(mServType).deserialise(clear_group_data,&clear_group_data_len) ; + free(clear_group_data); + clear_group_data = NULL ; + + RsNxsGrp *nxs_grp = dynamic_cast(item) ; + + if(nxs_grp == NULL) + { + std::cerr << "(EE) decrypted item is not a RsNxsGrp. Weird!" << std::endl; + return ; + } + std::vector new_grps(1,nxs_grp); + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___ << " passing the grp data to observer." << std::endl; +#endif + mObserver->receiveNewGroups(new_grps); +} + bool RsGxsNetService::search(const std::string& substring,std::list& group_infos) { RsGxsGrpMetaTemporaryMap grpMetaMap; @@ -5228,4 +5284,80 @@ bool RsGxsNetService::search(const std::string& substring,std::listsecond; + } + else + { + // Now check if the last request was too close in time, in which case, we dont retrieve data. + + if(mLastCacheReloadTS + 60 < time(NULL)) + { + std::cerr << "(WW) Not found in cache, and last cache reload less than 60 secs ago. Returning false. " << std::endl; + return false ; + } + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___ << " reloading group cache information" << std::endl; +#endif + RsNxsGrpDataTemporaryMap grpDataMap; + { + RS_STACK_MUTEX(mNxsMutex) ; + mDataStore->retrieveNxsGrps(grpDataMap, true, true); + } + + for(auto it(grpDataMap.begin());it!=grpDataMap.end();++it) + if(it->second->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) // only cache subscribed groups + { + RsNxsGrp *grp = it->second ; + delete grp->metaData ; // clean private keys + grp->metaData = NULL ; + + Sha1CheckSum hash(RsDirUtil::sha1sum(it->first.toByteArray(),it->first.SIZE_IN_BYTES)); + + mGroupHashCache[hash] = grp ; + + if(hash == hashed_group_id) + grp_data = grp ; + } + } + + if(!grp_data) + { +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___ << " no group found for hash " << hashed_group_id << ": returning false." << std::endl; +#endif + return false ; + } + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___ << " found corresponding group data group id in cache group_id=" << grp_data->grpId << std::endl; +#endif + // Finally, serialize and encrypt the grp data + + uint32_t size = RsNxsSerialiser(mServType).size(grp_data); + RsTemporaryMemory mem(size) ; + + RsNxsSerialiser(mServType).serialise(grp_data,mem,&size) ; + + uint8_t encryption_master_key[32]; + Sha256CheckSum s = RsDirUtil::sha256sum(grp_data->grpId.toByteArray(),grp_data->grpId.SIZE_IN_BYTES); + memcpy(encryption_master_key,s.toByteArray(),32); + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG___ << " sending data encrypted with master key " << RsUtil::BinToHex(encryption_master_key,32) << std::endl; +#endif + return librs::crypto::encryptAuthenticateData(mem,size,encryption_master_key,encrypted_group_data,encrypted_group_data_len); +} diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index d6496e795..9ab18b4d5 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -131,7 +131,10 @@ public: virtual TurtleRequestId turtleSearchRequest(const std::string& match_string); virtual bool search(const std::string& substring,std::list& group_infos) ; + virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len); virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos); + virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len); + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &group_infos); virtual bool clearDistantSearchResults(const TurtleRequestId& id); virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&); @@ -609,6 +612,10 @@ private: uint32_t mDefaultMsgStorePeriod ; uint32_t mDefaultMsgSyncPeriod ; + + std::map mGroupHashCache; + std::map mSearchRequests; + time_t mLastCacheReloadTS ; }; #endif // RSGXSNETSERVICE_H diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index cce855165..b6bf3c15a 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -163,6 +163,27 @@ public: RsTypeSerializer::serial_process(j,ctx,group_infos,"group_infos") ; } }; +class RsGxsNetTunnelTurtleSearchGroupDataItem: public RsGxsNetTunnelItem +{ +public: + explicit RsGxsNetTunnelTurtleSearchGroupDataItem(): RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA) {} + virtual ~RsGxsNetTunnelTurtleSearchGroupDataItem() {} + + uint16_t service ; + unsigned char *encrypted_group_data ; + uint32_t encrypted_group_data_len ; + + virtual void clear() { free(encrypted_group_data);encrypted_group_data=NULL;encrypted_group_data_len=0; } + + virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) + { + RsTypeSerializer::serial_process(j,ctx,service,"service") ; + + RsTypeSerializer::TlvMemBlock_proxy prox(encrypted_group_data,encrypted_group_data_len) ; + RsTypeSerializer::serial_process(j,ctx,prox,"encrypted_group_data") ; + } +}; + class RsGxsNetTunnelSerializer: public RsServiceSerializer { public: @@ -184,7 +205,7 @@ public: case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem; case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem; case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem; - //case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem; + case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_DATA : return new RsGxsNetTunnelTurtleSearchGroupDataItem; default: GXS_NET_TUNNEL_ERROR() << "type ID " << std::hex << (int)item_subtype << std::dec << " is not handled!" << std::endl; return NULL ; @@ -1033,20 +1054,22 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d if(substring_gr != NULL) { -#ifdef TODO - auto it = mSearchableGxsServices.find(substring_sr->service) ; + RS_STACK_MUTEX(mGxsNetTunnelMtx); + auto it = mSearchableServices.find(substring_gr->service) ; - RsNxsGrp *grp = NULL ; + unsigned char *encrypted_group_data = NULL ; + uint32_t encrypted_group_data_len = 0 ; - if(it != mSearchableGxsServices.end() && it->second.search(substring_sr->group_id,grp)) + if(it != mSearchableServices.end() && it->second->search(substring_gr->hashed_group_id,encrypted_group_data,encrypted_group_data_len)) { RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ; search_result_item.service = substring_sr->service ; - search_result_item.group_infos = results ; + search_result_item.encrypted_group_data = encrypted_group_data ; + search_result_item.encrypted_group_data_len = encrypted_group_data_len; search_result_data_size = RsGxsNetTunnelSerializer().size(&search_result_item) ; - search_result_data = (unsigned char*)rs_malloc(size) ; + search_result_data = (unsigned char*)rs_malloc(search_result_data_size) ; if(search_result_data == NULL) return false ; @@ -1055,7 +1078,6 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d return true ; } -#endif } return false ; diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 046b2e80f..7d617ee30 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -133,6 +133,7 @@ public: virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&)=0; virtual bool search(const std::string& substring,std::list& group_infos) =0; + virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0; /*! * Initiates a search through the network @@ -152,6 +153,9 @@ public: */ //virtual int searchGrps(RsGxsSearch* search, uint8_t hops = 1, bool retrieve = 0) = 0; + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + /// DATA ACCESS FUNCTIONS /// + /////////////////////////////////////////////////////////////////////////////////////////////////////////////////// /*! * pauses synchronisation of subscribed groups and request for group id From 4eb060e1545c14293c9169e770a6d0e97c6ec89d Mon Sep 17 00:00:00 2001 From: chelovechishko Date: Mon, 2 Jul 2018 17:41:26 +0900 Subject: [PATCH 59/84] gui: do not attempt to create context menu if idlist is empty ...and other side effects --- .../src/gui/chat/ChatLobbyDialog.cpp | 79 +++++++++---------- 1 file changed, 39 insertions(+), 40 deletions(-) diff --git a/retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp b/retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp index e490af1b8..1f7b20ac6 100644 --- a/retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp +++ b/retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp @@ -2,7 +2,7 @@ * * RetroShare is distributed under the following license: * - * Copyright (C) 2011, csoler + * Copyright (C) 2011, csoler * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -16,7 +16,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. * ****************************************************************/ @@ -87,7 +87,7 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi ui.participantsList->setColumnWidth(COLUMN_ICON, 1.7*S); ui.participantsList->setColumnHidden(COLUMN_ACTIVITY,true); ui.participantsList->setColumnHidden(COLUMN_ID,true); - + /* Set header resize modes and initial section sizes */ QHeaderView * header = ui.participantsList->header(); QHeaderView_setSectionResizeModeColumn(header, COLUMN_NAME, QHeaderView::Stretch); @@ -99,16 +99,16 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi distantChatAct = new QAction(QIcon(":/images/chat_24.png"), tr("Start private chat"), this); sendMessageAct = new QAction(QIcon(":/images/mail_new.png"), tr("Send Message"), this); showInPeopleAct = new QAction(QIcon(), tr("Show author in people tab"), this); - + QActionGroup *sortgrp = new QActionGroup(this); actionSortByName = new QAction(QIcon(), tr("Sort by Name"), this); actionSortByName->setCheckable(true); - actionSortByName->setChecked(true); + actionSortByName->setChecked(true); actionSortByName->setActionGroup(sortgrp); actionSortByActivity = new QAction(QIcon(), tr("Sort by Activity"), this); actionSortByActivity->setCheckable(true); - actionSortByActivity->setChecked(false); + actionSortByActivity->setChecked(false); actionSortByActivity->setActionGroup(sortgrp); @@ -122,7 +122,7 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi connect(actionSortByName, SIGNAL(triggered()), this, SLOT(sortParcipants())); connect(actionSortByActivity, SIGNAL(triggered()), this, SLOT(sortParcipants())); - + /* Add filter actions */ QTreeWidgetItem *headerItem = ui.participantsList->headerItem(); QString headerText = headerItem->text(COLUMN_NAME ); @@ -163,7 +163,7 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi } ownIdChooser = new GxsIdChooser() ; ownIdChooser->loadIds(idChooserFlag, current_id) ; - + QWidgetAction *checkableAction = new QWidgetAction(this); checkableAction->setDefaultWidget(ownIdChooser); @@ -246,8 +246,8 @@ void ChatLobbyDialog::participantsTreeWidgetCustomPopupMenu(QPoint) void ChatLobbyDialog::textBrowserAskContextMenu(QMenu* contextMnu, QString anchorForPosition, const QPoint /*point*/) { - if (anchorForPosition.startsWith(PERSONID)){ - QString strId = anchorForPosition.replace(PERSONID,""); + if (anchorForPosition.startsWith(PERSONID)) { + QString strId = anchorForPosition.replace(PERSONID, ""); if (strId.contains(" ")) strId.truncate(strId.indexOf(" ")); @@ -263,6 +263,8 @@ void ChatLobbyDialog::initParticipantsContextMenu(QMenu *contextMnu, QListaddAction(distantChatAct); contextMnu->addAction(sendMessageAct); @@ -281,7 +283,7 @@ void ChatLobbyDialog::initParticipantsContextMenu(QMenu *contextMnu, QListsetEnabled(false); voteNeutralAct->setEnabled(false); voteNegativeAct->setEnabled(false); - showInPeopleAct->setEnabled(idList.count()==1); + showInPeopleAct->setEnabled(idList.count() == 1); distantChatAct->setData(QVariant::fromValue(idList)); sendMessageAct->setData(QVariant::fromValue(idList)); @@ -291,19 +293,16 @@ void ChatLobbyDialog::initParticipantsContextMenu(QMenu *contextMnu, QListsetData(QVariant::fromValue(idList)); showInPeopleAct->setData(QVariant::fromValue(idList)); - if(idList.count()==1) - { - RsGxsId gxsid = idList.at(0); + RsGxsId gxsid = idList.at(0); - if(!gxsid.isNull() && !rsIdentity->isOwnId(gxsid)) - { - distantChatAct->setEnabled(true); - votePositiveAct->setEnabled(rsReputations->overallReputationLevel(gxsid) != RsReputations::REPUTATION_LOCALLY_POSITIVE); - voteNeutralAct->setEnabled((rsReputations->overallReputationLevel(gxsid) == RsReputations::REPUTATION_LOCALLY_POSITIVE) || (rsReputations->overallReputationLevel(gxsid) == RsReputations::REPUTATION_LOCALLY_NEGATIVE) ); - voteNegativeAct->setEnabled(rsReputations->overallReputationLevel(gxsid) != RsReputations::REPUTATION_LOCALLY_NEGATIVE); - muteAct->setEnabled(true); - muteAct->setChecked(isParticipantMuted(gxsid)); - } + if(!gxsid.isNull() && !rsIdentity->isOwnId(gxsid)) + { + distantChatAct->setEnabled(true); + votePositiveAct->setEnabled(rsReputations->overallReputationLevel(gxsid) != RsReputations::REPUTATION_LOCALLY_POSITIVE); + voteNeutralAct->setEnabled((rsReputations->overallReputationLevel(gxsid) == RsReputations::REPUTATION_LOCALLY_POSITIVE) || (rsReputations->overallReputationLevel(gxsid) == RsReputations::REPUTATION_LOCALLY_NEGATIVE) ); + voteNegativeAct->setEnabled(rsReputations->overallReputationLevel(gxsid) != RsReputations::REPUTATION_LOCALLY_NEGATIVE); + muteAct->setEnabled(true); + muteAct->setChecked(isParticipantMuted(gxsid)); } } @@ -443,17 +442,17 @@ void ChatLobbyDialog::processSettings(bool load) // state of splitter ui.splitter->restoreState(Settings->value("splitter").toByteArray()); - + // load sorting actionSortByActivity->setChecked(Settings->value("sortbyActivity", QVariant(false)).toBool()); actionSortByName->setChecked(Settings->value("sortbyName", QVariant(true)).toBool()); - + } else { // save settings // state of splitter Settings->setValue("splitter", ui.splitter->saveState()); - + //save sorting Settings->setValue("sortbyActivity", actionSortByActivity->isChecked()); Settings->setValue("sortbyName", actionSortByName->isChecked()); @@ -464,7 +463,7 @@ void ChatLobbyDialog::processSettings(bool load) /** * Change your Nickname - * + * * - send a Message to all Members => later: send hidden message to clients, so they can actualize there mutedParticipants list */ void ChatLobbyDialog::setIdentity(const RsGxsId& gxs_id) @@ -500,7 +499,7 @@ void ChatLobbyDialog::changeNickname() /** * We get a new Message from a chat participant - * + * * - Ignore Messages from muted chat participants */ void ChatLobbyDialog::addChatMsg(const ChatMessage& msg) @@ -509,7 +508,7 @@ void ChatLobbyDialog::addChatMsg(const ChatMessage& msg) QDateTime recvTime = QDateTime::fromTime_t(msg.recvTime); QString message = QString::fromUtf8(msg.msg.c_str()); RsGxsId gxs_id = msg.lobby_peer_gxs_id ; - + if(!isParticipantMuted(gxs_id)) { // We could change addChatMsg to display the peers icon, passing a ChatId @@ -551,7 +550,7 @@ void ChatLobbyDialog::addChatMsg(const ChatMessage& msg) /** * Regenerate the QTreeWidget participant list of a Chat Lobby - * + * * Show yellow icon for muted Participants */ void ChatLobbyDialog::updateParticipantsList() @@ -602,7 +601,7 @@ void ChatLobbyDialog::updateParticipantsList() time_t tLastAct=widgetitem->text(COLUMN_ACTIVITY).toInt(); time_t now = time(NULL); - + widgetitem->setSizeHint(COLUMN_ICON, QSize(20,20)); @@ -619,7 +618,7 @@ void ChatLobbyDialog::updateParticipantsList() if (RsGxsId(participant.toStdString()) == gxs_id) widgetitem->setIcon(COLUMN_ICON, bullet_yellow_128); widgetitem->updateBannedState(); - + QTime qtLastAct=QTime(0,0,0).addSecs(now-tLastAct); widgetitem->setToolTip(COLUMN_ICON,tr("Right click to mute/unmute participants
Double click to address this person
") +tr("This participant is not active since:") @@ -635,7 +634,7 @@ void ChatLobbyDialog::updateParticipantsList() /** * Called when a Participant get Clicked / Changed - * + * * Check if the Checkbox altered and Mute User */ void ChatLobbyDialog::changeParticipationState() @@ -787,15 +786,15 @@ bool ChatLobbyDialog::isNicknameInLobby(const RsGxsId& nickname) return clinfo.gxs_ids.find(nickname) != clinfo.gxs_ids.end() ; } -/** +/** * Should Messages from this Nickname be muted? - * + * * At the moment it is not possible to 100% know which peer sendet the message, and only - * the nickname is available. So this couldn't work for 100%. So, for example, if a peer - * change his name to the name of a other peer, we couldn't block him. A real implementation + * the nickname is available. So this couldn't work for 100%. So, for example, if a peer + * change his name to the name of a other peer, we couldn't block him. A real implementation * will be possible if we transfer a temporary Session ID from the sending Retroshare client * version 0.6 - * + * * @param QString nickname to check */ bool ChatLobbyDialog::isParticipantMuted(const RsGxsId& participant) @@ -901,7 +900,7 @@ bool ChatLobbyDialog::canClose() void ChatLobbyDialog::showDialog(uint chatflags) { - if (chatflags & RS_CHAT_FOCUS) + if (chatflags & RS_CHAT_FOCUS) { MainWindow::showWindow(MainWindow::ChatLobby); dynamic_cast(MainWindow::getPage(MainWindow::ChatLobby))->setCurrentChatPage(this) ; @@ -916,7 +915,7 @@ void ChatLobbyDialog::sortParcipants() } else if (actionSortByName->isChecked()) { ui.participantsList->sortItems(COLUMN_NAME, Qt::AscendingOrder); } - + } void ChatLobbyDialog::filterChanged(const QString& /*text*/) From 5925aa06fed41e770630811dd4d5d021882b7ae7 Mon Sep 17 00:00:00 2001 From: chelovechishko Date: Mon, 2 Jul 2018 21:22:22 +0900 Subject: [PATCH 60/84] gui: anchor not only first word of nickname, if any --- .../src/gui/common/RSTextBrowser.cpp | 55 +++++++++++++++---- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/retroshare-gui/src/gui/common/RSTextBrowser.cpp b/retroshare-gui/src/gui/common/RSTextBrowser.cpp index 03fbfe3de..8c59c815c 100644 --- a/retroshare-gui/src/gui/common/RSTextBrowser.cpp +++ b/retroshare-gui/src/gui/common/RSTextBrowser.cpp @@ -251,22 +251,53 @@ bool RSTextBrowser::checkImage(QPoint pos, QString &imageStr) */ QString RSTextBrowser::anchorForPosition(const QPoint &pos) const { + // Many calls when time label shows up QTextCursor cursor = cursorForPosition(pos); cursor.select(QTextCursor::WordUnderCursor); + QString word = cursor.selectedText(); QString anchor = ""; - if (!cursor.selectedText().isEmpty()){ - QRegExp rx(" Date: Wed, 4 Jul 2018 00:19:36 +0900 Subject: [PATCH 61/84] gui: disable quote option if no text is selected in chat --- retroshare-gui/src/gui/chat/ChatWidget.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/retroshare-gui/src/gui/chat/ChatWidget.cpp b/retroshare-gui/src/gui/chat/ChatWidget.cpp index 3ed194789..48e4b63a6 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.cpp +++ b/retroshare-gui/src/gui/chat/ChatWidget.cpp @@ -1096,6 +1096,10 @@ void ChatWidget::contextMenuTextBrowser(QPoint point) contextMnu->addSeparator(); contextMnu->addAction(ui->actionClearChatHistory); contextMnu->addAction(ui->actionQuote); + if (ui->textBrowser->textCursor().selection().toPlainText().length() == 0) + ui->actionQuote->setEnabled(false); + else + ui->actionQuote->setEnabled(true); contextMnu->addAction(ui->actionDropPlacemark); if(ui->textBrowser->checkImage(point)) From 77c3eae976f9733906729b48c7dade17b617c7ca Mon Sep 17 00:00:00 2001 From: chelovechishko Date: Wed, 4 Jul 2018 00:34:45 +0900 Subject: [PATCH 62/84] gui: check for show "is typing" before time call --- retroshare-gui/src/gui/chat/ChatWidget.cpp | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/retroshare-gui/src/gui/chat/ChatWidget.cpp b/retroshare-gui/src/gui/chat/ChatWidget.cpp index 48e4b63a6..674204d16 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.cpp +++ b/retroshare-gui/src/gui/chat/ChatWidget.cpp @@ -110,7 +110,7 @@ ChatWidget::ChatWidget(QWidget *parent) : ui->searchButton->setIconSize(iconSize); ui->sendButton->setFixedHeight(iconHeight); ui->sendButton->setIconSize(iconSize); - + //Initialize search iCharToStartSearch=Settings->getChatSearchCharToStartSearch(); bFindCaseSensitively=Settings->getChatSearchCaseSensitively(); @@ -177,7 +177,7 @@ ChatWidget::ChatWidget(QWidget *parent) : ui->infoFrame->setVisible(false); ui->statusMessageLabel->hide(); - + setAcceptDrops(true); ui->chatTextEdit->setAcceptDrops(false); ui->hashBox->setDropWidget(this); @@ -197,7 +197,7 @@ ChatWidget::ChatWidget(QWidget *parent) : menu->addAction(ui->actionMessageHistory); ui->pushtoolsButton->setMenu(menu); menu->addMenu(fontmenu); - + ui->actionSendAsPlainText->setChecked(Settings->getChatSendAsPlainTextByDef()); ui->chatTextEdit->setOnlyPlainText(ui->actionSendAsPlainText->isChecked()); connect(ui->actionSendAsPlainText, SIGNAL(toggled(bool)), ui->chatTextEdit, SLOT(setOnlyPlainText(bool)) ); @@ -357,12 +357,12 @@ void ChatWidget::init(const ChatId &chat_id, const QString &title) ui->titleBarFrame->setVisible(false); } - if (rsHistory->getEnable(hist_chat_type)) + if (rsHistory->getEnable(hist_chat_type)) { // get chat messages from history std::list historyMsgs; - if (messageCount > 0) + if (messageCount > 0) { rsHistory->getMessages(chatId, historyMsgs, messageCount); @@ -376,7 +376,7 @@ void ChatWidget::init(const ChatId &chat_id, const QString &title) continue; QString name; - if (chatId.isLobbyId() || chatId.isDistantChatId()) + if (chatId.isLobbyId() || chatId.isDistantChatId()) { RsIdentityDetails details; if (rsIdentity->getIdDetails(RsGxsId(historyIt->peerName), details)) @@ -962,7 +962,7 @@ void ChatWidget::addChatMsg(bool incoming, const QString &name, const RsGxsId gx unsigned int formatFlag = 0; bool addDate = false; - if (QDate::currentDate()>lastMsgDate) + if (QDate::currentDate()>lastMsgDate) { addDate=true; } @@ -1165,13 +1165,14 @@ void ChatWidget::resetStatusBar() void ChatWidget::updateStatusTyping() { + if(Settings->getChatDoNotSendIsTyping()) + return; if (time(NULL) - lastStatusSendTime > 5) // limit 'peer is typing' packets to at most every 10 sec { #ifdef ONLY_FOR_LINGUIST tr("is typing..."); #endif - if(!Settings->getChatDoNotSendIsTyping()) - rsMsgs->sendStatusString(chatId, "is typing..."); + rsMsgs->sendStatusString(chatId, "is typing..."); lastStatusSendTime = time(NULL) ; } } @@ -1677,7 +1678,7 @@ void ChatWidget::updateStatus(const QString &peer_id, int status) vpid = chatId.toPeerId(); /* set font size for status */ - if (peer_id.toStdString() == vpid.toStdString()) + if (peer_id.toStdString() == vpid.toStdString()) { // the peers status has changed From b3fb7abf9925634b7f565668f0eeeef32625937a Mon Sep 17 00:00:00 2001 From: chelovechishko Date: Wed, 4 Jul 2018 22:46:01 +0900 Subject: [PATCH 63/84] gui: reduce size of icons in chat for font.height lesser than 26 --- .../src/gui/chat/ChatLobbyDialog.cpp | 24 +++++++++++-------- retroshare-gui/src/gui/chat/ChatWidget.cpp | 20 +++++++++++----- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp b/retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp index 1f7b20ac6..35a6f0f54 100644 --- a/retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp +++ b/retroshare-gui/src/gui/chat/ChatLobbyDialog.cpp @@ -80,7 +80,7 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi connect(ui.filterLineEdit, SIGNAL(textChanged(QString)), this, SLOT(filterChanged(QString))); - int S = QFontMetricsF(font()).height() ; + int S = QFontMetricsF(font()).height() ; ui.participantsList->setIconSize(QSize(1.4*S,1.4*S)); ui.participantsList->setColumnCount(COLUMN_COUNT); @@ -128,23 +128,27 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi QString headerText = headerItem->text(COLUMN_NAME ); ui.filterLineEdit->addFilter(QIcon(), headerText, COLUMN_NAME , QString("%1 %2").arg(tr("Search"), headerText)); + // just empiric values + double scaler_factor = S > 25 ? 2.4 : 1.8; + QSize icon_size(scaler_factor * S, scaler_factor * S); + // Add a button to invite friends. // inviteFriendsButton = new QToolButton ; - inviteFriendsButton->setMinimumSize(QSize(2.4*S,2.4*S)) ; - inviteFriendsButton->setMaximumSize(QSize(2.4*S,2.4*S)) ; + inviteFriendsButton->setMinimumSize(icon_size); + inviteFriendsButton->setMaximumSize(icon_size); inviteFriendsButton->setText(QString()) ; inviteFriendsButton->setAutoRaise(true) ; inviteFriendsButton->setToolTip(tr("Invite friends to this lobby")); - mParticipantCompareRole = new RSTreeWidgetItemCompareRole; - mParticipantCompareRole->setRole(COLUMN_ACTIVITY, ROLE_SORT); + mParticipantCompareRole = new RSTreeWidgetItemCompareRole; + mParticipantCompareRole->setRole(COLUMN_ACTIVITY, ROLE_SORT); { QIcon icon ; icon.addPixmap(QPixmap(":/icons/png/invite.png")) ; inviteFriendsButton->setIcon(icon) ; - inviteFriendsButton->setIconSize(QSize(2.4*S,2.4*S)) ; + inviteFriendsButton->setIconSize(icon_size); } connect(inviteFriendsButton, SIGNAL(clicked()), this , SLOT(inviteFriends())); @@ -175,9 +179,9 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi connect(ownIdChooser,SIGNAL(currentIndexChanged(int)),this,SLOT(changeNickname())) ; - unsubscribeButton = new QToolButton ; - unsubscribeButton->setMinimumSize(QSize(2.4*S,2.4*S)) ; - unsubscribeButton->setMaximumSize(QSize(2.4*S,2.4*S)) ; + unsubscribeButton = new QToolButton; + unsubscribeButton->setMinimumSize(icon_size); + unsubscribeButton->setMaximumSize(icon_size); unsubscribeButton->setText(QString()) ; unsubscribeButton->setAutoRaise(true) ; unsubscribeButton->setToolTip(tr("Leave this chat room (Unsubscribe)")); @@ -186,7 +190,7 @@ ChatLobbyDialog::ChatLobbyDialog(const ChatLobbyId& lid, QWidget *parent, Qt::Wi QIcon icon ; icon.addPixmap(QPixmap(":/icons/png/leave.png")) ; unsubscribeButton->setIcon(icon) ; - unsubscribeButton->setIconSize(QSize(2.4*S,2.4*S)) ; + unsubscribeButton->setIconSize(icon_size); } /* Initialize splitter */ diff --git a/retroshare-gui/src/gui/chat/ChatWidget.cpp b/retroshare-gui/src/gui/chat/ChatWidget.cpp index 674204d16..016d87d49 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.cpp +++ b/retroshare-gui/src/gui/chat/ChatWidget.cpp @@ -65,6 +65,8 @@ #include #define FMM 2.5//fontMetricsMultiplicator +#define FMM_SMALLER 1.8 +#define FMM_THRESHOLD 25 /***** * #define CHAT_DEBUG 1 @@ -75,9 +77,12 @@ ChatWidget::ChatWidget(QWidget *parent) : { ui->setupUi(this); - int iconHeight = FMM*QFontMetricsF(font()).height() ; - QSize iconSize = QSize(iconHeight,iconHeight); - QSize buttonSize = QSize(iconSize + QSize((int)FMM,(int)FMM)); + int iconHeight = QFontMetricsF(font()).height(); + double fmm = iconHeight > FMM_THRESHOLD ? FMM : FMM_SMALLER; + iconHeight *= fmm; + QSize iconSize = QSize(iconHeight, iconHeight); + int butt_size(iconSize.height() + fmm); + QSize buttonSize = QSize(butt_size, butt_size); newMessages = false; typing = false; @@ -259,9 +264,12 @@ void ChatWidget::addChatHorizontalWidget(QWidget *w) void ChatWidget::addChatBarWidget(QWidget *w) { - int iconHeight = FMM*QFontMetricsF(font()).height() ; - QSize iconSize = QSize(iconHeight,iconHeight); - QSize buttonSize = QSize(iconSize + QSize((int)FMM,(int)FMM)); + int iconHeight = QFontMetricsF(font()).height(); + double fmm = iconHeight > FMM_THRESHOLD ? FMM : FMM_SMALLER; + iconHeight *= fmm; + QSize iconSize = QSize(iconHeight, iconHeight); + int butt_size(iconSize.height() + fmm); + QSize buttonSize = QSize(butt_size, butt_size); w->setFixedSize(buttonSize); ui->pluginButtonFrame->layout()->addWidget(w) ; } From 74075fddedc70e6fe3c83be72ca22df82afded69 Mon Sep 17 00:00:00 2001 From: chelovechishko Date: Wed, 4 Jul 2018 23:18:21 +0900 Subject: [PATCH 64/84] gui: do not show the quote section of context menu in chat if no text selected --- retroshare-gui/src/gui/chat/ChatWidget.cpp | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/retroshare-gui/src/gui/chat/ChatWidget.cpp b/retroshare-gui/src/gui/chat/ChatWidget.cpp index 016d87d49..f35c0425d 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.cpp +++ b/retroshare-gui/src/gui/chat/ChatWidget.cpp @@ -1103,11 +1103,8 @@ void ChatWidget::contextMenuTextBrowser(QPoint point) contextMnu->addSeparator(); contextMnu->addAction(ui->actionClearChatHistory); - contextMnu->addAction(ui->actionQuote); - if (ui->textBrowser->textCursor().selection().toPlainText().length() == 0) - ui->actionQuote->setEnabled(false); - else - ui->actionQuote->setEnabled(true); + if (ui->textBrowser->textCursor().selection().toPlainText().length()) + contextMnu->addAction(ui->actionQuote); contextMnu->addAction(ui->actionDropPlacemark); if(ui->textBrowser->checkImage(point)) @@ -1829,13 +1826,10 @@ bool ChatWidget::setStyle() void ChatWidget::quote() { QString text = ui->textBrowser->textCursor().selection().toPlainText(); - if(text.length() > 0) - { - QStringList sl = text.split(QRegExp("[\r\n]"),QString::SkipEmptyParts); - text = sl.join("\n> "); - text.replace(QChar(-4)," ");//Char used when image on text. - emit ui->chatTextEdit->append(QString("> ") + text); - } + QStringList sl = text.split(QRegExp("[\r\n]"), QString::SkipEmptyParts); + text = sl.join("\n> "); + text.replace(QChar(-4), " "); // Char used when image on text. + emit ui->chatTextEdit->append(QString("> ") + text); } void ChatWidget::dropPlacemark() From 8ad454723a37fa7a551195d64f312cad903284ac Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 4 Jul 2018 21:42:09 +0200 Subject: [PATCH 65/84] added group data request when search items are selected --- libretroshare/src/gxs/rsgxsnetservice.cc | 24 ++++++++++++++++++- libretroshare/src/gxs/rsgxsnetservice.h | 9 +++++++ .../src/gui/gxs/GxsGroupFrameDialog.cpp | 3 +++ .../src/gui/gxs/GxsGroupFrameDialog.h | 1 + .../src/gui/gxschannels/GxsChannelDialog.cpp | 11 +++++++++ .../src/gui/gxschannels/GxsChannelDialog.h | 2 ++ 6 files changed, 49 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 13d2341be..e58a7c52e 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -304,6 +304,7 @@ static const uint32_t REJECTED_MESSAGE_RETRY_DELAY = 24*3600; // static const uint32_t GROUP_STATS_UPDATE_DELAY = 240; // update unsubscribed group statistics every 3 mins static const uint32_t GROUP_STATS_UPDATE_NB_PEERS = 2; // number of peers to which the group stats are asked static const uint32_t MAX_ALLOWED_GXS_MESSAGE_SIZE = 199000; // 200,000 bytes including signature and headers +static const uint32_t MIN_DELAY_BETWEEN_GROUP_SEARCH = 40; // dont search same group more than every 40 secs. static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_UNKNOWN = 0x00 ; static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_NO_ERROR = 0x01 ; @@ -380,6 +381,8 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds, mOwnId = mNetMgr->getOwnId(); mUpdateCounter = 0; + mLastCacheReloadTS = 0; + // check the consistency if(mDefaultMsgStorePeriod > 0 && mDefaultMsgSyncPeriod > mDefaultMsgStorePeriod) @@ -5106,7 +5109,25 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid) TurtleRequestId RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id) { + time_t now = time(NULL); + auto it = mSearchedGroups.find(group_id) ; + + if(mSearchedGroups.end() != it && (it->second.ts + MIN_DELAY_BETWEEN_GROUP_SEARCH > now)) + { + std::cerr << "(WW) Last turtle request was " << now - it->second.ts << " secs ago. Not searching again." << std::endl; + return it->second.request_id; + } + +#ifdef NXS_NET_DEBUG_8 + GXSNETDEBUG__G(group_id) << " requesting group id " << group_id << " using turtle" << std::endl; +#endif TurtleRequestId req = mGxsNetTunnel->turtleGroupRequest(group_id,this) ; + + GroupRequestRecord& rec(mSearchedGroups[group_id]) ; + + rec.request_id = req; + rec.ts = now; + mSearchRequests[req] = group_id; return req; @@ -5302,7 +5323,7 @@ bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char * { // Now check if the last request was too close in time, in which case, we dont retrieve data. - if(mLastCacheReloadTS + 60 < time(NULL)) + if(mLastCacheReloadTS + 60 > time(NULL)) { std::cerr << "(WW) Not found in cache, and last cache reload less than 60 secs ago. Returning false. " << std::endl; return false ; @@ -5315,6 +5336,7 @@ bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char * { RS_STACK_MUTEX(mNxsMutex) ; mDataStore->retrieveNxsGrps(grpDataMap, true, true); + mLastCacheReloadTS = time(NULL); } for(auto it(grpDataMap.begin());it!=grpDataMap.end();++it) diff --git a/libretroshare/src/gxs/rsgxsnetservice.h b/libretroshare/src/gxs/rsgxsnetservice.h index 9ab18b4d5..295397ae2 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.h +++ b/libretroshare/src/gxs/rsgxsnetservice.h @@ -59,6 +59,14 @@ class RsGroupNetworkStatsRecord time_t update_TS ; }; +struct GroupRequestRecord +{ + GroupRequestRecord(): ts(0), request_id(0) {} + + time_t ts ; + TurtleRequestId request_id; +}; + /*! * This class implements the RsNetWorkExchangeService * using transactions to handle synchrnisation of Nxs items between @@ -615,6 +623,7 @@ private: std::map mGroupHashCache; std::map mSearchRequests; + std::map mSearchedGroups ; time_t mLastCacheReloadTS ; }; diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 1f2d2fc00..2ac49df42 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -767,6 +767,9 @@ void GxsGroupFrameDialog::changedCurrentGroup(const QString &groupId) return; } + // send a request for the group, if it has been distant-searched. + checkRequestGroup(mGroupId) ; + /* search exisiting tab */ GxsMessageFrameWidget *msgWidget = messageWidget(mGroupId, true); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index bcfbabe78..7908f8b5b 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -95,6 +95,7 @@ protected: virtual RetroShareLink::enumType getLinkType() = 0; virtual GroupFrameSettings::Type groupFrameSettingsType() { return GroupFrameSettings::Nothing; } virtual void groupInfoToGroupItemInfo(const RsGroupMetaData &groupInfo, GroupItemInfo &groupItemInfo, const RsUserdata *userdata); + virtual void checkRequestGroup(const RsGxsGroupId& grpId) {} private slots: void todo(); diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index 160119ee2..f839a0a3f 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -346,3 +346,14 @@ bool GxsChannelDialog::getDistantSearchResults(TurtleRequestId id, std::mapretrieveDistantSearchResults(id,group_infos); } + +void GxsChannelDialog::checkRequestGroup(const RsGxsGroupId& grpId) +{ + RsGxsChannelGroup distant_group; + + if( rsGxsChannels->retrieveDistantGroup(grpId,distant_group)) // normally we should also check that the group meta is not already here. + { + std::cerr << "GxsChannelDialog::checkRequestGroup() sending turtle request for group data for group " << grpId << std::endl; + rsGxsChannels->turtleGroupRequest(grpId); + } +} diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h index 38b27d805..43a81aca5 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.h @@ -53,6 +53,8 @@ protected: virtual bool getDistantSearchResults(TurtleRequestId id, std::map& group_infos); virtual TurtleRequestId distantSearch(const QString& search_string) ; + virtual void checkRequestGroup(const RsGxsGroupId& grpId) ; + private slots: void toggleAutoDownload(); void setDefaultDirectory(); From 2067b106e4b29fd23f6cc0eebcfb2f85c4b76041 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 4 Jul 2018 23:54:26 +0200 Subject: [PATCH 66/84] fixed missing code in passing turtle group data result to client service --- libretroshare/src/gxs/rsgxsnetservice.cc | 1 + libretroshare/src/gxs/rsgxsnettunnel.cc | 62 +++++++++++++------ libretroshare/src/gxs/rsnxs.h | 8 +++ .../src/gui/gxs/GxsGroupFrameDialog.h | 4 +- 4 files changed, 55 insertions(+), 20 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index e58a7c52e..054795873 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5349,6 +5349,7 @@ bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char * Sha1CheckSum hash(RsDirUtil::sha1sum(it->first.toByteArray(),it->first.SIZE_IN_BYTES)); mGroupHashCache[hash] = grp ; + it->second = NULL ; // prevents deletion if(hash == hashed_group_id) grp_data = grp ; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index b6bf3c15a..86bee73e7 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -1064,7 +1064,7 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d { RsGxsNetTunnelTurtleSearchGroupDataItem search_result_item ; - search_result_item.service = substring_sr->service ; + search_result_item.service = substring_gr->service ; search_result_item.encrypted_group_data = encrypted_group_data ; search_result_item.encrypted_group_data_len = encrypted_group_data_len; @@ -1091,26 +1091,52 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id RsGxsNetTunnelTurtleSearchGroupSummaryItem *result_gs = dynamic_cast(item) ; - if(result_gs == NULL) + if(result_gs != NULL) + { + GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; + + for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) + std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl; + + auto it = mSearchableServices.find(result_gs->service) ; + + if(it == mSearchableServices.end()) + { + GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is not in the searchable services list." << std::endl; + delete item; + return ; + } + + it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ; + + delete item; + return ; + } + + RsGxsNetTunnelTurtleSearchGroupDataItem *result_gd = dynamic_cast(item) ; + + if(result_gd != NULL) { - GXS_NET_TUNNEL_ERROR() << ": deserialized item is not a GroupSummary Item. Smething's wrong here." << std::endl; - return ; + GXS_NET_TUNNEL_DEBUG() << " : result is of type group data for service " << result_gd->service << std::dec << ": " << std::endl; + + auto it = mSearchableServices.find(result_gd->service) ; + + if(it == mSearchableServices.end()) + { + GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gd->service << std::dec << " that is not in the searchable services list." << std::endl; + delete item; + return ; + } + + it->second->receiveTurtleSearchResults(request_id,result_gd->encrypted_group_data,result_gd->encrypted_group_data_len) ; + + result_gd->encrypted_group_data = NULL ; // prevents deletion + delete item; + + return ; } - GXS_NET_TUNNEL_DEBUG() << " : result is of type group summary result for service " << result_gs->service << std::dec << ": " << std::endl; - - for(auto it(result_gs->group_infos.begin());it!=result_gs->group_infos.end();++it) - std::cerr << " group " << (*it).group_id << ": " << (*it).group_name << ", " << (*it).number_of_messages << " messages, last is " << time(NULL)-(*it).last_message_ts << " secs ago." << std::endl; - - auto it = mSearchableServices.find(result_gs->service) ; - - if(it == mSearchableServices.end()) - { - GXS_NET_TUNNEL_ERROR() << ": deserialized item is for service " << std::hex << result_gs->service << std::dec << " that is in the searchable services list." << std::endl; - return ; - } - - it->second->receiveTurtleSearchResults(request_id,result_gs->group_infos) ; + GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl; } diff --git a/libretroshare/src/gxs/rsnxs.h b/libretroshare/src/gxs/rsnxs.h index 7d617ee30..a6044aa46 100644 --- a/libretroshare/src/gxs/rsnxs.h +++ b/libretroshare/src/gxs/rsnxs.h @@ -114,6 +114,14 @@ public: */ virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list& group_infos)=0; + /*! + * \brief receiveTurtleSearchResults + * Called by turtle (through RsGxsNetTunnel) when new data is received + * \param req Turtle search request ID associated with this result + * \param encrypted_group_data Group data + */ + virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0; + /*! * \brief retrieveTurtleSearchResults * To be used to retrieve the search results that have been notified (or not) diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 7908f8b5b..1dfa8b78e 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -95,7 +95,7 @@ protected: virtual RetroShareLink::enumType getLinkType() = 0; virtual GroupFrameSettings::Type groupFrameSettingsType() { return GroupFrameSettings::Nothing; } virtual void groupInfoToGroupItemInfo(const RsGroupMetaData &groupInfo, GroupItemInfo &groupItemInfo, const RsUserdata *userdata); - virtual void checkRequestGroup(const RsGxsGroupId& grpId) {} + virtual void checkRequestGroup(const RsGxsGroupId& /* grpId */) {} // overload this one in order to retrieve full group data when the group is browsed private slots: void todo(); @@ -149,7 +149,7 @@ private: virtual void groupTreeCustomActions(RsGxsGroupId /*grpId*/, int /*subscribeFlags*/, QList &/*actions*/) {} virtual RsGxsCommentService *getCommentService() { return NULL; } virtual QWidget *createCommentHeaderWidget(const RsGxsGroupId &/*grpId*/, const RsGxsMessageId &/*msgId*/) { return NULL; } - virtual bool getDistantSearchResults(TurtleRequestId id, std::map& group_infos){ return false ;} + virtual bool getDistantSearchResults(TurtleRequestId /* id */, std::map& /* group_infos */){ return false ;} void initUi(); From 47e760a2c522f9508c65175b01bbed0106f05cab Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 5 Jul 2018 10:11:12 +0200 Subject: [PATCH 67/84] added a few missing mutexes in prevent searches to be shown when already known --- libretroshare/src/gxs/rsgxsnetservice.cc | 3 ++ .../gui/FileTransfer/SharedFilesDialog.cpp | 12 ++--- .../src/gui/gxs/GxsGroupFrameDialog.cpp | 45 +++++++++---------- .../src/gui/gxs/GxsGroupFrameDialog.h | 6 +-- 4 files changed, 34 insertions(+), 32 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 054795873..67e031b9e 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5109,6 +5109,8 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid) TurtleRequestId RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id) { + RS_STACK_MUTEX(mNxsMutex) ; + time_t now = time(NULL); auto it = mSearchedGroups.find(group_id) ; @@ -5157,6 +5159,7 @@ bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map< } bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSummary& gs) { + RS_STACK_MUTEX(mNxsMutex) ; for(auto it(mDistantSearchResults.begin());it!=mDistantSearchResults.end();++it) { auto it2 = it->second.find(group_id) ; diff --git a/retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp b/retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp index 240839a2e..1cfa95cb6 100644 --- a/retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/SharedFilesDialog.cpp @@ -1140,14 +1140,14 @@ void LocalSharedFilesDialog::spawnCustomPopupMenu( QPoint point ) { shareChannelMenu.setIcon(QIcon(IMAGE_CHANNEL)); - std::list grp_metas ; + std::map grp_metas ; channelDialog->getGroupList(grp_metas) ; std::vector > grplist ; // I dont use a std::map because two or more channels may have the same name. for(auto it(grp_metas.begin());it!=grp_metas.end();++it) - if(IS_GROUP_PUBLISHER((*it).mSubscribeFlags) && IS_GROUP_SUBSCRIBED((*it).mSubscribeFlags)) - grplist.push_back(std::make_pair((*it).mGroupName, (*it).mGroupId)); + if(IS_GROUP_PUBLISHER((*it).second.mSubscribeFlags) && IS_GROUP_SUBSCRIBED((*it).second.mSubscribeFlags)) + grplist.push_back(std::make_pair((*it).second.mGroupName, (*it).second.mGroupId)); std::sort(grplist.begin(),grplist.end(),ChannelCompare()) ; @@ -1164,14 +1164,14 @@ void LocalSharedFilesDialog::spawnCustomPopupMenu( QPoint point ) { shareForumMenu.setIcon(QIcon(IMAGE_FORUMS)); - std::list grp_metas ; + std::map grp_metas ; forumsDialog->getGroupList(grp_metas) ; std::vector > grplist ; // I dont use a std::map because two or more channels may have the same name. for(auto it(grp_metas.begin());it!=grp_metas.end();++it) - if(IS_GROUP_SUBSCRIBED((*it).mSubscribeFlags)) - grplist.push_back(std::make_pair((*it).mGroupName, (*it).mGroupId)); + if(IS_GROUP_SUBSCRIBED((*it).second.mSubscribeFlags)) + grplist.push_back(std::make_pair((*it).second.mGroupName, (*it).second.mGroupId)); std::sort(grplist.begin(),grplist.end(),ChannelCompare()) ; diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 2ac49df42..180ffa231 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -123,7 +123,7 @@ GxsGroupFrameDialog::~GxsGroupFrameDialog() delete(ui); } -void GxsGroupFrameDialog::getGroupList(std::list& group_list) +void GxsGroupFrameDialog::getGroupList(std::map &group_list) { group_list = mCachedGroupMetas ; @@ -265,8 +265,6 @@ void GxsGroupFrameDialog::updateSearchResults() auto it2 = mSearchGroupsItems.find(*it); - std::set& known_groups(mKnownGroups[*it]) ; - if(mSearchGroupsItems.end() == it2) { std::cerr << "GxsGroupFrameDialog::updateSearchResults(): received result notification for req " << std::hex << *it << std::dec << " but no item present!" << std::endl; @@ -276,24 +274,22 @@ void GxsGroupFrameDialog::updateSearchResults() QList group_items ; for(auto it3(group_infos.begin());it3!=group_infos.end();++it3) - if(known_groups.end() == known_groups.find(it3->first)) - { + if(mCachedGroupMetas.find(it3->first) == mCachedGroupMetas.end()) + { std::cerr << " adding new group " << it3->first << " " << it3->second.group_id << " \"" << it3->second.group_name << "\"" << std::endl; - known_groups.insert(it3->first) ; - - GroupItemInfo i ; - i.id = QString(it3->second.group_id.toStdString().c_str()) ; - i.name = QString::fromUtf8(it3->second.group_name.c_str()) ; - i.description = QString::fromUtf8(it3->second.group_description.c_str()) ; - i.popularity = 0; // could be set to the number of hits + GroupItemInfo i ; + i.id = QString(it3->second.group_id.toStdString().c_str()) ; + i.name = QString::fromUtf8(it3->second.group_name.c_str()) ; + i.description = QString::fromUtf8(it3->second.group_description.c_str()) ; + i.popularity = 0; // could be set to the number of hits i.lastpost = QDateTime::fromTime_t(it3->second.last_message_ts); i.subscribeFlags = 0; // irrelevant here i.publishKey = false ; // IS_GROUP_PUBLISHER(groupInfo.mSubscribeFlags) ; i.adminKey = false ; // IS_GROUP_ADMIN(groupInfo.mSubscribeFlags) ; i.max_visible_posts = it3->second.number_of_messages ; - group_items.push_back(i); + group_items.push_back(i); } ui->groupTreeWidget->fillGroupItems(it2->second, group_items); @@ -768,7 +764,9 @@ void GxsGroupFrameDialog::changedCurrentGroup(const QString &groupId) } // send a request for the group, if it has been distant-searched. - checkRequestGroup(mGroupId) ; + + if(mCachedGroupMetas.find(mGroupId) == mCachedGroupMetas.end()) + checkRequestGroup(mGroupId) ; /* search exisiting tab */ GxsMessageFrameWidget *msgWidget = messageWidget(mGroupId, true); @@ -894,7 +892,7 @@ void GxsGroupFrameDialog::groupInfoToGroupItemInfo(const RsGroupMetaData &groupI } } -void GxsGroupFrameDialog::insertGroupsData(const std::list &groupList, const RsUserdata *userdata) +void GxsGroupFrameDialog::insertGroupsData(const std::map &groupList, const RsUserdata *userdata) { if (!mInitialized) { return; @@ -902,20 +900,18 @@ void GxsGroupFrameDialog::insertGroupsData(const std::list &gro mInFill = true; - std::list::const_iterator it; - QList adminList; QList subList; QList popList; QList otherList; std::multimap popMap; - for (it = groupList.begin(); it != groupList.end(); ++it) { + for (auto it = groupList.begin(); it != groupList.end(); ++it) { /* sort it into Publish (Own), Subscribed, Popular and Other */ - uint32_t flags = it->mSubscribeFlags; + uint32_t flags = it->second.mSubscribeFlags; GroupItemInfo groupItemInfo; - groupInfoToGroupItemInfo(*it, groupItemInfo, userdata); + groupInfoToGroupItemInfo(it->second, groupItemInfo, userdata); if (IS_GROUP_SUBSCRIBED(flags)) { @@ -932,7 +928,7 @@ void GxsGroupFrameDialog::insertGroupsData(const std::list &gro else { //popMap.insert(std::make_pair(it->mPop, groupItemInfo)); /* rate the others by popularity */ - popMap.insert(std::make_pair(it->mLastPost, groupItemInfo)); /* rate the others by time of last post */ + popMap.insert(std::make_pair(it->second.mLastPost, groupItemInfo)); /* rate the others by time of last post */ } } @@ -1043,9 +1039,12 @@ void GxsGroupFrameDialog::loadGroupSummary(const uint32_t &token) RsUserdata *userdata = NULL; loadGroupSummaryToken(token, groupInfo, userdata); - mCachedGroupMetas = groupInfo ; + mCachedGroupMetas.clear(); + for(auto it(groupInfo.begin());it!=groupInfo.end();++it) + mCachedGroupMetas[(*it).mGroupId] = *it; - insertGroupsData(groupInfo, userdata); + insertGroupsData(mCachedGroupMetas, userdata); + updateSearchResults(); mStateHelper->setLoading(TOKEN_TYPE_GROUP_SUMMARY, false); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 1dfa8b78e..e429d98b2 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -82,7 +82,7 @@ public: virtual QString getHelpString() const =0; - virtual void getGroupList(std::list& groups) ; + virtual void getGroupList(std::map &groups) ; protected: virtual void showEvent(QShowEvent *event); @@ -162,7 +162,7 @@ private: void processSettings(bool load); // New Request/Response Loading Functions. - void insertGroupsData(const std::list &groupList, const RsUserdata *userdata); + void insertGroupsData(const std::map &groupList, const RsUserdata *userdata); void requestGroupSummary(); void loadGroupSummary(const uint32_t &token); @@ -209,7 +209,7 @@ private: /** Qt Designer generated object */ Ui::GxsGroupFrameDialog *ui; - std::list mCachedGroupMetas; + std::map mCachedGroupMetas; std::map mSearchGroupsItems ; std::map > mKnownGroups; From 59c51a250b23d38089fa27658d22c10cabe3da35 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 5 Jul 2018 10:16:48 +0200 Subject: [PATCH 68/84] removed debug switch to allow display of existing distant group search results --- libretroshare/src/gxs/rsgxsnetservice.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 67e031b9e..d8a882945 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -5196,8 +5196,7 @@ void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std: // only keep groups that are not locally known, and groups that are not already in the mDistantSearchResults structure for(auto it(group_infos.begin());it!=group_infos.end();++it) -#warning Uncomment when done with testing! -// if(grpMeta[(*it).group_id] == NULL) + if(grpMeta[(*it).group_id] == NULL) { filtered_results.push_back(*it) ; From 0e37de3e1145cc20432eda51516f0b5ed1a4b32d Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 5 Jul 2018 11:43:55 +0200 Subject: [PATCH 69/84] made retrieval of group data manual instead of automatic --- libretroshare/src/gxs/rsgxsnettunnel.cc | 8 ++-- libretroshare/src/turtle/p3turtle.cc | 2 +- .../src/gui/common/GroupTreeWidget.cpp | 17 +++++++++ .../src/gui/common/GroupTreeWidget.h | 1 + .../src/gui/gxs/GxsGroupFrameDialog.cpp | 37 ++++++++++++++++--- .../src/gui/gxs/GxsGroupFrameDialog.h | 1 + 6 files changed, 56 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 86bee73e7..90d73e85a 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -94,7 +94,7 @@ public: RsGxsNetTunnelKeepAliveItem() :RsGxsNetTunnelItem(RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE) {} virtual ~RsGxsNetTunnelKeepAliveItem() {} - virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) {} + virtual void serial_process(RsGenericSerializer::SerializeJob /* j */,RsGenericSerializer::SerializeContext& /* ctx */) {} }; class RsGxsNetTunnelRandomBiasItem: public RsGxsNetTunnelItem @@ -314,7 +314,7 @@ bool RsGxsNetTunnelService::receiveTunnelData(uint16_t service_id, unsigned char return true; } -bool RsGxsNetTunnelService::sendTunnelData(uint16_t service_id,unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) +bool RsGxsNetTunnelService::sendTunnelData(uint16_t /* service_id */,unsigned char *& data,uint32_t data_len,const RsGxsNetTunnelVirtualPeerId& virtual_peer) { RS_STACK_MUTEX(mGxsNetTunnelMtx); // The item is serialized and encrypted using chacha20+SHA256, using the generic turtle encryption, and then sent to the turtle router. @@ -398,7 +398,7 @@ bool RsGxsNetTunnelService::requestDistantPeers(uint16_t service_id, const RsGxs return true; } -bool RsGxsNetTunnelService::releaseDistantPeers(uint16_t service_id,const RsGxsGroupId& group_id) +bool RsGxsNetTunnelService::releaseDistantPeers(uint16_t /* service_id */,const RsGxsGroupId& group_id) { RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -492,7 +492,7 @@ void RsGxsNetTunnelService::connectToTurtleRouter(p3turtle *tr) mTurtle->registerTunnelService(this) ; } -bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) +bool RsGxsNetTunnelService::handleTunnelRequest(const RsFileHash &hash,const RsPeerId& /* peer_id */) { RS_STACK_MUTEX(mGxsNetTunnelMtx); // We simply check for wether a managed group has a hash that corresponds to the given hash. diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 31286de8f..1fe6adfb0 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -1125,7 +1125,7 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) // This is an error: how could we receive a search result corresponding to a search item we // have forwarded but that it not in the list ?? - std::cerr << __PRETTY_FUNCTION__ << ": search result has no peer direction!" << std::endl ; + std::cerr << __PRETTY_FUNCTION__ << ": search result for request " << std::hex << item->request_id << std::dec << " has no peer direction!" << std::endl ; return ; } diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index 4a88eee48..d823298d9 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -430,6 +430,23 @@ QTreeWidgetItem *GroupTreeWidget::addSearchItem(const QString& search_string, ui return item; } +bool GroupTreeWidget::isSearchRequestResult(QPoint &point,QString& group_id,uint32_t& search_req_id) +{ + QTreeWidgetItem *item = ui->treeWidget->itemAt(point); + if (item == NULL) + return false; + + QTreeWidgetItem *parent = item->parent(); + + if(parent == NULL) + return false ; + + search_req_id = parent->data(COLUMN_DATA, ROLE_REQUEST_ID).toUInt(); + group_id = itemId(item) ; + + return search_req_id > 0; +} + bool GroupTreeWidget::isSearchRequestItem(QPoint &point,uint32_t& search_req_id) { QTreeWidgetItem *item = ui->treeWidget->itemAt(point); diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.h b/retroshare-gui/src/gui/common/GroupTreeWidget.h index 4ad19f0e1..64506cfb0 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.h +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.h @@ -95,6 +95,7 @@ public: void setUnreadCount(QTreeWidgetItem *item, int unreadCount); bool isSearchRequestItem(QPoint &point,uint32_t& search_req_id); + bool isSearchRequestResult(QPoint &point, QString &group_id, uint32_t& search_req_id); QTreeWidgetItem *getItemFromId(const QString &id); QTreeWidgetItem *activateId(const QString &id, bool focus); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index 180ffa231..a1a66eed2 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -47,6 +47,7 @@ #define IMAGE_SHARE ":/images/share-icon-16.png" #define IMAGE_TABNEW ":/images/tab-new.png" #define IMAGE_DELETE ":/images/delete.png" +#define IMAGE_RETRIEVE ":/images/edit_add24.png" #define IMAGE_COMMENT "" #define TOKEN_TYPE_GROUP_SUMMARY 1 @@ -345,6 +346,19 @@ void GxsGroupFrameDialog::groupTreeCustomPopupMenu(QPoint point) return ; } + // Then check whether we have a searched item, or a normal group + + QString group_id_s ; + + if(ui->groupTreeWidget->isSearchRequestResult(point,group_id_s,search_request_id)) + { + QMenu contextMnu(this); + + contextMnu.addAction(QIcon(IMAGE_RETRIEVE), tr("Request data"), this, SLOT(distantRequestGroupData()))->setData(group_id_s); + contextMnu.exec(QCursor::pos()); + return ; + } + QString id = ui->groupTreeWidget->itemIdAt(point); if (id.isEmpty()) return; @@ -763,11 +777,6 @@ void GxsGroupFrameDialog::changedCurrentGroup(const QString &groupId) return; } - // send a request for the group, if it has been distant-searched. - - if(mCachedGroupMetas.find(mGroupId) == mCachedGroupMetas.end()) - checkRequestGroup(mGroupId) ; - /* search exisiting tab */ GxsMessageFrameWidget *msgWidget = messageWidget(mGroupId, true); @@ -1198,3 +1207,21 @@ void GxsGroupFrameDialog::searchNetwork(const QString& search_string) mSearchGroupsItems[request_id] = ui->groupTreeWidget->addSearchItem(tr("Search for")+ " \"" + search_string + "\"",(uint32_t)request_id,QIcon(icon(ICON_SEARCH))); } +void GxsGroupFrameDialog::distantRequestGroupData() +{ + QAction *action = dynamic_cast(sender()) ; + + if(!action) + return ; + + RsGxsGroupId group_id(action->data().toString().toStdString()); + + if(group_id.isNull()) + { + std::cerr << "Cannot retrieve group! Group id is null!" << std::endl; + } + + std::cerr << "Explicit request for group " << group_id << std::endl; + checkRequestGroup(group_id) ; +} + diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index e429d98b2..4951caf36 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -108,6 +108,7 @@ private slots: void restoreGroupKeys(); void newGroup(); + void distantRequestGroupData(); void changedCurrentGroup(const QString &groupId); void groupTreeMiddleButtonClicked(QTreeWidgetItem *item); From 0cc87c988046ffc2f947ba3396edaf60add04255 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 5 Jul 2018 12:20:22 +0200 Subject: [PATCH 70/84] fixed bug causing search data result to not be forwarded correctly (missing request id) --- libretroshare/src/turtle/rsturtleitem.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index 20e9666f1..182e6deca 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -184,6 +184,8 @@ RsTurtleSearchResultItem *RsTurtleGenericSearchResultItem::duplicate() const sr->result_data = (unsigned char*)rs_malloc(result_data_len) ; memcpy(sr->result_data,result_data,result_data_len) ; sr->result_data_len = result_data_len ; + sr->request_id = request_id ; + sr->depth = depth ; return sr ; } From 4a64ea5f1f3099952bb80de959406c77fd43fe97 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 5 Jul 2018 14:00:04 +0200 Subject: [PATCH 71/84] added optional use of dist sync in GroupFrameDialog --- libretroshare/src/gxs/rsgxsnettunnel.cc | 26 +++---- libretroshare/src/retroshare/rsgxschannels.h | 68 ++++++++----------- .../src/gui/common/GroupTreeWidget.cpp | 5 ++ .../src/gui/common/GroupTreeWidget.h | 1 + .../src/gui/gxs/GxsGroupFrameDialog.cpp | 9 ++- .../src/gui/gxs/GxsGroupFrameDialog.h | 3 +- .../src/gui/gxschannels/GxsChannelDialog.cpp | 2 +- 7 files changed, 57 insertions(+), 57 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 90d73e85a..2b234fe57 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -199,9 +199,9 @@ public: switch(item_subtype) { - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER: return new RsGxsNetTunnelVirtualPeerItem ; - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ; - case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ; + case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_VIRTUAL_PEER : return new RsGxsNetTunnelVirtualPeerItem ; + case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_KEEP_ALIVE : return new RsGxsNetTunnelKeepAliveItem ; + case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_RANDOM_BIAS : return new RsGxsNetTunnelRandomBiasItem ; case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_SUBSTRING : return new RsGxsNetTunnelTurtleSearchSubstringItem; case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_REQUEST : return new RsGxsNetTunnelTurtleSearchGroupRequestItem; case RS_PKT_SUBTYPE_GXS_NET_TUNNEL_TURTLE_SEARCH_GROUP_SUMMARY : return new RsGxsNetTunnelTurtleSearchGroupSummaryItem; @@ -216,16 +216,16 @@ public: template<> void RsTypeSerializer::serial_process( RsGenericSerializer::SerializeJob j, RsGenericSerializer::SerializeContext& ctx, RsGxsGroupSummary& gs, const std::string& member_name ) { - RsTypeSerializer::serial_process(j,ctx,gs.group_id,member_name+"-group_id") ; // RsGxsGroupId group_id ; - RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_NAME ,gs.group_name,member_name+"-group_name") ; // std::string group_name ; - RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_COMMENT,gs.group_description,member_name+"-group_description") ; // std::string group_description ; - RsTypeSerializer::serial_process(j,ctx,TLV_TYPE_STR_VALUE ,gs.search_context,member_name+"-group_name") ; // std::string search_context ; - RsTypeSerializer::serial_process(j,ctx,gs.author_id ,member_name+"-author_id") ; // RsGxsId author_id ; - RsTypeSerializer::serial_process(j,ctx,gs.publish_ts ,member_name+"-publish_ts") ; // time_t publish_ts ; - RsTypeSerializer::serial_process(j,ctx,gs.number_of_messages,member_name+"-number_of_messages") ; // uint32_t number_of_messages ; - RsTypeSerializer::serial_process(j,ctx,gs.last_message_ts,member_name+"-last_message_ts") ; // time_t last_message_ts ; - RsTypeSerializer::serial_process(j,ctx,gs.sign_flags,member_name+"-sign_flags") ; // uint32_t sign_flags ; - RsTypeSerializer::serial_process(j,ctx,gs.popularity,member_name+"-popularity") ; // uint32_t popularity ; + RsTypeSerializer::serial_process (j,ctx,gs.group_id ,member_name+"-group_id") ; // RsGxsGroupId group_id ; + RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_NAME ,gs.group_name,member_name+"-group_name") ; // std::string group_name ; + RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_COMMENT ,gs.group_description,member_name+"-group_description") ; // std::string group_description ; + RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_VALUE ,gs.search_context,member_name+"-group_name") ; // std::string search_context ; + RsTypeSerializer::serial_process (j,ctx,gs.author_id ,member_name+"-author_id") ; // RsGxsId author_id ; + RsTypeSerializer::serial_process (j,ctx,gs.publish_ts ,member_name+"-publish_ts") ; // time_t publish_ts ; + RsTypeSerializer::serial_process (j,ctx,gs.number_of_messages,member_name+"-number_of_messages") ; // uint32_t number_of_messages ; + RsTypeSerializer::serial_process (j,ctx,gs.last_message_ts ,member_name+"-last_message_ts") ; // time_t last_message_ts ; + RsTypeSerializer::serial_process(j,ctx,gs.sign_flags ,member_name+"-sign_flags") ; // uint32_t sign_flags ; + RsTypeSerializer::serial_process(j,ctx,gs.popularity ,member_name+"-popularity") ; // uint32_t popularity ; } diff --git a/libretroshare/src/retroshare/rsgxschannels.h b/libretroshare/src/retroshare/rsgxschannels.h index a23275b3b..c80b75f31 100644 --- a/libretroshare/src/retroshare/rsgxschannels.h +++ b/libretroshare/src/retroshare/rsgxschannels.h @@ -76,61 +76,49 @@ std::ostream &operator<<(std::ostream &out, const RsGxsChannelPost &post); class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService { - public: +public: explicit RsGxsChannels(RsGxsIface *gxs) - :RsGxsIfaceHelper(gxs) {} + :RsGxsIfaceHelper(gxs) {} virtual ~RsGxsChannels() {} /* Specific Service Data */ -virtual bool getGroupData(const uint32_t &token, std::vector &groups) = 0; -virtual bool getPostData(const uint32_t &token, std::vector &posts, std::vector &cmts) = 0; -virtual bool getPostData(const uint32_t &token, std::vector &posts) = 0; -//Not currently used -//virtual bool getRelatedPosts(const uint32_t &token, std::vector &posts) = 0; + virtual bool getGroupData(const uint32_t &token, std::vector &groups) = 0; + virtual bool getPostData(const uint32_t &token, std::vector &posts, std::vector &cmts) = 0; + virtual bool getPostData(const uint32_t &token, std::vector &posts) = 0; - /* From RsGxsCommentService */ -//virtual bool getCommentData(const uint32_t &token, std::vector &comments) = 0; -//virtual bool getRelatedComments(const uint32_t &token, std::vector &comments) = 0; -//virtual bool createComment(uint32_t &token, RsGxsComment &comment) = 0; -//virtual bool createVote(uint32_t &token, RsGxsVote &vote) = 0; + ////////////////////////////////////////////////////////////////////////////// + /// Distant synchronisation methods /// + ////////////////////////////////////////////////////////////////////////////// + /// + virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; + virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; + virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) =0; + virtual bool clearDistantSearchResults(TurtleRequestId req)=0; + virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0; - virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0; - virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0; - virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map &results) =0; - virtual bool clearDistantSearchResults(TurtleRequestId req)=0; - virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0; + ////////////////////////////////////////////////////////////////////////////// + virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0; - ////////////////////////////////////////////////////////////////////////////// -virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0; + virtual bool setChannelAutoDownload(const RsGxsGroupId &groupId, bool enabled) = 0; + virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled) = 0; -virtual bool setChannelAutoDownload(const RsGxsGroupId &groupId, bool enabled) = 0; -virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled) = 0; + virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory)=0; + virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory)=0; -virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory)=0; -virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory)=0; - -//virtual void setChannelAutoDownload(uint32_t& token, const RsGxsGroupId& groupId, bool autoDownload) = 0; - -//virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask); -//virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask); - -//virtual bool groupRestoreKeys(const std::string &groupId); - virtual bool groupShareKeys(const RsGxsGroupId &groupId, std::set& peers)=0; + virtual bool groupShareKeys(const RsGxsGroupId &groupId, std::set& peers)=0; // Overloaded subscribe fn. -virtual bool subscribeToGroup(uint32_t &token, const RsGxsGroupId &groupId, bool subscribe) = 0; + virtual bool subscribeToGroup(uint32_t &token, const RsGxsGroupId &groupId, bool subscribe) = 0; -virtual bool createGroup(uint32_t &token, RsGxsChannelGroup &group) = 0; -virtual bool createPost(uint32_t &token, RsGxsChannelPost &post) = 0; - -virtual bool updateGroup(uint32_t &token, RsGxsChannelGroup &group) = 0; - - // File Interface -virtual bool ExtraFileHash(const std::string &path, std::string filename) = 0; -virtual bool ExtraFileRemove(const RsFileHash &hash) = 0; + virtual bool createGroup(uint32_t &token, RsGxsChannelGroup &group) = 0; + virtual bool createPost(uint32_t &token, RsGxsChannelPost &post) = 0; + virtual bool updateGroup(uint32_t &token, RsGxsChannelGroup &group) = 0; + // File Interface + virtual bool ExtraFileHash(const std::string &path, std::string filename) = 0; + virtual bool ExtraFileRemove(const RsFileHash &hash) = 0; }; diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp index d823298d9..87be83e91 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.cpp +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.cpp @@ -430,6 +430,11 @@ QTreeWidgetItem *GroupTreeWidget::addSearchItem(const QString& search_string, ui return item; } +void GroupTreeWidget::setDistSearchVisible(bool visible) +{ + ui->distantSearchLineEdit->setVisible(visible); +} + bool GroupTreeWidget::isSearchRequestResult(QPoint &point,QString& group_id,uint32_t& search_req_id) { QTreeWidgetItem *item = ui->treeWidget->itemAt(point); diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.h b/retroshare-gui/src/gui/common/GroupTreeWidget.h index 64506cfb0..59e0ae261 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.h +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.h @@ -83,6 +83,7 @@ public: // Add a new category item QTreeWidgetItem *addCategoryItem(const QString &name, const QIcon &icon, bool expand); // Add a new search item + void setDistSearchVisible(bool) ; // shows/hides distant search UI parts. QTreeWidgetItem *addSearchItem(const QString& search_string, uint32_t id, const QIcon &icon) ; void removeSearchItem(QTreeWidgetItem *item); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp index a1a66eed2..58e7bb93a 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.cpp @@ -67,7 +67,7 @@ */ /** Constructor */ -GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *parent) +GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *parent,bool allow_dist_sync) : RsGxsUpdateBroadcastPage(ifaceImpl, parent) { /* Invoke the Qt Designer generated object setup routine */ @@ -75,6 +75,7 @@ GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *p ui->setupUi(this); mInitialized = false; + mDistSyncAllowed = allow_dist_sync; mInFill = false; mCountChildMsgs = false; mYourGroups = NULL; @@ -96,12 +97,16 @@ GxsGroupFrameDialog::GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *p connect(ui->groupTreeWidget, SIGNAL(treeCustomContextMenuRequested(QPoint)), this, SLOT(groupTreeCustomPopupMenu(QPoint))); connect(ui->groupTreeWidget, SIGNAL(treeCurrentItemChanged(QString)), this, SLOT(changedCurrentGroup(QString))); connect(ui->groupTreeWidget->treeWidget(), SIGNAL(signalMouseMiddleButtonClicked(QTreeWidgetItem*)), this, SLOT(groupTreeMiddleButtonClicked(QTreeWidgetItem*))); - connect(ui->groupTreeWidget, SIGNAL(distantSearchRequested(const QString&)), this, SLOT(searchNetwork(const QString&))); connect(ui->messageTabWidget, SIGNAL(tabCloseRequested(int)), this, SLOT(messageTabCloseRequested(int))); connect(ui->messageTabWidget, SIGNAL(currentChanged(int)), this, SLOT(messageTabChanged(int))); connect(ui->todoPushButton, SIGNAL(clicked()), this, SLOT(todo())); + ui->groupTreeWidget->setDistSearchVisible(allow_dist_sync) ; + + if(allow_dist_sync) + connect(ui->groupTreeWidget, SIGNAL(distantSearchRequested(const QString&)), this, SLOT(searchNetwork(const QString&))); + /* Set initial size the splitter */ ui->splitter->setStretchFactor(0, 0); ui->splitter->setStretchFactor(1, 1); diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h index 4951caf36..46ff05ec3 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.h @@ -72,7 +72,7 @@ public: }; public: - GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *parent = 0); + GxsGroupFrameDialog(RsGxsIfaceHelper *ifaceImpl, QWidget *parent = 0,bool allow_dist_sync=false); virtual ~GxsGroupFrameDialog(); bool navigate(const RsGxsGroupId &groupId, const RsGxsMessageId& msgId); @@ -190,6 +190,7 @@ protected: private: bool mInitialized; bool mInFill; + bool mDistSyncAllowed; QString mSettingsName; RsGxsGroupId mGroupId; RsGxsIfaceHelper *mInterface; diff --git a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp index f839a0a3f..8090dc353 100644 --- a/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp +++ b/retroshare-gui/src/gui/gxschannels/GxsChannelDialog.cpp @@ -47,7 +47,7 @@ public: /** Constructor */ GxsChannelDialog::GxsChannelDialog(QWidget *parent) - : GxsGroupFrameDialog(rsGxsChannels, parent) + : GxsGroupFrameDialog(rsGxsChannels, parent,true) { } From 98f8e4da0a296ef3db6bee3b92e9d791f16d430b Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 5 Jul 2018 17:40:06 +0200 Subject: [PATCH 72/84] added per-item type limit for turtle search results --- libretroshare/src/gxs/rsgxsnettunnel.cc | 8 ++- libretroshare/src/gxs/rsgxsnettunnel.h | 2 +- libretroshare/src/turtle/p3turtle.cc | 52 ++++++++++--------- libretroshare/src/turtle/p3turtle.h | 7 +-- libretroshare/src/turtle/rsturtleitem.cc | 6 +++ libretroshare/src/turtle/rsturtleitem.h | 11 ++-- .../src/turtle/turtleclientservice.h | 7 ++- 7 files changed, 59 insertions(+), 34 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 2b234fe57..fc8d5eb79 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -36,6 +36,8 @@ #define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL: " << __FUNCTION__ << " : " #define GXS_NET_TUNNEL_ERROR() std::cerr << "(EE) GXS_NET_TUNNEL ERROR : " +static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1; +static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100; RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") { @@ -1011,7 +1013,7 @@ TurtleRequestId RsGxsNetTunnelService::turtleSearchRequest(const std::string& ma return mTurtle->turtleSearch(mem,size,this) ; } -bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size) +bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_size,uint32_t& max_allowed_hits) { GXS_NET_TUNNEL_DEBUG() << ": received a request." << std::endl; @@ -1023,6 +1025,8 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d { GXS_NET_TUNNEL_DEBUG() << " : type is substring for service " << std::hex << (int)substring_sr->service << std::dec << std::endl; + max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH ; + std::list results ; RS_STACK_MUTEX(mGxsNetTunnelMtx); @@ -1057,6 +1061,8 @@ bool RsGxsNetTunnelService::receiveSearchRequest(unsigned char *search_request_d RS_STACK_MUTEX(mGxsNetTunnelMtx); auto it = mSearchableServices.find(substring_gr->service) ; + max_allowed_hits = RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA ; + unsigned char *encrypted_group_data = NULL ; uint32_t encrypted_group_data_len = 0 ; diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index ade780fe8..0fdff3ccc 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -244,7 +244,7 @@ public: * \brief receiveSearchRequest * See RsTurtleClientService::@ */ - virtual bool receiveSearchRequest(unsigned char *search_request_data,uint32_t search_request_data_len,unsigned char *& search_result_data,uint32_t& search_result_data_len); + virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits); virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len); // Overloaded from RsTickingThread diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 1fe6adfb0..0db63a7df 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -83,17 +83,18 @@ void TS_dumpState() ; // - The total number of TR per second emmited from self will be MAX_TUNNEL_REQS_PER_SECOND / TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 0.5 // - I updated forward probabilities to higher values, and min them to 1/nb_connected_friends to prevent blocking tunnels. // -static const time_t TUNNEL_REQUESTS_LIFE_TIME = 240 ; /// life time for tunnel requests in the cache. -static const time_t SEARCH_REQUESTS_LIFE_TIME = 240 ; /// life time for search requests in the cache -static const time_t REGULAR_TUNNEL_DIGGING_TIME = 300 ; /// maximum interval between two tunnel digging campaigns. -static const time_t MAXIMUM_TUNNEL_IDLE_TIME = 60 ; /// maximum life time of an unused tunnel. -static const time_t EMPTY_TUNNELS_DIGGING_TIME = 50 ; /// look into tunnels regularly every 50 sec. -static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE = 5 ; /// estimate tunnel speed every 5 seconds -static const time_t TUNNEL_CLEANING_LAPS_TIME = 10 ; /// clean tunnels every 10 secs -static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 2 ; /// Tunnel management calls every 2 secs. -static const uint32_t MAX_TUNNEL_REQS_PER_SECOND = 1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before -static const uint32_t MAX_ALLOWED_SR_IN_CACHE = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec. -static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS =5000 ; /// maximum number of search results forwarded back to the source. +static const time_t TUNNEL_REQUESTS_LIFE_TIME = 240 ; /// life time for tunnel requests in the cache. +static const time_t SEARCH_REQUESTS_LIFE_TIME = 240 ; /// life time for search requests in the cache +static const time_t REGULAR_TUNNEL_DIGGING_TIME = 300 ; /// maximum interval between two tunnel digging campaigns. +static const time_t MAXIMUM_TUNNEL_IDLE_TIME = 60 ; /// maximum life time of an unused tunnel. +static const time_t EMPTY_TUNNELS_DIGGING_TIME = 50 ; /// look into tunnels regularly every 50 sec. +static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE = 5 ; /// estimate tunnel speed every 5 seconds +static const time_t TUNNEL_CLEANING_LAPS_TIME = 10 ; /// clean tunnels every 10 secs +static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 2 ; /// Tunnel management calls every 2 secs. +static const uint32_t MAX_TUNNEL_REQS_PER_SECOND = 1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before +static const uint32_t MAX_ALLOWED_SR_IN_CACHE = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec. +static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_FILES =5000 ; /// maximum number of search results forwarded back to the source. +static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT= 100 ; /// default maximum number of search results forwarded back source. static const float depth_peer_probability[7] = { 1.0f,0.99f,0.9f,0.7f,0.6f,0.5,0.4f } ; @@ -916,6 +917,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) // Perform local search off-mutex,because this might call some services that are above turtle in the mutex chain. uint32_t search_result_count = 0; + uint32_t max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT; if(item->PeerId() != _own_id) // is the request not coming from us? { @@ -924,7 +926,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) #endif std::list search_results ; - performLocalSearch(item,search_result_count,search_results) ; + performLocalSearch(item,search_result_count,search_results,max_allowed_hits) ; for(auto it(search_results.begin());it!=search_results.end();++it) { @@ -948,13 +950,14 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) req.result_count = search_result_count; req.keywords = item->GetKeywords() ; req.service_id = item->serviceId() ; + req.max_allowed_hits = max_allowed_hits; // if enough has been sent back already, do not sarch further #ifdef P3TURTLE_DEBUG std::cerr << " result count = " << req.result_count << std::endl; #endif - if(req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + if(req.result_count >= max_allowed_hits) return ; // If search depth not too large, also forward this search request to all other peers. @@ -1014,13 +1017,13 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item) // This function should be removed in the future, when file search will also use generic search items. -void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list& search_results) +void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list& search_results,uint32_t& max_allowed_hits) { RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast(item) ; if(ftsearch != NULL) { - performLocalSearch_files(ftsearch,req_result_count,search_results) ; + performLocalSearch_files(ftsearch,req_result_count,search_results,max_allowed_hits) ; return ; } @@ -1028,12 +1031,12 @@ void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_ if(gnsearch != NULL) { - performLocalSearch_generic(gnsearch,req_result_count,search_results) ; + performLocalSearch_generic(gnsearch,req_result_count,search_results,max_allowed_hits) ; return ; } } -void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list& result) +void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list& result,uint32_t& max_allowed_hits) { unsigned char *search_result_data = NULL ; uint32_t search_result_data_len = 0 ; @@ -1050,7 +1053,7 @@ void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item client = it->second ; } - if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len)) + if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len,max_allowed_hits)) { RsTurtleGenericSearchResultItem *result_item = new RsTurtleGenericSearchResultItem ; @@ -1061,7 +1064,7 @@ void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item } } -void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list& result) +void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list& result,uint32_t& max_allowed_hits) { #ifdef P3TURTLE_DEBUG std::cerr << "Performing rsFiles->search()" << std::endl ; @@ -1078,6 +1081,7 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint uint32_t item_size = 0 ; static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ; + max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_FILES; for(auto it(initialResults.begin());it!=initialResults.end();++it) { @@ -1096,7 +1100,7 @@ void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ; - if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= max_allowed_hits) { #ifdef P3TURTLE_DEBUG std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ; @@ -1151,18 +1155,18 @@ void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item) uint32_t n = item->count(); // not so good! - if(it->second.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS) + if(it->second.result_count >= it->second.max_allowed_hits) { std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl; return ; } - if(it->second.result_count + n > TURTLE_SEARCH_RESULT_MAX_HITS) + if(it->second.result_count + n > it->second.max_allowed_hits) { - for(uint32_t i=it->second.result_count + n; i>TURTLE_SEARCH_RESULT_MAX_HITS;--i) + for(uint32_t i=it->second.result_count + n; i>it->second.max_allowed_hits;--i) item->pop() ; - it->second.result_count = TURTLE_SEARCH_RESULT_MAX_HITS ; + it->second.result_count = it->second.max_allowed_hits ; } else it->second.result_count += n ; diff --git a/libretroshare/src/turtle/p3turtle.h b/libretroshare/src/turtle/p3turtle.h index be34a56a7..93d397d67 100644 --- a/libretroshare/src/turtle/p3turtle.h +++ b/libretroshare/src/turtle/p3turtle.h @@ -174,6 +174,7 @@ class TurtleSearchRequestInfo uint32_t result_count; // responses to this request. Useful to avoid spamming tunnel responses. std::string keywords; uint16_t service_id; // ID of the client service who issues the request. This is null if the request does not have a local origin. + uint32_t max_allowed_hits;// Max number of hits allowed for this search. This actually depends on the type of search (files, GXS groups, GXS group data, etc) }; class TurtleTunnelRequestInfo { @@ -397,9 +398,9 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config //------ Functions connecting the turtle router to other components.----------// /// Performs a search calling local cache and search structure. - void performLocalSearch (RsTurtleSearchRequestItem *item, uint32_t& req_result_count,std::list& result) ; - void performLocalSearch_files (RsTurtleFileSearchRequestItem *item, uint32_t& req_result_count,std::list& result) ; - void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count,std::list& result) ; + void performLocalSearch (RsTurtleSearchRequestItem *item, uint32_t& req_result_count,std::list& result,uint32_t& max_allowed_hits) ; + void performLocalSearch_files (RsTurtleFileSearchRequestItem *item, uint32_t& req_result_count, std::list& result, uint32_t &max_allowed_hits) ; + void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list& result, uint32_t &max_allowed_hits) ; /// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer. virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service); diff --git a/libretroshare/src/turtle/rsturtleitem.cc b/libretroshare/src/turtle/rsturtleitem.cc index 182e6deca..cfc775f24 100644 --- a/libretroshare/src/turtle/rsturtleitem.cc +++ b/libretroshare/src/turtle/rsturtleitem.cc @@ -50,6 +50,11 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c return NULL ; } +std::string RsTurtleGenericSearchRequestItem::GetKeywords() +{ + return std::string("Generic search : " + RsUtil::BinToHex(search_data,search_data_len,10)); +} + void RsTurtleStringSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ; @@ -67,6 +72,7 @@ void RsTurtleGenericSearchRequestItem::serial_process(RsGenericSerializer::Seria RsTypeSerializer::serial_process(j,ctx,request_id,"request_id") ; RsTypeSerializer::serial_process(j,ctx,depth,"depth") ; RsTypeSerializer::serial_process(j,ctx,service_id,"service_id") ; + RsTypeSerializer::serial_process(j,ctx,request_type,"request_type") ; RsTypeSerializer::TlvMemBlock_proxy prox(search_data,search_data_len) ; RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ; diff --git a/libretroshare/src/turtle/rsturtleitem.h b/libretroshare/src/turtle/rsturtleitem.h index d35cbb8fd..d8ee48165 100644 --- a/libretroshare/src/turtle/rsturtleitem.h +++ b/libretroshare/src/turtle/rsturtleitem.h @@ -82,7 +82,7 @@ class RsTurtleSearchRequestItem: public RsTurtleItem virtual RsTurtleSearchRequestItem *clone() const = 0 ; // used for cloning in routing methods virtual std::string GetKeywords() = 0; - virtual uint16_t serviceId() = 0 ; + virtual uint16_t serviceId() const= 0 ; uint32_t request_id ; // randomly generated request id. uint16_t depth ; // Used for limiting search depth. @@ -94,7 +94,7 @@ class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {} virtual ~RsTurtleFileSearchRequestItem() {} - virtual uint16_t serviceId() { return RS_SERVICE_TYPE_FILE_TRANSFER ; } + virtual uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; } virtual void search(std::list &) const =0; }; @@ -149,12 +149,15 @@ class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem uint16_t service_id ; // service to search uint32_t search_data_len ; + uint8_t request_type ; // type of request. This is used to limit the number of responses. unsigned char *search_data ; - std::string GetKeywords() { return std::string("Generic search " + RsUtil::BinToHex(search_data,search_data_len,10)); } - virtual uint16_t serviceId() { return service_id ; } + std::string GetKeywords() ; + virtual uint16_t serviceId() const { return service_id ; } virtual RsTurtleSearchRequestItem *clone() const ; + virtual uint32_t requestType() const { return request_type; } + void clear() { free(search_data); search_data=NULL; search_data_len=0; } protected: diff --git a/libretroshare/src/turtle/turtleclientservice.h b/libretroshare/src/turtle/turtleclientservice.h index b3aa37af1..039827ef4 100644 --- a/libretroshare/src/turtle/turtleclientservice.h +++ b/libretroshare/src/turtle/turtleclientservice.h @@ -101,10 +101,15 @@ class RsTurtleClientService * \param search_request_data_len length of the serialized search data * \param search_result_data generic serialized search result data * \param search_result_data_len length of the serialized search result data + * \param max_allowed_hits max number of hits allowed to be sent back and forwarded * * \return true if the search is successful. */ - virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,uint32_t /*search_request_data_len*/,unsigned char *& /*search_result_data*/,uint32_t& /*search_result_data_len*/) + virtual bool receiveSearchRequest(unsigned char */*search_request_data*/, + uint32_t /*search_request_data_len*/, + unsigned char *& /*search_result_data*/, + uint32_t& /*search_result_data_len*/, + uint32_t& /* max_allows_hits */) { std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ; return false; From b3dddeafdffaa0b739a7067dccfdccad246ac74e Mon Sep 17 00:00:00 2001 From: chelovechishko Date: Fri, 6 Jul 2018 23:55:12 +0900 Subject: [PATCH 73/84] add missing namespaces --- libretroshare/src/pqi/p3peermgr.cc | 174 +++++++++--------- libretroshare/src/services/p3banlist.cc | 30 +-- retroshare-gui/src/gui/MainWindow.cpp | 10 +- retroshare-gui/src/gui/PluginsPage.h | 12 +- .../src/gui/settings/PluginsPage.cpp | 20 +- retroshare-gui/src/gui/settings/PluginsPage.h | 3 + .../src/gui/settings/rsettingswin.cpp | 2 +- 7 files changed, 133 insertions(+), 118 deletions(-) diff --git a/libretroshare/src/pqi/p3peermgr.cc b/libretroshare/src/pqi/p3peermgr.cc index fb17dddb6..243b26135 100644 --- a/libretroshare/src/pqi/p3peermgr.cc +++ b/libretroshare/src/pqi/p3peermgr.cc @@ -89,11 +89,11 @@ static const std::string kConfigKeyProxyServerIpAddrTor = "PROXY_SERVER_IPADDR"; static const std::string kConfigKeyProxyServerPortTor = "PROXY_SERVER_PORT"; static const std::string kConfigKeyProxyServerIpAddrI2P = "PROXY_SERVER_IPADDR_I2P"; static const std::string kConfigKeyProxyServerPortI2P = "PROXY_SERVER_PORT_I2P"; - + void printConnectState(std::ostream &out, peerState &peer); peerState::peerState() - :netMode(RS_NET_MODE_UNKNOWN), vs_disc(RS_VS_DISC_FULL), vs_dht(RS_VS_DHT_FULL), lastcontact(0), + :netMode(RS_NET_MODE_UNKNOWN), vs_disc(RS_VS_DISC_FULL), vs_dht(RS_VS_DHT_FULL), lastcontact(0), hiddenNode(false), hiddenPort(0), hiddenType(RS_HIDDEN_TYPE_NONE) { sockaddr_storage_clear(localaddr); @@ -107,13 +107,13 @@ std::string textPeerConnectState(peerState &state) std::string out = "Id: " + state.id.toStdString() + "\n"; rs_sprintf_append(out, "NetMode: %lu\n", state.netMode); rs_sprintf_append(out, "VisState: Disc: %u Dht: %u\n", state.vs_disc, state.vs_dht); - + out += "laddr: "; out += sockaddr_storage_tostring(state.localaddr); out += "\neaddr: "; out += sockaddr_storage_tostring(state.serveraddr); out += "\n"; - + return out; } @@ -136,7 +136,7 @@ p3PeerMgrIMPL::p3PeerMgrIMPL(const RsPeerId& ssl_own_id, const RsPgpId& gpg_own_ mOwnState.netMode = RS_NET_MODE_UPNP; // Default to UPNP. mOwnState.vs_disc = RS_VS_DISC_FULL; mOwnState.vs_dht = RS_VS_DHT_FULL; - + // setup default ProxyServerAddress. // Tor sockaddr_storage_clear(mProxyServerAddressTor); @@ -154,7 +154,7 @@ p3PeerMgrIMPL::p3PeerMgrIMPL(const RsPeerId& ssl_own_id, const RsPgpId& gpg_own_ mProxyServerStatusTor = RS_NET_PROXY_STATUS_UNKNOWN ; mProxyServerStatusI2P = RS_NET_PROXY_STATUS_UNKNOWN; } - + #ifdef PEER_DEBUG std::cerr << "p3PeerMgr() Startup" << std::endl; #endif @@ -222,8 +222,8 @@ bool p3PeerMgrIMPL::forceHiddenNode() struct sockaddr_storage loopback; sockaddr_storage_clear(loopback); sockaddr_storage_ipv4_aton(loopback, "127.0.0.1"); - uint16_t port = sockaddr_storage_port(mOwnState.localaddr); - sockaddr_storage_ipv4_setport(loopback, port); + uint16_t port = sockaddr_storage_port(mOwnState.localaddr); + sockaddr_storage_ipv4_setport(loopback, port); setLocalAddress(AuthSSL::getAuthSSL()->OwnId(), loopback); @@ -254,7 +254,7 @@ bool p3PeerMgrIMPL::setOwnNetworkMode(uint32_t netMode) changed = true; } } - + // Pass on Flags to NetMgr. mNetMgr->setNetworkMode((netMode & RS_NET_MODE_ACTUAL)); return changed; @@ -267,7 +267,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht) RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ std::string out; - rs_sprintf(out, "p3PeerMgr::setOwnVisState() Existing vis: %u/%u Input vis: %u/%u", + rs_sprintf(out, "p3PeerMgr::setOwnVisState() Existing vis: %u/%u Input vis: %u/%u", mOwnState.vs_disc, mOwnState.vs_dht, vs_disc, vs_dht); rslog(RSL_WARNING, p3peermgrzone, out); @@ -275,7 +275,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht) std::cerr << out.c_str() << std::endl; #endif - if (mOwnState.vs_disc != vs_disc || mOwnState.vs_dht != vs_dht) + if (mOwnState.vs_disc != vs_disc || mOwnState.vs_dht != vs_dht) { mOwnState.vs_disc = vs_disc; mOwnState.vs_dht = vs_dht; @@ -283,7 +283,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht) IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ } } - + // Pass on Flags to NetMgr. mNetMgr->setVisState(vs_disc, vs_dht); @@ -575,7 +575,7 @@ bool p3PeerMgrIMPL::setHiddenDomainPort(const RsPeerId &ssl_id, const std::strin IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ - if (ssl_id == AuthSSL::getAuthSSL()->OwnId()) + if (ssl_id == AuthSSL::getAuthSSL()->OwnId()) { mOwnState.hiddenNode = true; mOwnState.hiddenDomain = domain; @@ -917,7 +917,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ - if (id == AuthSSL::getAuthSSL()->OwnId()) + if (id == AuthSSL::getAuthSSL()->OwnId()) { #ifdef PEER_DEBUG std::cerr << "p3PeerMgrIMPL::addFriend() cannot add own id as a friend." << std::endl; @@ -973,7 +973,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg /* setup connectivity parameters */ it->second.vs_disc = vs_disc; it->second.vs_dht = vs_dht; - + it->second.netMode = netMode; it->second.lastcontact = lastContact; @@ -995,7 +995,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg pstate.id = id; pstate.gpg_id = gpg_id; pstate.name = AuthGPG::getAuthGPG()->getGPGName(gpg_id); - + pstate.vs_disc = vs_disc; pstate.vs_dht = vs_dht; pstate.netMode = netMode; @@ -1018,7 +1018,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg mLinkMgr->addFriend(id, vs_dht != RS_VS_DHT_OFF); } - service_flags &= servicePermissionFlags(gpg_id) ; // Always reduce the permissions. + service_flags &= servicePermissionFlags(gpg_id) ; // Always reduce the permissions. #ifdef RS_CHATSERVER //Defined by chatserver setServicePermissionFlags(gpg_id,RS_NODE_PERM_NONE) ; #else @@ -1029,7 +1029,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg printPeerLists(std::cerr); mLinkMgr->printPeerLists(std::cerr); #endif - + return true; } @@ -1071,10 +1071,10 @@ bool p3PeerMgrIMPL::removeFriend(const RsPgpId &id) } for(std::list::iterator rit = sslid_toRemove.begin(); rit != sslid_toRemove.end(); ++rit) - if (mFriendList.end() != (it = mFriendList.find(*rit))) + if (mFriendList.end() != (it = mFriendList.find(*rit))) mFriendList.erase(it); - std::map::iterator it2 = mFriendsPermissionFlags.find(id) ; + std::map::iterator it2 = mFriendsPermissionFlags.find(id) ; if(it2 != mFriendsPermissionFlags.end()) mFriendsPermissionFlags.erase(it2); @@ -1146,13 +1146,13 @@ bool p3PeerMgrIMPL::removeFriend(const RsPeerId &id, bool removePgpId) } for(std::list::iterator rit = sslid_toRemove.begin(); rit != sslid_toRemove.end(); ++rit) - if (mFriendList.end() != (it = mFriendList.find(*rit))) + if (mFriendList.end() != (it = mFriendList.find(*rit))) mFriendList.erase(it); std::map::iterator it2 ; for(std::list::iterator rit = pgpid_toRemove.begin(); rit != pgpid_toRemove.end(); ++rit) - if (mFriendsPermissionFlags.end() != (it2 = mFriendsPermissionFlags.find(*rit))) + if (mFriendsPermissionFlags.end() != (it2 = mFriendsPermissionFlags.find(*rit))) mFriendsPermissionFlags.erase(it2); #ifdef PEER_DEBUG @@ -1432,7 +1432,7 @@ bool p3PeerMgrIMPL::setLocalAddress( const RsPeerId &id, if (changed) { IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ - + mNetMgr->setLocalAddress(addr); mLinkMgr->setLocalAddress(addr); } @@ -1499,9 +1499,9 @@ bool p3PeerMgrIMPL::setExtAddress( const RsPeerId &id, changed = true; } } - + mNetMgr->setExtAddress(addr); - + return changed; } @@ -1584,12 +1584,16 @@ bool p3PeerMgrIMPL::setDynDNS(const RsPeerId &id, const std::string &dyndns) return changed; } +namespace pqi { + struct ZeroedInt { ZeroedInt() { n=0 ;} int n ; }; +} + bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, const sockaddr_storage &addr) { // The algorithm is the following: @@ -1617,10 +1621,10 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons // Update a list of own IPs: // - remove old values for that same peer // - remove values for non connected peers - + { RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ - + mReportedOwnAddresses[from] = addr_filtered ; for(std::map::iterator it(mReportedOwnAddresses.begin());it!=mReportedOwnAddresses.end();) @@ -1641,8 +1645,8 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons std::cerr << "p3PeerMgr:: Current external address is calculated to be: " << sockaddr_storage_iptostring(current_best_ext_address_guess) << " (simultaneously reported by " << count << " peers)." << std::endl; } - - // now current + + // now current sockaddr_storage own_addr ; @@ -1665,7 +1669,7 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons RsServer::notify()->AddFeedItem(RS_FEED_ITEM_SEC_IP_WRONG_EXTERNAL_IP_REPORTED, from.toStdString(), sockaddr_storage_iptostring(own_addr), sockaddr_storage_iptostring(addr)); } - + // we could also sweep over all connected friends and see if some report a different address. return true ; @@ -1673,46 +1677,46 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons bool p3PeerMgrIMPL::locked_computeCurrentBestOwnExtAddressCandidate(sockaddr_storage& addr, uint32_t& count) { - std::map addr_counts ; - + std::map addr_counts ; + for(std::map::iterator it(mReportedOwnAddresses.begin());it!=mReportedOwnAddresses.end();++it) ++addr_counts[it->second].n ; #ifdef PEER_DEBUG std::cerr << "Current ext addr statistics:" << std::endl; #endif - + count = 0 ; - - for(std::map::const_iterator it(addr_counts.begin());it!=addr_counts.end();++it) + + for(std::map::const_iterator it(addr_counts.begin());it!=addr_counts.end();++it) { if(uint32_t(it->second.n) > count) { addr = it->first ; count = it->second.n ; } - + #ifdef PEER_DEBUG std::cerr << sockaddr_storage_iptostring(it->first) << " : " << it->second.n << std::endl; #endif } - + return true ; } - + bool p3PeerMgrIMPL::getExtAddressReportedByFriends(sockaddr_storage &addr, uint8_t& /*isstable*/) { RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ - + uint32_t count ; - + locked_computeCurrentBestOwnExtAddressCandidate(addr,count) ; - + #ifdef PEER_DEBUG std::cerr << "Estimation count = " << count << ". Trusted? = " << (count>=2) << std::endl; #endif - - return count >= 2 ;// 2 is not conservative enough. 3 should be probably better. + + return count >= 2 ;// 2 is not conservative enough. 3 should be probably better. } static bool cleanIpList(std::list& lst,const RsPeerId& pid,p3LinkMgr *link_mgr) @@ -1765,7 +1769,7 @@ bool p3PeerMgrIMPL::updateAddressList(const RsPeerId& id, const pqiIpAddrSet RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ /* check if it is our own ip */ - if (id == getOwnId()) + if (id == getOwnId()) { mOwnState.ipAddrs.updateAddrs(clean_set); return true; @@ -1811,11 +1815,11 @@ bool p3PeerMgrIMPL::updateCurrentAddress(const RsPeerId& id, const pqiIpAddre #ifdef PEER_DEBUG std::cerr << "p3PeerMgrIMPL::updateCurrentAddress() called for id : " << id << std::endl; #endif - + RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ - + /* cannot be own id */ - + /* check if it is a friend */ std::map::iterator it; if (mFriendList.end() == (it = mFriendList.find(id))) @@ -1846,23 +1850,23 @@ bool p3PeerMgrIMPL::updateCurrentAddress(const RsPeerId& id, const pqiIpAddre std::cerr << addrstr; std::cerr << std::endl; #endif - + IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ - + return true; } - + bool p3PeerMgrIMPL::updateLastContact(const RsPeerId& id) { #ifdef PEER_DEBUG std::cerr << "p3PeerMgrIMPL::updateLastContact() called for id : " << id << std::endl; #endif - + RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ - + /* cannot be own id */ - + /* check if it is a friend */ std::map::iterator it; if (mFriendList.end() == (it = mFriendList.find(id))) @@ -1877,7 +1881,7 @@ bool p3PeerMgrIMPL::updateLastContact(const RsPeerId& id) it->second.lastcontact = time(NULL); IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/ - + return true; } @@ -2061,7 +2065,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list& saveData) getProxyServerAddress(RS_HIDDEN_TYPE_TOR, proxy_addr_tor); getProxyServerAddress(RS_HIDDEN_TYPE_I2P, proxy_addr_i2p); - mPeerMtx.lock(); /****** MUTEX LOCKED *******/ + mPeerMtx.lock(); /****** MUTEX LOCKED *******/ RsPeerNetItem *item = new RsPeerNetItem(); item->clear(); @@ -2088,14 +2092,14 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list& saveData) item->vs_disc = mOwnState.vs_disc; item->vs_dht = mOwnState.vs_dht; - + item->lastContact = mOwnState.lastcontact; item->localAddrV4.addr = mOwnState.localaddr; item->extAddrV4.addr = mOwnState.serveraddr; sockaddr_storage_clear(item->localAddrV6.addr); sockaddr_storage_clear(item->extAddrV6.addr); - + item->dyndns = mOwnState.dyndns; mOwnState.ipAddrs.mLocal.loadTlv(item->localAddrList); mOwnState.ipAddrs.mExt.loadTlv(item->extAddrList); @@ -2125,20 +2129,20 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list& saveData) item->vs_dht = (it->second).vs_dht; item->lastContact = (it->second).lastcontact; - + item->localAddrV4.addr = (it->second).localaddr; item->extAddrV4.addr = (it->second).serveraddr; sockaddr_storage_clear(item->localAddrV6.addr); sockaddr_storage_clear(item->extAddrV6.addr); - - + + item->dyndns = (it->second).dyndns; (it->second).ipAddrs.mLocal.loadTlv(item->localAddrList); (it->second).ipAddrs.mExt.loadTlv(item->extAddrList); item->domain_addr = (it->second).hiddenDomain; item->domain_port = (it->second).hiddenPort; - + saveData.push_back(item); #ifdef PEER_DEBUG std::cerr << "p3PeerMgrIMPL::saveList() Peer Config Item:" << std::endl; @@ -2150,7 +2154,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list& saveData) RsPeerBandwidthLimitsItem *pblitem = new RsPeerBandwidthLimitsItem ; pblitem->peers = mPeerBandwidthLimits ; saveData.push_back(pblitem) ; - + RsPeerServicePermissionItem *sitem = new RsPeerServicePermissionItem ; for(std::map::const_iterator it(mFriendsPermissionFlags.begin());it!=mFriendsPermissionFlags.end();++it) @@ -2158,11 +2162,11 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list& saveData) sitem->pgp_ids.push_back(it->first) ; sitem->service_flags.push_back(it->second) ; } - + saveData.push_back(sitem) ; // Now save config for network digging strategies - + RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet ; RsTlvKeyValue kv; @@ -2199,7 +2203,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list& saveData) kv.key = kConfigKeyProxyServerPortI2P; kv.value = sockaddr_storage_porttostring(proxy_addr_i2p); vitem->tlvkvs.pairs.push_back(kv) ; - + saveData.push_back(vitem); /* save groups */ @@ -2213,7 +2217,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list& saveData) return true; } -bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& maxDn) +bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& maxDn) { RsPgpId pgp_id ; @@ -2235,7 +2239,7 @@ bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& ma return getMaxRates(pgp_id,maxUp,maxDn) ; } -bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& maxDn) +bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& maxDn) { RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ @@ -2254,25 +2258,25 @@ bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& max return false ; } } -bool p3PeerMgrIMPL::setMaxRates(const RsPgpId& pid,uint32_t maxUp,uint32_t maxDn) +bool p3PeerMgrIMPL::setMaxRates(const RsPgpId& pid,uint32_t maxUp,uint32_t maxDn) { RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ PeerBandwidthLimits& p(mPeerBandwidthLimits[pid]) ; - + if(maxUp == p.max_up_rate_kbs && maxDn == p.max_dl_rate_kbs) return true ; - + std::cerr << "Updating max rates for peer " << pid << " to " << maxUp << " kB/s (up), " << maxDn << " kB/s (dn)" << std::endl; - + p.max_up_rate_kbs = maxUp ; p.max_dl_rate_kbs = maxDn ; - + IndicateConfigChanged(); - + return true ; } - + void p3PeerMgrIMPL::saveDone() { /* clean up the save List */ @@ -2396,7 +2400,7 @@ bool p3PeerMgrIMPL::loadList(std::list& load) #ifdef PEER_DEBUG std::cerr << "setting use_extr_addr_finder to " << useExtAddrFinder << std::endl ; #endif - } + } // Tor else if (kit->key == kConfigKeyProxyServerIpAddrTor) { @@ -2859,13 +2863,13 @@ bool p3PeerMgrIMPL::removeAllFriendLocations(const RsPgpId &gpgid) { return false; } - + std::list::iterator it; for(it = sslIds.begin(); it != sslIds.end(); ++it) { removeFriend(*it, true); } - + return true; } @@ -2877,7 +2881,7 @@ bool p3PeerMgrIMPL::getAssociatedPeers(const RsPgpId &gpg_id, std::list::iterator it; for(it = mFriendList.begin(); it != mFriendList.end(); ++it) @@ -2890,10 +2894,10 @@ bool p3PeerMgrIMPL::getAssociatedPeers(const RsPgpId &gpg_id, std::listfirst << std::endl; #endif - + } } - + return (count > 0); } @@ -2930,10 +2934,10 @@ bool p3PeerMgrIMPL::removeBannedIps() return true ; } -// /* This only removes SSL certs, that are old... Can end up with no Certs per GPG Id +// /* This only removes SSL certs, that are old... Can end up with no Certs per GPG Id // * We are removing the concept of a "DummyId" - There is no need for it. // */ -// +// // bool isDummyFriend(RsPeerId id) // { // bool ret = (id.substr(0,5) == "dummy"); @@ -2974,7 +2978,7 @@ bool p3PeerMgrIMPL::removeUnusedLocations() { RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/ - + // First put a sensible number in all PGP ids for(std::list::const_iterator it = pgpList.begin(); it != pgpList.end(); ++it) @@ -2984,7 +2988,7 @@ bool p3PeerMgrIMPL::removeUnusedLocations() std::cerr << "p3PeerMgr::removeUnusedLocations()" << std::endl; #endif // Then compute the most recently used location for all PGP ids - + for( std::map::iterator it = mFriendList.begin(); it != mFriendList.end(); ++it) { time_t& bst(mostRecentTime[it->second.gpg_id]) ; @@ -3010,4 +3014,4 @@ bool p3PeerMgrIMPL::removeUnusedLocations() return true; } - + diff --git a/libretroshare/src/services/p3banlist.cc b/libretroshare/src/services/p3banlist.cc index b1fc07113..fdfcb05b5 100644 --- a/libretroshare/src/services/p3banlist.cc +++ b/libretroshare/src/services/p3banlist.cc @@ -55,7 +55,7 @@ #define RSBANLIST_DELAY_BETWEEN_TALK_TO_DHT 240 // every 4 mins. /************ IMPLEMENTATION NOTES ********************************* - * + * * Get Bad Peers passed to us (from DHT mainly). * we distribute and track the network list of bad peers. * @@ -113,6 +113,8 @@ void p3BanList::setAutoRangeLimit(int n) IndicateConfigChanged(); } +namespace services { + class ZeroedInt { public: @@ -120,6 +122,8 @@ class ZeroedInt uint32_t n ; }; +} + BanListPeer::BanListPeer() { memset(&addr, 0, sizeof(addr)); @@ -220,14 +224,14 @@ void p3BanList::autoFigureOutBanRanges() std::cerr << "Automatically figuring out IP ranges from banned IPs." << std::endl; #endif - std::map range_map ; + std::map range_map ; for(std::map::iterator it(mBanSet.begin());it!=mBanSet.end();++it) ++range_map[makeBitsRange(it->first,1)].n ; time_t now = time(NULL) ; - for(std::map::const_iterator it=range_map.begin();it!=range_map.end();++it) + for(std::map::const_iterator it=range_map.begin();it!=range_map.end();++it) { #ifdef DEBUG_BANLIST std::cerr << "Ban range: " << sockaddr_storage_iptostring(it->first) << " : " << it->second.n << std::endl; @@ -646,7 +650,7 @@ bool p3BanList::processIncoming() break; case RS_PKT_SUBTYPE_BANLIST_ITEM: { - // Order is important!. + // Order is important!. updated = (recvBanItem((RsBanListItem *) item) || updated); } break; @@ -669,8 +673,8 @@ bool p3BanList::processIncoming() } return true ; -} - +} + bool p3BanList::recvBanItem(RsBanListItem *item) { @@ -681,7 +685,7 @@ bool p3BanList::recvBanItem(RsBanListItem *item) for(it = item->peerList.mList.begin(); it != item->peerList.mList.end(); ++it) { - // Order is important!. + // Order is important!. updated = (addBanEntry(item->PeerId(), it->addr.addr, it->level, it->reason, now - it->age) || updated); } return updated; @@ -961,7 +965,7 @@ bool p3BanList::addBanEntry( const RsPeerId &peerId, it = mBanSources.find(peerId); updated = true; } - + // index is FAMILY + IP - the rest should be Zeros.. struct sockaddr_storage bannedaddr; sockaddr_storage_clear(bannedaddr); @@ -980,7 +984,7 @@ bool p3BanList::addBanEntry( const RsPeerId &peerId, blp.level = level; blp.mTs = time_stamp ; blp.masked_bytes = 0 ; - + it->second.mBanPeers[bannedaddr] = blp; it->second.mLastUpdate = now; updated = true; @@ -1036,7 +1040,7 @@ int p3BanList::condenseBanSources_locked() time_t now = time(NULL); RsPeerId ownId = mServiceCtrl->getOwnId(); - + #ifdef DEBUG_BANLIST std::cerr << "p3BanList::condenseBanSources_locked()"; std::cerr << std::endl; @@ -1062,7 +1066,7 @@ int p3BanList::condenseBanSources_locked() std::cerr << " Condensing Info from peer: " << it->first; std::cerr << std::endl; #endif - + std::map::const_iterator lit; for(lit = it->second.mBanPeers.begin(); lit != it->second.mBanPeers.end(); ++lit) { @@ -1135,7 +1139,7 @@ int p3BanList::condenseBanSources_locked() } } - + #ifdef DEBUG_BANLIST std::cerr << "p3BanList::condenseBanSources_locked() Printing New Set:"; std::cerr << std::endl; @@ -1280,7 +1284,7 @@ int p3BanList::printBanSet_locked(std::ostream &out) int p3BanList::printBanSources_locked(std::ostream &out) { time_t now = time(NULL); - + std::map::const_iterator it; for(it = mBanSources.begin(); it != mBanSources.end(); ++it) { diff --git a/retroshare-gui/src/gui/MainWindow.cpp b/retroshare-gui/src/gui/MainWindow.cpp index e00417080..ffa7a49fd 100644 --- a/retroshare-gui/src/gui/MainWindow.cpp +++ b/retroshare-gui/src/gui/MainWindow.cpp @@ -433,7 +433,7 @@ void MainWindow::initStackedPage() #ifndef RS_RELEASE_VERSION #ifdef PLUGINMGR - addPage(pluginsPage = new PluginsPage(ui->stackPages), grp, NULL); + addPage(pluginsPage = new gui::PluginsPage(ui->stackPages), grp, NULL); #endif #endif @@ -643,10 +643,10 @@ const QList &MainWindow::getUserNotifyList() /*static*/ void MainWindow::displayLobbySystrayMsg(const QString& title,const QString& msg) { - if (_instance == NULL) + if (_instance == NULL) return; - if(Settings->getDisplayTrayChatLobby()) + if(Settings->getDisplayTrayChatLobby()) _instance->displaySystrayMsg(title,msg) ; } @@ -1011,7 +1011,7 @@ void SetForegroundWindowInternal(HWND hWnd) return NULL; } - switch (page) + switch (page) { case Network: return _instance->friendsDialog->networkDialog; @@ -1457,7 +1457,7 @@ void MainWindow::externalLinkActivated(const QUrl &url) int res = mb.exec() ; - if (res == QMessageBox::No) + if (res == QMessageBox::No) return ; if(dontAsk_CB->isChecked()) diff --git a/retroshare-gui/src/gui/PluginsPage.h b/retroshare-gui/src/gui/PluginsPage.h index de90a3d8f..fac165396 100644 --- a/retroshare-gui/src/gui/PluginsPage.h +++ b/retroshare-gui/src/gui/PluginsPage.h @@ -15,7 +15,7 @@ * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. ****************************************************************/ @@ -39,6 +39,8 @@ class QScriptEngine; class PluginManager; +namespace gui { + //! A demo widget for showing plugin engine in action :) @@ -46,7 +48,7 @@ class PluginManager; //! loaded plugin widgets. All specific actions moved to //! PluginManagerWidget class. It contains a PluginManager instance, but it's //! supposed that in future a pluginManager will become a global variable -class PluginsPage : public MainPage +class PluginsPage : public MainPage { Q_OBJECT @@ -70,10 +72,10 @@ protected: QVBoxLayout* pluginPageLayout; QGroupBox* pluginPanel; QVBoxLayout* pluginPanelLayout; - + //! Plugin widgets will be loaded into this tabs QTabWidget* pluginTabs ; - + QVBoxLayout* pmLay; QFrame* pmFrame; QSpacerItem* pmSpacer; @@ -82,5 +84,7 @@ protected: PluginManager* pluginManager; }; +} // namespace gui + #endif diff --git a/retroshare-gui/src/gui/settings/PluginsPage.cpp b/retroshare-gui/src/gui/settings/PluginsPage.cpp index cdc326ec0..4be141c6e 100644 --- a/retroshare-gui/src/gui/settings/PluginsPage.cpp +++ b/retroshare-gui/src/gui/settings/PluginsPage.cpp @@ -33,7 +33,7 @@ #include "../MainWindow.h" -PluginsPage::PluginsPage(QWidget * parent, Qt::WindowFlags flags) +settings::PluginsPage::PluginsPage(QWidget * parent, Qt::WindowFlags flags) : ConfigPage(parent, flags) { ui.setupUi(this); @@ -123,7 +123,7 @@ PluginsPage::PluginsPage(QWidget * parent, Qt::WindowFlags flags) if(plugin == NULL || plugin->qt_config_panel() == NULL) item->_configure_PB->hide() ; - + if(plugin != NULL){ item->enableButton->hide(); @@ -159,7 +159,7 @@ PluginsPage::PluginsPage(QWidget * parent, Qt::WindowFlags flags) QObject::connect(ui.enableAll,SIGNAL(toggled(bool)),this,SLOT(toggleEnableAll(bool))) ; } -QString PluginsPage::helpText() const +QString settings::PluginsPage::helpText() const { return tr("

  Plugins

\

Plugins are loaded from the directories listed in the bottom list.

\ @@ -171,11 +171,11 @@ QString PluginsPage::helpText() const

If you want to develop your own plugins, contact the developpers team \ they will be happy to help you out!

") ; } -void PluginsPage::toggleEnableAll(bool b) +void settings::PluginsPage::toggleEnableAll(bool b) { rsPlugins->allowAllPlugins(b) ; } -void PluginsPage::aboutPlugin(int i) +void settings::PluginsPage::aboutPlugin(int i) { std::cerr << "Launching about window for plugin " << i << std::endl; @@ -183,7 +183,7 @@ void PluginsPage::aboutPlugin(int i) if(rsPlugins->plugin(i) != NULL && (dialog = rsPlugins->plugin(i)->qt_about_page()) != NULL) dialog->exec() ; } -void PluginsPage::configurePlugin(int i) +void settings::PluginsPage::configurePlugin(int i) { std::cerr << "Launching configuration window for plugin " << i << std::endl; @@ -191,14 +191,14 @@ void PluginsPage::configurePlugin(int i) rsPlugins->plugin(i)->qt_config_panel()->show() ; } -void PluginsPage::enablePlugin(const QString& hash) +void settings::PluginsPage::enablePlugin(const QString& hash) { std::cerr << "Switching status of plugin " << hash.toStdString() << " to enable" << std::endl; rsPlugins->enablePlugin(RsFileHash(hash.toStdString()) ); } -void PluginsPage::disablePlugin(const QString& hash) +void settings::PluginsPage::disablePlugin(const QString& hash) { std::cerr << "Switching status of plugin " << hash.toStdString() << " to disable " << std::endl; @@ -206,11 +206,11 @@ void PluginsPage::disablePlugin(const QString& hash) } -PluginsPage::~PluginsPage() +settings::PluginsPage::~PluginsPage() { } /** Loads the settings for this page */ -void PluginsPage::load() +void settings::PluginsPage::load() { } diff --git a/retroshare-gui/src/gui/settings/PluginsPage.h b/retroshare-gui/src/gui/settings/PluginsPage.h index 0d8f72dac..0151592af 100644 --- a/retroshare-gui/src/gui/settings/PluginsPage.h +++ b/retroshare-gui/src/gui/settings/PluginsPage.h @@ -24,6 +24,8 @@ #include #include "ui_PluginsPage.h" +namespace settings { + class PluginsPage : public ConfigPage { Q_OBJECT @@ -52,3 +54,4 @@ class PluginsPage : public ConfigPage Ui::PluginsPage ui; }; +} // namespace settings diff --git a/retroshare-gui/src/gui/settings/rsettingswin.cpp b/retroshare-gui/src/gui/settings/rsettingswin.cpp index 0a2b348f0..7f2dcf115 100644 --- a/retroshare-gui/src/gui/settings/rsettingswin.cpp +++ b/retroshare-gui/src/gui/settings/rsettingswin.cpp @@ -158,7 +158,7 @@ SettingsPage::initStackedWidget() addPage(new ForumPage()); // FORUMS addPage(new PostedPage()); // POSTED RENAME TO LINKS addPage(new NotifyPage()); // NOTIFY - addPage(new PluginsPage() ); // PLUGINS + addPage(new settings::PluginsPage() ); // PLUGINS addPage(new AppearancePage()); // APPEARENCE addPage(new SoundPage() ); // SOUND addPage(new ServicePermissionsPage() ); // PERMISSIONS From fd45d44826ff63b37d09986bf13db3e3b1c693ef Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 15 Jul 2018 11:14:58 +0200 Subject: [PATCH 74/84] added interface to get statistics about GxsNetTunnel --- libretroshare/src/gxs/rsgxsnettunnel.cc | 8 ++++ libretroshare/src/gxs/rsgxsnettunnel.h | 60 ++++--------------------- libretroshare/src/libretroshare.pro | 1 + libretroshare/src/rsserver/rsinit.cc | 2 + 4 files changed, 20 insertions(+), 51 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index fc8d5eb79..85fcec2d8 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -39,6 +39,8 @@ static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_DATA = 1; static const uint32_t RS_GXS_NET_TUNNEL_MAX_ALLOWED_HITS_GROUP_SEARCH = 100; +RsGxsDistSync *rsGxsDistSync = NULL; + RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") { #warning this is for testing only. In the final version this needs to be initialized with some random content, saved and re-used for a while (e.g. 1 month) @@ -1145,6 +1147,12 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl; } +void RsGxsNetTunnelService::getStatistics( std::map& groups,std::map& virtual_peers,Bias20Bytes& bias ) const +{ + groups = mGroups ; + virtual_peers = mVirtualPeers ; + bias = mRandomBias ; +} diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 0fdff3ccc..53cd42b96 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -27,7 +27,8 @@ #include -#include +#include "turtle/p3turtle.h" +#include "retroshare/rsgxsdistsync.h" /*! * \brief The RsGxsNetTunnelService class takes care of requesting tunnels to the turtle router, through which it is possible to sync @@ -102,59 +103,10 @@ // and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen. // -typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; - class RsGxsNetTunnelItem ; class RsNetworkExchangeService ; -struct RsGxsNetTunnelVirtualPeerInfo -{ - enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. - RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01, // tunnel has been established and we're waiting for virtual peer id - RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. - }; - - RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; } - virtual ~RsGxsNetTunnelVirtualPeerInfo(){} - - uint8_t vpid_status ; // status of the peer - time_t last_contact ; // last time some data was sent/recvd - uint8_t side ; // client/server - uint8_t encryption_master_key[32]; - - TurtleVirtualPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. - - RsGxsGroupId group_id ; // group that virtual peer is providing - uint16_t service_id ; // this is used for checkng consistency of the incoming data -}; - -struct RsGxsNetTunnelGroupInfo -{ - enum GroupStatus { - RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status - RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE = 0x01, // no virtual peers requested, just waiting - RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x02 // some virtual peers are available. Data can be read/written - }; - - enum GroupPolicy { - RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN = 0x00, // nothing has been set - RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE = 0x01, // group is available for server side tunnels, but does not explicitely request tunnels - RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group will only explicitely request tunnels if none available - RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING = 0x03, // group explicitely requests tunnels - }; - - RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {} - - GroupPolicy group_policy ; - GroupStatus group_status ; - time_t last_contact ; - TurtleFileHash hash ; - uint16_t service_id ; - - std::set virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. -}; - -class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread, public p3Config +class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread, public p3Config, public RsGxsDistSync { public: RsGxsNetTunnelService() ; @@ -257,6 +209,12 @@ public: bool saveList(bool& cleanup, std::list& save); bool loadList(std::list &load); + // Overloads RsGxsDistSync + + void getStatistics(std::map& groups, // groups on the client and server side + std::map& virtual_peers, // current virtual peers, which group they provide, and how to talk to them through turtle + Bias20Bytes& bias) const; + protected: // interaction with turtle router diff --git a/libretroshare/src/libretroshare.pro b/libretroshare/src/libretroshare.pro index 1b8018167..998f32b51 100644 --- a/libretroshare/src/libretroshare.pro +++ b/libretroshare/src/libretroshare.pro @@ -137,6 +137,7 @@ PUBLIC_HEADERS = retroshare/rsdisc.h \ retroshare/rsconfig.h \ retroshare/rsversion.h \ retroshare/rsservicecontrol.h \ + retroshare/rsgxsdistsync.h HEADERS += plugins/pluginmanager.h \ plugins/dlfcn_win32.h \ diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 8ec0d7e8c..877c45488 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -62,6 +62,7 @@ #include #include "gxstunnel/p3gxstunnel.h" +#include "retroshare/rsgxsdistsync.h" #include "file_sharing/p3filelists.h" #define ENABLE_GROUTER @@ -1283,6 +1284,7 @@ int RsServer::StartupRetroShare() #ifdef RS_USE_GXS_DISTANT_SYNC RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ; + rsGxsDistSync = mGxsNetTunnel ; #else RsGxsNetTunnelService *mGxsNetTunnel = NULL ; #endif From 34e924f99bc5359ef22dccc37abbdc21315d8918 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 15 Jul 2018 14:40:50 +0200 Subject: [PATCH 75/84] added visualisation for GXS net tunnels (unfinished) --- .../src/gui/statistics/TurtleRouterDialog.cpp | 250 ++++++++++++++---- .../src/gui/statistics/TurtleRouterDialog.h | 44 ++- .../gui/statistics/TurtleRouterStatistics.cpp | 10 +- 3 files changed, 242 insertions(+), 62 deletions(-) diff --git a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp index 7572bdfae..f7d10374e 100644 --- a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp +++ b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp @@ -2,6 +2,9 @@ #include #include #include +#include +#include +#include #include "TurtleRouterDialog.h" #include #include @@ -240,19 +243,14 @@ QTreeWidgetItem *TurtleRouterDialog::findParentHashItem(const std::string& hash) else return items.front() ; } + //=======================================================================================================================// - -GxsTunnelsDialog::GxsTunnelsDialog(QWidget *parent) +TunnelStatisticsDialog::TunnelStatisticsDialog(QWidget *parent) : RsAutoUpdatePage(2000,parent) { -// setupUi(this) ; - m_bProcessSettings = false; - //float fontHeight = QFontMetricsF(font()).height(); - //float fact = fontHeight/14.0; - maxWidth = 200 ; maxHeight = 200 ; @@ -260,14 +258,13 @@ GxsTunnelsDialog::GxsTunnelsDialog(QWidget *parent) processSettings(true); } -GxsTunnelsDialog::~GxsTunnelsDialog() +TunnelStatisticsDialog::~TunnelStatisticsDialog() { - // save settings processSettings(false); } -void GxsTunnelsDialog::processSettings(bool bLoad) +void TunnelStatisticsDialog::processSettings(bool bLoad) { m_bProcessSettings = true; @@ -284,7 +281,77 @@ void GxsTunnelsDialog::processSettings(bool bLoad) m_bProcessSettings = false; } -void GxsTunnelsDialog::updateDisplay() +QString TunnelStatisticsDialog::getPeerName(const RsPeerId &peer_id) +{ + static std::map names ; + + std::map::const_iterator it = names.find(peer_id) ; + + if( it != names.end()) + return it->second ; + else + { + RsPeerDetails detail ; + if(!rsPeers->getPeerDetails(peer_id,detail)) + return tr("Unknown Peer"); + + return (names[peer_id] = QString::fromUtf8(detail.name.c_str())) ; + } +} + +QString TunnelStatisticsDialog::getPeerName(const RsGxsId& gxs_id) +{ + static std::map names ; + + std::map::const_iterator it = names.find(gxs_id) ; + + if( it != names.end()) + return it->second ; + else + { + RsIdentityDetails detail ; + + if(!rsIdentity->getIdDetails(gxs_id,detail)) + return tr("Unknown Peer"); + + return (names[gxs_id] = QString::fromUtf8(detail.mNickname.c_str())) ; + } +} + + +QString TunnelStatisticsDialog::speedString(float f) +{ + if(f < 1.0f) + return QString("0 B/s") ; + if(f < 1024.0f) + return QString::number((int)f)+" B/s" ; + + return QString::number(f/1024.0,'f',2) + " KB/s"; +} + +void TunnelStatisticsDialog::paintEvent(QPaintEvent */*event*/) +{ + QStylePainter(this).drawPixmap(0, 0, pixmap); +} + +void TunnelStatisticsDialog::resizeEvent(QResizeEvent *event) +{ + QRect TaskGraphRect = geometry(); + + maxWidth = TaskGraphRect.width(); + maxHeight = TaskGraphRect.height() ; + + QWidget::resizeEvent(event); + update(); +} +//=======================================================================================================================// + +GxsAuthenticatedTunnelsDialog::GxsAuthenticatedTunnelsDialog(QWidget *parent) + : TunnelStatisticsDialog(parent) +{ +} + +void GxsAuthenticatedTunnelsDialog::updateDisplay() { // Request info about ongoing tunnels @@ -334,8 +401,8 @@ void GxsTunnelsDialog::updateDisplay() // draw... painter.drawText(ox+4*cellx,oy+celly,tr("Tunnel ID: %1").arg(QString::fromStdString(tunnel_infos[i].tunnel_id.toStdString()))) ; oy += celly ; - painter.drawText(ox+6*cellx,oy+celly,tr("from: %1").arg(QString::fromStdString(tunnel_infos[i].source_gxs_id.toStdString()))) ; oy += celly ; - painter.drawText(ox+6*cellx,oy+celly,tr("to: %1").arg(QString::fromStdString(tunnel_infos[i].destination_gxs_id.toStdString()))) ; oy += celly ; + painter.drawText(ox+6*cellx,oy+celly,tr("from: %1 (%2)").arg(QString::fromStdString(tunnel_infos[i].source_gxs_id.toStdString())).arg(getPeerName(tunnel_infos[i].source_gxs_id))) ; oy += celly ; + painter.drawText(ox+6*cellx,oy+celly,tr("to: %1 (%2)").arg(QString::fromStdString(tunnel_infos[i].destination_gxs_id.toStdString())).arg(getPeerName(tunnel_infos[i].destination_gxs_id))) ; oy += celly ; painter.drawText(ox+6*cellx,oy+celly,tr("status: %1").arg(QString::number(tunnel_infos[i].tunnel_status))) ; oy += celly ; painter.drawText(ox+6*cellx,oy+celly,tr("total sent: %1 bytes").arg(QString::number(tunnel_infos[i].total_size_sent))) ; oy += celly ; painter.drawText(ox+6*cellx,oy+celly,tr("total recv: %1 bytes").arg(QString::number(tunnel_infos[i].total_size_received))) ; oy += celly ; @@ -350,46 +417,131 @@ void GxsTunnelsDialog::updateDisplay() maxHeight = std::max(oy,10*celly); } -QString GxsTunnelsDialog::getPeerName(const RsPeerId &peer_id) +//=======================================================================================================================// + +GxsNetTunnelsDialog::GxsNetTunnelsDialog(QWidget *parent) + : TunnelStatisticsDialog(parent) { - static std::map names ; +} - std::map::const_iterator it = names.find(peer_id) ; +static QString getGroupStatusString(RsGxsNetTunnelGroupInfo::GroupStatus group_status) +{ + switch(group_status) + { + default: + case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN : return QObject::tr("Unknown") ; + case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE : return QObject::tr("Idle"); + case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE : return QObject::tr("Virtual peers available"); + } + return QString(); +} - if( it != names.end()) - return it->second ; - else +static QString getGroupPolicyString(RsGxsNetTunnelGroupInfo::GroupPolicy group_policy) +{ + switch(group_policy) + { + default: + case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN : return QObject::tr("Unknown") ; + case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE : return QObject::tr("Passive") ; + case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE : return QObject::tr("Active") ; + case RsGxsNetTunnelGroupInfo::RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING : return QObject::tr("Requesting peers") ; + } + return QString(); +} + +static QString getLastContactString(time_t last_contact) +{ + time_t now = time(NULL); + + if(last_contact == 0) + return QObject::tr("Never"); + + return QString::number(now - last_contact) + " secs ago" ; +} + +static QString getServiceNameString(uint16_t service_id) +{ + static RsPeerServiceInfo ownServices; + + if(ownServices.mServiceList.find(service_id) == ownServices.mServiceList.end()) + rsServiceControl->getOwnServices(ownServices); + + return QString::fromUtf8(ownServices.mServiceList[service_id].mServiceName.c_str()) ; +} + +void GxsNetTunnelsDialog::updateDisplay() +{ + // Request info about ongoing tunnels + + std::map groups; // groups on the client and server side + std::map virtual_peers; // current virtual peers, which group they provide, and how to talk to them through turtle + Bias20Bytes bias; + + rsGxsDistSync->getStatistics(groups,virtual_peers,bias) ; + + // RsGxsNetTunnelGroupInfo: + // + // enum GroupStatus { + // RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status + // RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE = 0x01, // no virtual peers requested, just waiting + // RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x02 // some virtual peers are available. Data can be read/written + // }; + // enum GroupPolicy { + // RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN = 0x00, // nothing has been set + // RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE = 0x01, // group is available for server side tunnels, but does not explicitely request tunnels + // RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group will only explicitely request tunnels if none available + // RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING = 0x03, // group explicitely requests tunnels + // }; + // + // RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {} + // + // GroupPolicy group_policy ; + // GroupStatus group_status ; + // time_t last_contact ; + // RsFileHash hash ; + // uint16_t service_id ; + // std::set virtual_peers ; + + // now draw the shit + QPixmap tmppixmap(maxWidth, maxHeight); + tmppixmap.fill(Qt::transparent); + //setFixedHeight(maxHeight); + + QPainter painter(&tmppixmap); + painter.initFrom(this); + + // extracts the height of the fonts in pixels. This is used to calibrate the size of the objects to draw. + + float fontHeight = QFontMetricsF(font()).height(); + float fact = fontHeight/14.0; + + int cellx = 6*fact ; + int celly = (10+4)*fact ; + int ox=5*fact,oy=5*fact ; + + painter.setPen(QColor::fromRgb(0,0,0)) ; + painter.drawText(ox+2*cellx,oy+celly,tr("GXS Groups:")) ; oy += celly ; + + for(auto it(groups.begin());it!=groups.end();++it) { - RsPeerDetails detail ; - if(!rsPeers->getPeerDetails(peer_id,detail)) - return tr("Unknown Peer"); + // std::cerr << "Drawing into pixmap of size " << maxWidth << "x" << maxHeight << std::endl; + // draw... - return (names[peer_id] = QString::fromUtf8(detail.name.c_str())) ; + painter.drawText(ox+4*cellx,oy+celly,tr("Service: %1 (%2) - Group ID: %3,\t policy: %4, \tstatus: %5, \tlast contact: %6, \t%7 virtual peers.") + .arg(QString::number(it->second.service_id)) + .arg(getServiceNameString(it->second.service_id)) + .arg(QString::fromStdString(it->first.toStdString())) + .arg(getGroupPolicyString(it->second.group_policy)) + .arg(getGroupStatusString(it->second.group_status)) + .arg(getLastContactString(it->second.last_contact)) + .arg(QString::number(it->second.virtual_peers.size())) + ) ; oy += celly ; } -} - -QString GxsTunnelsDialog::speedString(float f) -{ - if(f < 1.0f) - return QString("0 B/s") ; - if(f < 1024.0f) - return QString::number((int)f)+" B/s" ; - - return QString::number(f/1024.0,'f',2) + " KB/s"; -} - -void GxsTunnelsDialog::paintEvent(QPaintEvent */*event*/) -{ - QStylePainter(this).drawPixmap(0, 0, pixmap); -} - -void GxsTunnelsDialog::resizeEvent(QResizeEvent *event) -{ - QRect TaskGraphRect = geometry(); - - maxWidth = TaskGraphRect.width(); - maxHeight = TaskGraphRect.height() ; - - QWidget::resizeEvent(event); - update(); + + painter.drawText(ox+2*cellx,oy+celly,tr("Virtual peers:")) ; oy += celly ; + + // update the pixmap + // + pixmap = tmppixmap; + maxHeight = std::max(oy,10*celly); } diff --git a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.h b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.h index 22bc6dd56..d51e69c68 100644 --- a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.h +++ b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.h @@ -37,29 +37,55 @@ class TurtleRouterDialog: public RsAutoUpdatePage, public Ui::TurtleRouterDialog } ; -class GxsTunnelsDialog: public RsAutoUpdatePage +class TunnelStatisticsDialog: public RsAutoUpdatePage { Q_OBJECT public: - GxsTunnelsDialog(QWidget *parent = NULL) ; - ~GxsTunnelsDialog(); + TunnelStatisticsDialog(QWidget *parent = NULL) ; + ~TunnelStatisticsDialog(); // Cache for peer names. static QString getPeerName(const RsPeerId &peer_id) ; + static QString getPeerName(const RsGxsId& gxs_id); protected: virtual void paintEvent(QPaintEvent *); virtual void resizeEvent(QResizeEvent *event); + + int maxWidth ; + int maxHeight ; + + QPixmap pixmap; + private: void processSettings(bool bLoad); bool m_bProcessSettings; static QString speedString(float f); - virtual void updateDisplay() ; - - int maxWidth ; - int maxHeight ; - - QPixmap pixmap; + virtual void updateDisplay() =0; +} ; + +class GxsAuthenticatedTunnelsDialog: public TunnelStatisticsDialog +{ + Q_OBJECT + +public: + GxsAuthenticatedTunnelsDialog(QWidget *parent = NULL) ; + ~GxsAuthenticatedTunnelsDialog() {} + +private: + virtual void updateDisplay() ; +} ; + +class GxsNetTunnelsDialog: public TunnelStatisticsDialog +{ + Q_OBJECT + +public: + GxsNetTunnelsDialog(QWidget *parent = NULL) ; + ~GxsNetTunnelsDialog() {} + +private: + virtual void updateDisplay() ; } ; diff --git a/retroshare-gui/src/gui/statistics/TurtleRouterStatistics.cpp b/retroshare-gui/src/gui/statistics/TurtleRouterStatistics.cpp index d3e912bbf..ce983f8b5 100644 --- a/retroshare-gui/src/gui/statistics/TurtleRouterStatistics.cpp +++ b/retroshare-gui/src/gui/statistics/TurtleRouterStatistics.cpp @@ -195,11 +195,13 @@ TurtleRouterStatistics::TurtleRouterStatistics(QWidget *parent) _tunnel_statistics_F->setFrameStyle(QFrame::NoFrame); _tunnel_statistics_F->setFocusPolicy(Qt::NoFocus); - routertabWidget->addTab(new TurtleRouterDialog(),QString(tr("Anonymous tunnels"))); - routertabWidget->addTab(new GxsTunnelsDialog(),QString(tr("Authenticated tunnels"))); + routertabWidget->addTab(new TurtleRouterDialog(), QString(tr("File transfer tunnels"))); + routertabWidget->addTab(new GxsAuthenticatedTunnelsDialog(),QString(tr("Authenticated tunnels"))); + routertabWidget->addTab(new GxsNetTunnelsDialog(), QString(tr("GXS sync tunnels") )); + + float fontHeight = QFontMetricsF(font()).height(); + float fact = fontHeight/14.0; - float fontHeight = QFontMetricsF(font()).height(); - float fact = fontHeight/14.0; frmGraph->setMinimumHeight(200*fact); // load settings From cca986ad75f90aa30f7977ca8f157f00adb94f9e Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 15 Jul 2018 15:07:24 +0200 Subject: [PATCH 76/84] added more info to tunnel display --- .../src/gui/statistics/TurtleRouterDialog.cpp | 56 +++++++++++++++++-- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp index f7d10374e..fa849c1a8 100644 --- a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp +++ b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp @@ -469,6 +469,23 @@ static QString getServiceNameString(uint16_t service_id) return QString::fromUtf8(ownServices.mServiceList[service_id].mServiceName.c_str()) ; } +static QString getVirtualPeerStatusString(uint8_t status) +{ + switch(status) + { + default: + case RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN : return QObject::tr("Unknown") ; + case RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK : return QObject::tr("Tunnel OK") ; + case RsGxsNetTunnelVirtualPeerInfo::RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE : return QObject::tr("Tunnel active") ; + } + return QString(); +} + +static QString getMasterKeyString(uint8_t *key) +{ + return QString(); +} + void GxsNetTunnelsDialog::updateDisplay() { // Request info about ongoing tunnels @@ -523,10 +540,6 @@ void GxsNetTunnelsDialog::updateDisplay() painter.drawText(ox+2*cellx,oy+celly,tr("GXS Groups:")) ; oy += celly ; for(auto it(groups.begin());it!=groups.end();++it) - { - // std::cerr << "Drawing into pixmap of size " << maxWidth << "x" << maxHeight << std::endl; - // draw... - painter.drawText(ox+4*cellx,oy+celly,tr("Service: %1 (%2) - Group ID: %3,\t policy: %4, \tstatus: %5, \tlast contact: %6, \t%7 virtual peers.") .arg(QString::number(it->second.service_id)) .arg(getServiceNameString(it->second.service_id)) @@ -535,11 +548,42 @@ void GxsNetTunnelsDialog::updateDisplay() .arg(getGroupStatusString(it->second.group_status)) .arg(getLastContactString(it->second.last_contact)) .arg(QString::number(it->second.virtual_peers.size())) - ) ; oy += celly ; - } + ),oy+=celly ; + + oy += celly ; + + // struct RsGxsNetTunnelVirtualPeerInfo: + // + // enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. + // RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01, // tunnel has been established and we're waiting for virtual peer id + // RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. + // }; + // + // RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; } + // virtual ~RsGxsNetTunnelVirtualPeerInfo(){} + // + // uint8_t vpid_status ; // status of the peer + // time_t last_contact ; // last time some data was sent/recvd + // uint8_t side ; // client/server + // uint8_t encryption_master_key[32]; + // + // RsPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. + // + // RsGxsGroupId group_id ; // group that virtual peer is providing + // uint16_t service_id ; // this is used for checkng consistency of the incoming data painter.drawText(ox+2*cellx,oy+celly,tr("Virtual peers:")) ; oy += celly ; + for(auto it(virtual_peers.begin());it!=virtual_peers.end();++it) + painter.drawText(ox+4*cellx,oy+celly,tr("Peer: %1 - Group ID: %2 (service %3),\t status: %4, \tlast contact: %5, \tside %6 \tMaster key: %7.") + .arg(QString::fromStdString(it->first.toStdString())) + .arg(QString::fromStdString(it->second.group_id.toStdString())) + .arg(getServiceNameString(it->second.service_id)) + .arg(getVirtualPeerStatusString(it->second.vpid_status)) + .arg(getLastContactString(it->second.last_contact)) + .arg(getMasterKeyString(it->second.encryption_master_key)) + ),oy+=celly ; + // update the pixmap // pixmap = tmppixmap; From 820841668e72017212cc12a7631e833b5d3a8b3b Mon Sep 17 00:00:00 2001 From: Phenom Date: Sun, 15 Jul 2018 16:37:33 +0200 Subject: [PATCH 77/84] Change Lobby text to Room in CreateLobbyDialog. --- .../src/gui/chat/CreateLobbyDialog.cpp | 2 +- .../src/gui/chat/CreateLobbyDialog.ui | 125 ++++++------------ .../src/gui/qss/stylesheet/Standard.qss | 2 +- 3 files changed, 44 insertions(+), 85 deletions(-) diff --git a/retroshare-gui/src/gui/chat/CreateLobbyDialog.cpp b/retroshare-gui/src/gui/chat/CreateLobbyDialog.cpp index a9871ab77..b0e5da106 100644 --- a/retroshare-gui/src/gui/chat/CreateLobbyDialog.cpp +++ b/retroshare-gui/src/gui/chat/CreateLobbyDialog.cpp @@ -106,7 +106,7 @@ void CreateLobbyDialog::checkTextFields() break ; } - RsIdentityDetails(idd) ; + RsIdentityDetails idd; rsIdentity->getIdDetails(id,idd) ; diff --git a/retroshare-gui/src/gui/chat/CreateLobbyDialog.ui b/retroshare-gui/src/gui/chat/CreateLobbyDialog.ui index 1d2340e08..77aab56ea 100644 --- a/retroshare-gui/src/gui/chat/CreateLobbyDialog.ui +++ b/retroshare-gui/src/gui/chat/CreateLobbyDialog.ui @@ -11,17 +11,26 @@ - Create Chat Lobby + Create Chat Room :/images/logo/logo_32.png:/images/logo/logo_32.png - + 0 - + + 0 + + + 0 + + + 0 + + 0 @@ -29,11 +38,11 @@ - + - A chat lobby is a decentralized and anonymous chat group. All participants receive all messages. Once the lobby is created you can invite other friends from the Friends tab. + A chat room is a decentralized and anonymous chat group. All participants receive all messages. Once the room is created you can invite other friend nodes with invite button on top right. true @@ -41,16 +50,16 @@ - + - - + + - + - + - Lobby name: + Room name: Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter @@ -61,9 +70,9 @@ - + - Lobby topic: + Room topic: Qt::AlignRight|Qt::AlignTrailing|Qt::AlignVCenter @@ -74,7 +83,7 @@ - + Identity to use: @@ -84,7 +93,7 @@ - + Visibility: @@ -113,7 +122,7 @@ - <html><head/><body><p>If you check this, only PGP-signed ids can be used to join and talk in this lobby. This limitation prevents anonymous spamming as it becomes possible for at least some people in the lobby to locate the spammer's node.</p></body></html> + <html><head/><body><p>If you check this, only PGP-signed ids can be used to join and talk in this room. This limitation prevents anonymous spamming as it becomes possible for at least some people in the room to locate the spammer's node.</p></body></html> require PGP-signed identities @@ -121,7 +130,7 @@ - + Qt::LeftToRight @@ -136,7 +145,7 @@ - + Qt::Vertical @@ -152,34 +161,37 @@ - + - + Select the Friends with which you want to group chat. - - - true - + - + 0 - 0 + 4 + + + 20 + 0 + + - 52487 - 524287 + 1677215 + 16777215 - 220 + 0 0 @@ -189,59 +201,6 @@ 0 - - false - - - QDockWidget::NoDockWidgetFeatures - - - Invited friends - - - - - 0 - - - 0 - - - - - - 0 - 4 - - - - - 20 - 0 - - - - - 1677215 - 16777215 - - - - - 0 - 0 - - - - - 0 - 0 - - - - - - @@ -249,7 +208,7 @@ - + QLayout::SetDefaultConstraint diff --git a/retroshare-gui/src/gui/qss/stylesheet/Standard.qss b/retroshare-gui/src/gui/qss/stylesheet/Standard.qss index 7d518d4c2..2996d4ee5 100644 --- a/retroshare-gui/src/gui/qss/stylesheet/Standard.qss +++ b/retroshare-gui/src/gui/qss/stylesheet/Standard.qss @@ -186,7 +186,7 @@ ChatLobbyDialog QListWidget#participantsList { background: white; } -CreateLobbyDialog QFrame#lobbyFrame { +CreateLobbyDialog QFrame#roomFrame { border: 2px solid #CCCCCC; border-radius:6px; background: white; From 1de31493a9ea5823d8c990cdda85b217ea00ee40 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 15 Jul 2018 19:09:12 +0200 Subject: [PATCH 78/84] fixed up display of GXS net tunnel info --- libretroshare/src/gxs/rsgxsnettunnel.cc | 3 +- libretroshare/src/gxs/rsgxsnettunnel.h | 1 + libretroshare/src/retroshare/rsgxsdistsync.h | 92 +++++++++++++++++ libretroshare/src/retroshare/rsturtle.h | 1 + .../src/gui/statistics/TurtleRouterDialog.cpp | 98 ++++++++++++------- 5 files changed, 156 insertions(+), 39 deletions(-) create mode 100644 libretroshare/src/retroshare/rsgxsdistsync.h diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 85fcec2d8..1e04c6621 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -1147,10 +1147,11 @@ void RsGxsNetTunnelService::receiveSearchResult(TurtleSearchRequestId request_id GXS_NET_TUNNEL_ERROR() << ": deserialized item is of unknown type. Dropping!" << std::endl; } -void RsGxsNetTunnelService::getStatistics( std::map& groups,std::map& virtual_peers,Bias20Bytes& bias ) const +void RsGxsNetTunnelService::getStatistics(std::map& groups, std::map& virtual_peers, std::map &turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias ) const { groups = mGroups ; virtual_peers = mVirtualPeers ; + turtle_vpid_to_net_tunnel_vpid = mTurtle2GxsPeer; bias = mRandomBias ; } diff --git a/libretroshare/src/gxs/rsgxsnettunnel.h b/libretroshare/src/gxs/rsgxsnettunnel.h index 53cd42b96..cbac109a9 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.h +++ b/libretroshare/src/gxs/rsgxsnettunnel.h @@ -213,6 +213,7 @@ public: void getStatistics(std::map& groups, // groups on the client and server side std::map& virtual_peers, // current virtual peers, which group they provide, and how to talk to them through turtle + std::map& turtle_vpid_to_net_tunnel_vpid, Bias20Bytes& bias) const; protected: diff --git a/libretroshare/src/retroshare/rsgxsdistsync.h b/libretroshare/src/retroshare/rsgxsdistsync.h new file mode 100644 index 000000000..df597f03a --- /dev/null +++ b/libretroshare/src/retroshare/rsgxsdistsync.h @@ -0,0 +1,92 @@ +/* + * RetroShare C++ Interface. + * + * Copyright 2018 by Cyril Soler + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Library General Public + * License Version 2 as published by the Free Software Foundation. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Library General Public License for more details. + * + * You should have received a copy of the GNU Library General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA. + * + * Please report all bugs and problems to "retroshare@lunamutt.com". + * + */ + +#pragma once + +#include "retroshare/rsfiles.h" +#include "retroshare/rsturtle.h" + +typedef RsPeerId RsGxsNetTunnelVirtualPeerId ; + +struct RsGxsNetTunnelVirtualPeerInfo +{ + enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. + RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01, // tunnel has been established and we're waiting for virtual peer id + RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. + }; + + RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; } + virtual ~RsGxsNetTunnelVirtualPeerInfo(){} + + uint8_t vpid_status ; // status of the peer + time_t last_contact ; // last time some data was sent/recvd + uint8_t side ; // client/server + uint8_t encryption_master_key[32]; + + TurtleVirtualPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. + + RsGxsGroupId group_id ; // group that virtual peer is providing + uint16_t service_id ; // this is used for checkng consistency of the incoming data +}; + +struct RsGxsNetTunnelGroupInfo +{ + enum GroupStatus { + RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status + RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE = 0x01, // no virtual peers requested, just waiting + RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x02 // some virtual peers are available. Data can be read/written + }; + + enum GroupPolicy { + RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN = 0x00, // nothing has been set + RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE = 0x01, // group is available for server side tunnels, but does not explicitely request tunnels + RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group will only explicitely request tunnels if none available + RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING = 0x03, // group explicitely requests tunnels + }; + + RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {} + + GroupPolicy group_policy ; + GroupStatus group_status ; + time_t last_contact ; + RsFileHash hash ; + uint16_t service_id ; + + std::set virtual_peers ; // list of which virtual peers provide this group. Can me more than 1. +}; + +// This class is here to provide statistics about GXS dist sync internals. It +// +class RsGxsDistSync +{ + public: + virtual void getStatistics( + std::map& groups, // groups on the client and server side + std::map& virtual_peers, // current virtual peers, which group they provide, and how to talk to them through turtle + std::map& turtle_vpid_to_net_tunnel_vpid, + Bias20Bytes& bias + ) const =0; +}; + +extern RsGxsDistSync *rsGxsDistSync ; + diff --git a/libretroshare/src/retroshare/rsturtle.h b/libretroshare/src/retroshare/rsturtle.h index 1b3cb1b7e..4130a9bea 100644 --- a/libretroshare/src/retroshare/rsturtle.h +++ b/libretroshare/src/retroshare/rsturtle.h @@ -44,6 +44,7 @@ class RsTurtle; extern RsTurtle *rsTurtle ; typedef uint32_t TurtleRequestId ; +typedef RsPeerId TurtleVirtualPeerId; // This is the structure used to send back results of the turtle search // to the notifyBase class, or send info to the GUI. diff --git a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp index fa849c1a8..bbabb8fd6 100644 --- a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp +++ b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -481,9 +482,14 @@ static QString getVirtualPeerStatusString(uint8_t status) return QString(); } +static QString getSideString(uint8_t side) +{ + return side?QObject::tr("Client"):QObject::tr("Server") ; +} + static QString getMasterKeyString(uint8_t *key) { - return QString(); + return QString::fromStdString(RsUtil::BinToHex(key,10)); } void GxsNetTunnelsDialog::updateDisplay() @@ -491,10 +497,11 @@ void GxsNetTunnelsDialog::updateDisplay() // Request info about ongoing tunnels std::map groups; // groups on the client and server side + std::map turtle2gxsnettunnel; // convertion table from turtle to net tunnel virtual peer id std::map virtual_peers; // current virtual peers, which group they provide, and how to talk to them through turtle Bias20Bytes bias; - rsGxsDistSync->getStatistics(groups,virtual_peers,bias) ; + rsGxsDistSync->getStatistics(groups,virtual_peers,turtle2gxsnettunnel,bias) ; // RsGxsNetTunnelGroupInfo: // @@ -518,6 +525,29 @@ void GxsNetTunnelsDialog::updateDisplay() // RsFileHash hash ; // uint16_t service_id ; // std::set virtual_peers ; + // + // struct RsGxsNetTunnelVirtualPeerInfo: + // + // enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. + // RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01, // tunnel has been established and we're waiting for virtual peer id + // RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. + // }; + // + // RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; } + // virtual ~RsGxsNetTunnelVirtualPeerInfo(){} + // + // uint8_t vpid_status ; // status of the peer + // time_t last_contact ; // last time some data was sent/recvd + // uint8_t side ; // client/server + // uint8_t encryption_master_key[32]; + // + // RsPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. + // + // RsGxsGroupId group_id ; // group that virtual peer is providing + // uint16_t service_id ; // this is used for checkng consistency of the incoming data + + // update the pixmap + // // now draw the shit QPixmap tmppixmap(maxWidth, maxHeight); @@ -540,52 +570,44 @@ void GxsNetTunnelsDialog::updateDisplay() painter.drawText(ox+2*cellx,oy+celly,tr("GXS Groups:")) ; oy += celly ; for(auto it(groups.begin());it!=groups.end();++it) - painter.drawText(ox+4*cellx,oy+celly,tr("Service: %1 (%2) - Group ID: %3,\t policy: %4, \tstatus: %5, \tlast contact: %6, \t%7 virtual peers.") + { + painter.drawText(ox+4*cellx,oy+celly,tr("Service: %1 (%2) - Group ID: %3,\t policy: %4, \tstatus: %5, \tlast contact: %6") .arg(QString::number(it->second.service_id)) .arg(getServiceNameString(it->second.service_id)) .arg(QString::fromStdString(it->first.toStdString())) .arg(getGroupPolicyString(it->second.group_policy)) .arg(getGroupStatusString(it->second.group_status)) .arg(getLastContactString(it->second.last_contact)) - .arg(QString::number(it->second.virtual_peers.size())) ),oy+=celly ; + + for(auto it2(it->second.virtual_peers.begin());it2!=it->second.virtual_peers.end();++it2) + { + auto it4 = turtle2gxsnettunnel.find(*it2) ; + + if(it4 != turtle2gxsnettunnel.end()) + { + auto it3 = virtual_peers.find(it4->second) ; + + if(virtual_peers.end() != it3) + painter.drawText(ox+6*cellx,oy+celly,tr("Peer: %1:\tstatus: %2/%3, \tlast contact: %4, \tMaster key: %5.") + .arg(QString::fromStdString((*it2).toStdString())) + .arg(getVirtualPeerStatusString(it3->second.vpid_status)) + .arg(getSideString(it3->second.side)) + .arg(getLastContactString(it3->second.last_contact)) + .arg(getMasterKeyString(it3->second.encryption_master_key)) + ),oy+=celly ; + } + else + painter.drawText(ox+6*cellx,oy+celly,tr("Peer: %1: no information available") + .arg(QString::fromStdString((*it2).toStdString())) + ),oy+=celly; + + } + } + oy += celly ; - // struct RsGxsNetTunnelVirtualPeerInfo: - // - // enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status. - // RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01, // tunnel has been established and we're waiting for virtual peer id - // RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer. - // }; - // - // RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; } - // virtual ~RsGxsNetTunnelVirtualPeerInfo(){} - // - // uint8_t vpid_status ; // status of the peer - // time_t last_contact ; // last time some data was sent/recvd - // uint8_t side ; // client/server - // uint8_t encryption_master_key[32]; - // - // RsPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid. - // - // RsGxsGroupId group_id ; // group that virtual peer is providing - // uint16_t service_id ; // this is used for checkng consistency of the incoming data - - painter.drawText(ox+2*cellx,oy+celly,tr("Virtual peers:")) ; oy += celly ; - - for(auto it(virtual_peers.begin());it!=virtual_peers.end();++it) - painter.drawText(ox+4*cellx,oy+celly,tr("Peer: %1 - Group ID: %2 (service %3),\t status: %4, \tlast contact: %5, \tside %6 \tMaster key: %7.") - .arg(QString::fromStdString(it->first.toStdString())) - .arg(QString::fromStdString(it->second.group_id.toStdString())) - .arg(getServiceNameString(it->second.service_id)) - .arg(getVirtualPeerStatusString(it->second.vpid_status)) - .arg(getLastContactString(it->second.last_contact)) - .arg(getMasterKeyString(it->second.encryption_master_key)) - ),oy+=celly ; - - // update the pixmap - // pixmap = tmppixmap; maxHeight = std::max(oy,10*celly); } From a52c94d23c0f061c4a6f746a59fc8fc7d78dc8ef Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 15 Jul 2018 19:15:05 +0200 Subject: [PATCH 79/84] improved display of encryption master key --- retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp index bbabb8fd6..c8f34c8b9 100644 --- a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp +++ b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp @@ -489,7 +489,7 @@ static QString getSideString(uint8_t side) static QString getMasterKeyString(uint8_t *key) { - return QString::fromStdString(RsUtil::BinToHex(key,10)); + return QString::fromStdString(RsUtil::BinToHex(key,32,10)); } void GxsNetTunnelsDialog::updateDisplay() From 55e99ef0d1f9b292ffb0c75816c8b2480c14a9dd Mon Sep 17 00:00:00 2001 From: sehraf Date: Mon, 16 Jul 2018 23:22:04 +0200 Subject: [PATCH 80/84] add auto detection of installed rapidjson --- libretroshare/src/serialiser/rsserializer.h | 4 ++++ libretroshare/src/serialiser/rstypeserializer.cc | 5 ++++- libretroshare/src/serialiser/rstypeserializer.h | 4 ++++ libretroshare/src/use_libretroshare.pri | 10 +++++++++- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/serialiser/rsserializer.h b/libretroshare/src/serialiser/rsserializer.h index 35dc091c3..6bde6930c 100644 --- a/libretroshare/src/serialiser/rsserializer.h +++ b/libretroshare/src/serialiser/rsserializer.h @@ -154,7 +154,11 @@ #include #include #include +#ifdef HAS_RAPIDJSON +#include +#else #include +#endif // HAS_RAPIDJSON #include "retroshare/rsflags.h" #include "serialiser/rsserial.h" diff --git a/libretroshare/src/serialiser/rstypeserializer.cc b/libretroshare/src/serialiser/rstypeserializer.cc index 72d20ecc2..79a3eaab0 100644 --- a/libretroshare/src/serialiser/rstypeserializer.cc +++ b/libretroshare/src/serialiser/rstypeserializer.cc @@ -36,8 +36,11 @@ #include #include // for typeid -#include +#ifdef HAS_RAPIDJSON +#include +#else #include +#endif // HAS_RAPIDJSON //static const uint32_t MAX_SERIALIZED_ARRAY_SIZE = 500 ; static const uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB. diff --git a/libretroshare/src/serialiser/rstypeserializer.h b/libretroshare/src/serialiser/rstypeserializer.h index 3e3c0b251..e91aa0c45 100644 --- a/libretroshare/src/serialiser/rstypeserializer.h +++ b/libretroshare/src/serialiser/rstypeserializer.h @@ -35,7 +35,11 @@ #include "serialiser/rsserializer.h" #include "serialiser/rsserializable.h" +#ifdef HAS_RAPIDJSON +#include +#else #include +#endif // HAS_RAPIDJSON #include // for typeid #include #include diff --git a/libretroshare/src/use_libretroshare.pri b/libretroshare/src/use_libretroshare.pri index 3a3d1acb7..7c5af09b7 100644 --- a/libretroshare/src/use_libretroshare.pri +++ b/libretroshare/src/use_libretroshare.pri @@ -16,7 +16,15 @@ bitdht { # when rapidjson is mainstream on all distribs, we will not need the sources # anymore in the meantime, they are part of the RS directory so that it is # always possible to find them -INCLUDEPATH *= $$system_path($$clean_path($${PWD}/../../rapidjson-1.1.0)) +RAPIDJSON_AVAILABLE = $$system(pkg-config --atleast-version 1.1 RapidJSON && echo yes) +isEmpty(RAPIDJSON_AVAILABLE) { + message("using built-in rapidjson") + INCLUDEPATH *= $$system_path($$clean_path($${PWD}/../../rapidjson-1.1.0)) +} else { + message("using systems rapidjson") + DEFINES *= HAS_RAPIDJSON +} + sLibs = mLibs = $$RS_SQL_LIB ssl crypto $$RS_THREAD_LIB $$RS_UPNP_LIB From b9436fbef27c941a0818f53280b5e765d0f5aa09 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 17 Jul 2018 10:08:39 +0200 Subject: [PATCH 81/84] fixed leading zeroes problem in title bar --- retroshare-gui/src/rshare.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/retroshare-gui/src/rshare.cpp b/retroshare-gui/src/rshare.cpp index ff6038de0..a87ad358f 100644 --- a/retroshare-gui/src/rshare.cpp +++ b/retroshare-gui/src/rshare.cpp @@ -360,7 +360,7 @@ QString Rshare::retroshareVersion(bool withRevision) { QString version = QString("%1.%2.%3%4").arg(RS_MAJOR_VERSION).arg(RS_MINOR_VERSION).arg(RS_BUILD_NUMBER).arg(RS_BUILD_NUMBER_ADD); if (withRevision) { - version += QString(" %1 %2").arg(tr("Revision")).arg(QString::number(RS_REVISION_NUMBER,16)); + version += QString(" %1 %2").arg(tr("Revision")).arg(RS_REVISION_NUMBER,8,16,QChar('0')); } return version; From 9b0a4b966e2d90bb563bd97ead25edec4b873d33 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 18 Jul 2018 21:20:51 +0200 Subject: [PATCH 82/84] removed extra call to loadConfiguration() --- libretroshare/src/rsserver/rsinit.cc | 41 +++++++++++++--------------- 1 file changed, 19 insertions(+), 22 deletions(-) diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index 877c45488..ba84eda17 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -1631,46 +1631,43 @@ int RsServer::StartupRetroShare() //mConfigMgr->addConfiguration("ftserver.cfg", ftserver); // - mConfigMgr->addConfiguration("gpg_prefs.cfg", AuthGPG::getAuthGPG()); + mConfigMgr->addConfiguration("gpg_prefs.cfg" , AuthGPG::getAuthGPG()); mConfigMgr->addConfiguration("gxsnettunnel.cfg", mGxsNetTunnel); - mConfigMgr->loadConfiguration(); - - mConfigMgr->addConfiguration("peers.cfg", mPeerMgr); - mConfigMgr->addConfiguration("general.cfg", mGeneralConfig); - mConfigMgr->addConfiguration("msgs.cfg", msgSrv); - mConfigMgr->addConfiguration("chat.cfg", chatSrv); - mConfigMgr->addConfiguration("p3History.cfg", mHistoryMgr); - mConfigMgr->addConfiguration("p3Status.cfg", mStatusSrv); - mConfigMgr->addConfiguration("turtle.cfg", tr); + mConfigMgr->addConfiguration("peers.cfg" , mPeerMgr); + mConfigMgr->addConfiguration("general.cfg" , mGeneralConfig); + mConfigMgr->addConfiguration("msgs.cfg" , msgSrv); + mConfigMgr->addConfiguration("chat.cfg" , chatSrv); + mConfigMgr->addConfiguration("p3History.cfg" , mHistoryMgr); + mConfigMgr->addConfiguration("p3Status.cfg" , mStatusSrv); + mConfigMgr->addConfiguration("turtle.cfg" , tr); #ifndef RETROTOR - mConfigMgr->addConfiguration("banlist.cfg", mBanList); + mConfigMgr->addConfiguration("banlist.cfg" , mBanList); #endif mConfigMgr->addConfiguration("servicecontrol.cfg", serviceCtrl); - mConfigMgr->addConfiguration("reputations.cfg", mReputations); + mConfigMgr->addConfiguration("reputations.cfg" , mReputations); #ifdef ENABLE_GROUTER - mConfigMgr->addConfiguration("grouter.cfg", gr); + mConfigMgr->addConfiguration("grouter.cfg" , gr); #endif #ifdef RS_USE_BITDHT - mConfigMgr->addConfiguration("bitdht.cfg", mBitDht); + mConfigMgr->addConfiguration("bitdht.cfg" , mBitDht); #endif #ifdef RS_ENABLE_GXS # ifdef RS_GXS_TRANS mConfigMgr->addConfiguration("gxs_trans_ns.cfg", gxstrans_ns); - mConfigMgr->addConfiguration("gxs_trans.cfg", mGxsTrans); + mConfigMgr->addConfiguration("gxs_trans.cfg" , mGxsTrans); # endif // RS_GXS_TRANS - mConfigMgr->addConfiguration("p3identity.cfg", mGxsIdService); - - mConfigMgr->addConfiguration("identity.cfg", gxsid_ns); - mConfigMgr->addConfiguration("gxsforums.cfg", gxsforums_ns); + mConfigMgr->addConfiguration("p3identity.cfg" , mGxsIdService); + mConfigMgr->addConfiguration("identity.cfg" , gxsid_ns); + mConfigMgr->addConfiguration("gxsforums.cfg" , gxsforums_ns); mConfigMgr->addConfiguration("gxsforums_srv.cfg", mGxsForums); - mConfigMgr->addConfiguration("gxschannels.cfg", gxschannels_ns); + mConfigMgr->addConfiguration("gxschannels.cfg" , gxschannels_ns); mConfigMgr->addConfiguration("gxschannels_srv.cfg", mGxsChannels); - mConfigMgr->addConfiguration("gxscircles.cfg", gxscircles_ns); - mConfigMgr->addConfiguration("posted.cfg", posted_ns); + mConfigMgr->addConfiguration("gxscircles.cfg" , gxscircles_ns); + mConfigMgr->addConfiguration("posted.cfg" , posted_ns); #ifdef RS_USE_WIKI mConfigMgr->addConfiguration("wiki.cfg", wiki_ns); #endif From dac885e24dc7172aae1071c7d45c5e4b541def1d Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 18 Jul 2018 21:22:38 +0200 Subject: [PATCH 83/84] made randomBias initilized with true random bytes at start --- libretroshare/src/gxs/rsgxsnettunnel.cc | 16 +++++++--------- libretroshare/src/pqi/p3cfgmgr.cc | 6 +++--- retroshare-gui/src/gui/common/GroupTreeWidget.ui | 6 +++++- .../src/gui/gxs/GxsGroupFrameDialog.ui | 13 +++++++++++-- .../src/gui/statistics/TurtleRouterDialog.cpp | 7 ++++--- 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index 1e04c6621..a35026a74 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -30,7 +30,7 @@ #include "gxs/rsnxs.h" #include "rsgxsnettunnel.h" -//#define DEBUG_RSGXSNETTUNNEL 1 +#define DEBUG_RSGXSNETTUNNEL 1 #define GXS_NET_TUNNEL_NOT_IMPLEMENTED() { std::cerr << __PRETTY_FUNCTION__ << ": not yet implemented." << std::endl; } #define GXS_NET_TUNNEL_DEBUG() std::cerr << time(NULL) << " : GXS_NET_TUNNEL: " << __FUNCTION__ << " : " @@ -43,7 +43,6 @@ RsGxsDistSync *rsGxsDistSync = NULL; RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") { -#warning this is for testing only. In the final version this needs to be initialized with some random content, saved and re-used for a while (e.g. 1 month) mRandomBias.clear(); mLastKeepAlive = time(NULL) + (lrand48()%20); // adds some variance in order to avoid doing all this tasks at once across services @@ -779,14 +778,10 @@ const Bias20Bytes& RsGxsNetTunnelService::locked_randomBias() { if(mRandomBias.isNull()) { -#ifdef DEBUG_RSGXSNETTUNNEL -#warning /!\ this is for testing only! Remove this when done! Can not be done at initialization when rsPeer is not started. - RsPeerId ssl_id = rsPeers->getOwnId() ; - mRandomBias = Bias20Bytes(RsDirUtil::sha1sum(ssl_id.toByteArray(),ssl_id.SIZE_IN_BYTES)) ; -#else mRandomBias = Bias20Bytes::random(); -#endif IndicateConfigChanged(); + + std::cerr << "Initialized RsGxsNetTunnel random bias to " << RsUtil::BinToHex(mRandomBias.toByteArray(),mRandomBias.SIZE_IN_BYTES) << std::endl; } return mRandomBias ; @@ -796,7 +791,7 @@ RsGxsNetTunnelVirtualPeerId RsGxsNetTunnelService::locked_makeVirtualPeerId(cons { assert(RsPeerId::SIZE_IN_BYTES <= Sha1CheckSum::SIZE_IN_BYTES) ;// so that we can build the virtual PeerId from a SHA1 sum. - // We compute sha1( SSL_id | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId + // We compute sha1( GroupId | mRandomBias ) and trunk it to 16 bytes in order to compute a RsPeerId Bias20Bytes rb(locked_randomBias()); @@ -934,6 +929,7 @@ bool RsGxsNetTunnelService::saveList(bool& cleanup, std::list& save) { RS_STACK_MUTEX(mGxsNetTunnelMtx); it2->mRandomBias = mRandomBias; + std::cerr << "Saving RsGxsNetTunnel random bias to disc" << std::endl; } save.push_back(it2) ; @@ -952,6 +948,8 @@ bool RsGxsNetTunnelService::loadList(std::list &load) { RS_STACK_MUTEX(mGxsNetTunnelMtx); mRandomBias = rbsi->mRandomBias; + + std::cerr << "Loaded RsGxsNetTunnel random bias from disc: " << RsUtil::BinToHex(mRandomBias.toByteArray(),mRandomBias.SIZE_IN_BYTES) << std::endl; } else GXS_NET_TUNNEL_ERROR() << " unknown item in config file: type=" << std::hex << (*it)->PacketId() << std::dec << std::endl; diff --git a/libretroshare/src/pqi/p3cfgmgr.cc b/libretroshare/src/pqi/p3cfgmgr.cc index 3266f68d4..97d8165a3 100644 --- a/libretroshare/src/pqi/p3cfgmgr.cc +++ b/libretroshare/src/pqi/p3cfgmgr.cc @@ -65,7 +65,7 @@ void p3ConfigMgr::tick() #ifdef CONFIG_DEBUG std::cerr << "p3ConfigMgr::tick() Config Changed - Element: "; - std::cerr << it->first; + std::cerr << *it; std::cerr << std::endl; #endif @@ -111,7 +111,7 @@ void p3ConfigMgr::saveConfig() { #ifdef CONFIG_DEBUG std::cerr << "p3ConfigMgr::globalSaveConfig() Saving Element: "; - std::cerr << it->first; + std::cerr << *it; std::cerr << std::endl; #endif ok &= (*it)->saveConfiguration(); @@ -137,7 +137,7 @@ void p3ConfigMgr::loadConfig() { #ifdef CONFIG_DEBUG std::cerr << "p3ConfigMgr::loadConfig() Element: "; - std::cerr << cit->first <<"Dummy Hash: " << dummyHash; + std::cerr << *cit <<" Dummy Hash: " << dummyHash; std::cerr << std::endl; #endif diff --git a/retroshare-gui/src/gui/common/GroupTreeWidget.ui b/retroshare-gui/src/gui/common/GroupTreeWidget.ui index 7591c6a1c..26167dcd7 100644 --- a/retroshare-gui/src/gui/common/GroupTreeWidget.ui +++ b/retroshare-gui/src/gui/common/GroupTreeWidget.ui @@ -125,7 +125,11 @@ - + + + <html><head/><body><p>Searches a single keyword into the reachable network.</p><p>Objects already provided by friend nodes are not reported.</p></body></html> + + diff --git a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.ui b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.ui index 274207b5c..7431deb97 100644 --- a/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.ui +++ b/retroshare-gui/src/gui/gxs/GxsGroupFrameDialog.ui @@ -7,7 +7,7 @@ 0 0 619 - 420 + 493 @@ -38,7 +38,16 @@ QFrame::Sunken - + + 2 + + + 2 + + + 2 + + 2 diff --git a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp index c8f34c8b9..b81264e77 100644 --- a/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp +++ b/retroshare-gui/src/gui/statistics/TurtleRouterDialog.cpp @@ -487,9 +487,9 @@ static QString getSideString(uint8_t side) return side?QObject::tr("Client"):QObject::tr("Server") ; } -static QString getMasterKeyString(uint8_t *key) +static QString getMasterKeyString(const uint8_t *key,uint32_t size) { - return QString::fromStdString(RsUtil::BinToHex(key,32,10)); + return QString::fromStdString(RsUtil::BinToHex(key,size,10)); } void GxsNetTunnelsDialog::updateDisplay() @@ -567,6 +567,7 @@ void GxsNetTunnelsDialog::updateDisplay() int ox=5*fact,oy=5*fact ; painter.setPen(QColor::fromRgb(0,0,0)) ; + painter.drawText(ox+2*cellx,oy+celly,tr("Random Bias: %1").arg(getMasterKeyString(bias.toByteArray(),20))) ; oy += celly ; painter.drawText(ox+2*cellx,oy+celly,tr("GXS Groups:")) ; oy += celly ; for(auto it(groups.begin());it!=groups.end();++it) @@ -595,7 +596,7 @@ void GxsNetTunnelsDialog::updateDisplay() .arg(getVirtualPeerStatusString(it3->second.vpid_status)) .arg(getSideString(it3->second.side)) .arg(getLastContactString(it3->second.last_contact)) - .arg(getMasterKeyString(it3->second.encryption_master_key)) + .arg(getMasterKeyString(it3->second.encryption_master_key,32)) ),oy+=celly ; } else From d4fce07e4c2d375ac65415fcb6c1b0450e1b5b62 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 19 Jul 2018 23:46:31 +0200 Subject: [PATCH 84/84] changed lrand48 to RSRandom for cross-plateform compatibility, in rsgxsnettunnel.cc --- libretroshare/src/gxs/rsgxsnettunnel.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnettunnel.cc b/libretroshare/src/gxs/rsgxsnettunnel.cc index a35026a74..ff3134493 100644 --- a/libretroshare/src/gxs/rsgxsnettunnel.cc +++ b/libretroshare/src/gxs/rsgxsnettunnel.cc @@ -45,9 +45,9 @@ RsGxsNetTunnelService::RsGxsNetTunnelService(): mGxsNetTunnelMtx("GxsNetTunnel") { mRandomBias.clear(); - mLastKeepAlive = time(NULL) + (lrand48()%20); // adds some variance in order to avoid doing all this tasks at once across services - mLastAutoWash = time(NULL) + (lrand48()%20); - mLastDump = time(NULL) + (lrand48()%20); + mLastKeepAlive = time(NULL) + (RSRandom::random_u32()%20); // adds some variance in order to avoid doing all this tasks at once across services + mLastAutoWash = time(NULL) + (RSRandom::random_u32()%20); + mLastDump = time(NULL) + (RSRandom::random_u32()%20); } //===========================================================================================================================================//