Merge pull request #1522 from csoler/v0.6-Turtle

Add Fast Track items in Turtle and make GxsTunnel to use them. Improves distant chat.

libretroshare/src/gxstunnel/p3gxstunnel.cc

@@ -692,6 +692,20 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
     (void) direction;
 #endif
 
+    void *data_bytes;
+    uint32_t data_size ;
+    bool accept_fast_items = false;
+
+    const RsTurtleGenericFastDataItem *fitem = dynamic_cast<const RsTurtleGenericFastDataItem*>(gitem) ;
+
+    if(fitem != NULL)
+    {
+        data_bytes = fitem->data_bytes ;
+        data_size = fitem->data_size ;
+        accept_fast_items = true;
+    }
+    else
+    {
 		const RsTurtleGenericDataItem *item = dynamic_cast<const RsTurtleGenericDataItem*>(gitem) ;
 
 		if(item == NULL)
@@ -699,15 +713,19 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
 			std::cerr << "(EE) item is not a data item. That is an error." << std::endl;
 			return ;
 		}
+        data_bytes = item->data_bytes ;
+        data_size = item->data_size ;
+    }
+
 	// Call the AES crypto module
     // - the IV is the first 8 bytes of item->data_bytes
 
-    if(item->data_size < 8)
+    if(data_size < 8)
     {
-        std::cerr << "(EE) item encrypted data stream is too small: size = " << item->data_size << std::endl;
+        std::cerr << "(EE) item encrypted data stream is too small: size = " << data_size << std::endl;
         return ;
     }
-    if(*((uint64_t*)item->data_bytes) != 0)	// WTF?? we should use flags
+    if(*((uint64_t*)data_bytes) != 0)	// WTF?? we should use flags
     {
 #ifdef DEBUG_GXS_TUNNEL
         std::cerr << "  Item is encrypted." << std::endl;
@@ -715,7 +733,7 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
 
         // if cannot decrypt, it means the key is wrong. We need to re-negociate a new key.
 
-        handleEncryptedData((uint8_t*)item->data_bytes,item->data_size,hash,virtual_peer_id) ;
+        handleEncryptedData((uint8_t*)data_bytes,data_size,hash,virtual_peer_id,accept_fast_items) ;
     }
     else
     {
@@ -725,8 +743,8 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
 
         // Now try deserialise the decrypted data to make an RsItem out of it.
         //
-        uint32_t pktsize = item->data_size-8;
+        uint32_t pktsize = data_size-8;
-        RsItem *citem = RsGxsTunnelSerialiser().deserialise(&((uint8_t*)item->data_bytes)[8],&pktsize) ;
+        RsItem *citem = RsGxsTunnelSerialiser().deserialise(&((uint8_t*)data_bytes)[8],&pktsize) ;
 
         if(citem == NULL)
         {
@@ -752,7 +770,11 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
 
 // This function encrypts the given data and adds a MAC and an IV into a serialised memory chunk that is then sent through the tunnel.
 
+#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
+bool p3GxsTunnelService::handleEncryptedData(const uint8_t *data_bytes,uint32_t data_size,const TurtleFileHash& hash,const RsPeerId& virtual_peer_id,bool accepts_fast_items)
+#else
 bool p3GxsTunnelService::handleEncryptedData(const uint8_t *data_bytes,uint32_t data_size,const TurtleFileHash& hash,const RsPeerId& virtual_peer_id)
+#endif
 {
 #ifdef DEBUG_GXS_TUNNEL
     std::cerr << "p3GxsTunnelService::handleEncryptedDataItem()" << std::endl;
@@ -795,6 +817,16 @@ bool p3GxsTunnelService::handleEncryptedData(const uint8_t *data_bytes,uint32_t
             std::cerr << "(EE) no tunnel data for tunnel ID=" << tunnel_id << ". This is a bug." << std::endl;
             return false ;
         }
+#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
+        if(accepts_fast_items)
+        {
+            if(!it2->second.accepts_fast_turtle_items)
+                std::cerr << "(II) received probe for Fast track turtle items for tunnel VPID " << it2->second.virtual_peer_id << ": switching to Fast items mode." << std::endl;
+
+            it2->second.accepts_fast_turtle_items = true;
+        }
+#endif
+
         memcpy(aes_key,it2->second.aes_key,GXS_TUNNEL_AES_KEY_SIZE) ;
 
 #ifdef DEBUG_GXS_TUNNEL
@@ -1291,34 +1323,77 @@ bool p3GxsTunnelService::locked_sendEncryptedTunnelData(RsGxsTunnelItem *item)
 
     // make a TurtleGenericData item out of it:
     //
-    RsTurtleGenericDataItem *gitem = new RsTurtleGenericDataItem ;
 
-    gitem->data_size  = encrypted_size + GXS_TUNNEL_ENCRYPTION_IV_SIZE + GXS_TUNNEL_ENCRYPTION_HMAC_SIZE ;
+    uint32_t data_size  = encrypted_size + GXS_TUNNEL_ENCRYPTION_IV_SIZE + GXS_TUNNEL_ENCRYPTION_HMAC_SIZE ;
-    gitem->data_bytes = rs_malloc(gitem->data_size) ;
+    void *data_bytes = rs_malloc(data_size) ;
 
-    if(gitem->data_bytes == NULL)
+    if(data_bytes == NULL)
         return false ;
     
-    memcpy(& ((uint8_t*)gitem->data_bytes)[0]                                       ,&IV,8) ;
+    memcpy(& ((uint8_t*)data_bytes)[0]                                       ,&IV,8) ;
 
     unsigned int md_len = GXS_TUNNEL_ENCRYPTION_HMAC_SIZE ;
-    HMAC(EVP_sha1(),aes_key,GXS_TUNNEL_AES_KEY_SIZE,encrypted_data,encrypted_size,&(((uint8_t*)gitem->data_bytes)[GXS_TUNNEL_ENCRYPTION_IV_SIZE]),&md_len) ;
+    HMAC(EVP_sha1(),aes_key,GXS_TUNNEL_AES_KEY_SIZE,encrypted_data,encrypted_size,&(((uint8_t*)data_bytes)[GXS_TUNNEL_ENCRYPTION_IV_SIZE]),&md_len) ;
     
-    memcpy(& (((uint8_t*)gitem->data_bytes)[GXS_TUNNEL_ENCRYPTION_HMAC_SIZE+GXS_TUNNEL_ENCRYPTION_IV_SIZE]),encrypted_data,encrypted_size) ;
+    memcpy(& (((uint8_t*)data_bytes)[GXS_TUNNEL_ENCRYPTION_HMAC_SIZE+GXS_TUNNEL_ENCRYPTION_IV_SIZE]),encrypted_data,encrypted_size) ;
     
 #ifdef DEBUG_GXS_TUNNEL
     std::cerr << "   Using  IV: " << std::hex << IV << std::dec << std::endl;
     std::cerr << "   Using Key: " << RsUtil::BinToHex((char*)aes_key,GXS_TUNNEL_AES_KEY_SIZE) ; std::cerr << std::endl;
-    std::cerr << "        hmac: " << RsUtil::BinToHex((char*)gitem->data_bytes,GXS_TUNNEL_ENCRYPTION_HMAC_SIZE) << std::endl;
+    std::cerr << "        hmac: " << RsUtil::BinToHex((char*)data_bytes,GXS_TUNNEL_ENCRYPTION_HMAC_SIZE) << std::endl;
 #endif
 #ifdef DEBUG_GXS_TUNNEL
     std::cerr << "GxsTunnelService::sendEncryptedTunnelData(): Sending encrypted data to virtual peer: " << virtual_peer_id << std::endl;
-    std::cerr << "   gitem->data_size = " << gitem->data_size << std::endl;
+    std::cerr << "   data_size = " << data_size << std::endl;
-    std::cerr << "    serialised data = " << RsUtil::BinToHex((unsigned char*)gitem->data_bytes,gitem->data_size,100) ;
+    std::cerr << "    serialised data = " << RsUtil::BinToHex((unsigned char*)data_bytes,data_size,100) ;
     std::cerr << std::endl;
 #endif
 
-    mTurtle->sendTurtleData(virtual_peer_id,gitem) ;
+    // We send the item through the turtle tunnel. We do that using the new 'fast' item type if the tunnel accepts it, and using the old slow one otherwise.
+    // Still if the tunnel hasn't been probed, we duplicate the packet using the new fast item format. The packet being received twice on the other side will
+    // be discarded with a warning.
+    // Note that because the data is deleted by sendTurtleData(), we need to send all packets at the end, and properly duplicate the data when needed.
+
+#ifdef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
+	RsTurtleGenericFastDataItem *gitem = new RsTurtleGenericFastDataItem;
+	gitem->data_bytes = data_bytes;
+	gitem->data_size  = data_size ;
+#else
+	RsTurtleGenericDataItem *gitem = NULL;
+	RsTurtleGenericFastDataItem *gitem2 = NULL;
+
+	if(!it->second.accepts_fast_turtle_items)
+	{
+        std::cerr << "Sending old format (slow) item for packet IV=" << std::hex << IV << std::dec << " in tunnel VPID=" << it->second.virtual_peer_id << std::endl;
+		gitem = new RsTurtleGenericDataItem ;
+
+		gitem->data_bytes = data_bytes;
+		gitem->data_size  = data_size ;
+	}
+
+	if(it->second.accepts_fast_turtle_items || !it->second.already_probed_for_fast_items)
+	{
+        std::cerr << "Sending new format (fast) item for packet IV=" << std::hex << IV << std::dec << " in tunnel VPID=" << it->second.virtual_peer_id << std::endl;
+		gitem2 = new RsTurtleGenericFastDataItem ;
+
+        if(gitem != NULL)	// duplicate the data because it was already sent in gitem.
+		{
+			gitem2->data_bytes = rs_malloc(data_size);
+			gitem2->data_size  = data_size ;
+            memcpy(gitem2->data_bytes,data_bytes,data_size);
+		}
+		else
+		{
+			gitem2->data_bytes = data_bytes;
+			gitem2->data_size  = data_size ;
+		}
+
+		it->second.already_probed_for_fast_items = true;
+
+	}
+	if(gitem2) mTurtle->sendTurtleData(virtual_peer_id,gitem2) ;
+#endif
+    if(gitem)  mTurtle->sendTurtleData(virtual_peer_id,gitem) ;
 
 	return true ;
 }

libretroshare/src/gxstunnel/p3gxstunnel.h

@@ -128,14 +128,12 @@ public:
     // Creates the invite if the public key of the distant peer is available.
     // Om success, stores the invite in the map above, so that we can respond to tunnel requests.
     //
-    virtual bool requestSecuredTunnel(const RsGxsId& to_id,const RsGxsId& from_id,RsGxsTunnelId& tunnel_id,uint32_t service_id,uint32_t& error_code) ;
+    virtual bool requestSecuredTunnel(const RsGxsId& to_id,const RsGxsId& from_id,RsGxsTunnelId& tunnel_id,uint32_t service_id,uint32_t& error_code) override ;
-    
+    virtual bool closeExistingTunnel(const RsGxsTunnelId &tunnel_id,uint32_t service_id) override ;
-    virtual bool closeExistingTunnel(const RsGxsTunnelId &tunnel_id,uint32_t service_id) ;
+    virtual bool getTunnelsInfo(std::vector<GxsTunnelInfo>& infos) override ;
-    virtual bool getTunnelsInfo(std::vector<GxsTunnelInfo>& infos);
+    virtual bool getTunnelInfo(const RsGxsTunnelId& tunnel_id,GxsTunnelInfo& info) override ;
-    virtual bool getTunnelInfo(const RsGxsTunnelId& tunnel_id,GxsTunnelInfo& info);
+    virtual bool sendData(const RsGxsTunnelId& tunnel_id,uint32_t service_id,const uint8_t *data,uint32_t size) override ;
-    virtual bool sendData(const RsGxsTunnelId& tunnel_id,uint32_t service_id,const uint8_t *data,uint32_t size) ;
+    virtual bool registerClientService(uint32_t service_id,RsGxsTunnelClientService *service) override ;
-    
-    virtual bool registerClientService(uint32_t service_id,RsGxsTunnelClientService *service) ;
 
     // derived from p3service
     
@@ -150,6 +148,9 @@ private:
     {
     public:
         GxsTunnelPeerInfo() : last_contact(0), last_keep_alive_sent(0), status(0), direction(0)
+#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
+					          ,accepts_fast_turtle_items(false)
+#endif
         {
             memset(aes_key, 0, GXS_TUNNEL_AES_KEY_SIZE);
             
@@ -168,10 +169,14 @@ private:
         RsGxsId own_gxs_id ;                                  // gxs id we're using to talk.
         RsTurtleGenericTunnelItem::Direction direction ;      // specifiec wether we are client(managing the tunnel) or server.
         TurtleFileHash hash ;                                 // hash that is last used. This is necessary for handling tunnel establishment
-        std::set<uint32_t> client_services ;// services that used this tunnel
+        std::set<uint32_t> client_services ;                  // services that used this tunnel
         std::map<uint64_t,rstime_t> received_data_prints ;    // list of recently received messages, to avoid duplicates. Kept for 20 mins at most.
-        uint32_t total_sent ;
+        uint32_t total_sent ;                                 // total data sent to this peer
-        uint32_t total_received ;
+        uint32_t total_received ;                             // total data received by this peer
+#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
+        bool accepts_fast_turtle_items;                       // does the tunnel accept RsTurtleGenericFastDataItem type?
+        bool already_probed_for_fast_items;                   // has the tunnel been probed already? If not, a fast item will be sent
+#endif
     };
 
     class GxsTunnelDHInfo
@@ -243,7 +248,11 @@ private:
     bool locked_sendEncryptedTunnelData(RsGxsTunnelItem *item) ;
     bool locked_sendClearTunnelData(RsGxsTunnelDHPublicKeyItem *item);	// this limits the usage to DH items. Others should be encrypted!
     
-    bool handleEncryptedData(const uint8_t *data_bytes,uint32_t data_size,const TurtleFileHash& hash,const RsPeerId& virtual_peer_id) ;
+#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
+    bool handleEncryptedData(const uint8_t *data_bytes, uint32_t data_size, const TurtleFileHash& hash, const RsPeerId& virtual_peer_id, bool accepts_fast_items) ;
+#else
+    bool handleEncryptedData(const uint8_t *data_bytes, uint32_t data_size, const TurtleFileHash& hash, const RsPeerId& virtual_peer_id) ;
+#endif
 
     // local data
     

libretroshare/src/rsitems/itempriorities.h

@@ -46,6 +46,7 @@ const uint8_t QOS_PRIORITY_RS_TURTLE_FILE_MAP         = 3 ;
 const uint8_t QOS_PRIORITY_RS_TURTLE_GENERIC_ITEM     = 3 ;
 const uint8_t QOS_PRIORITY_RS_TURTLE_FORWARD_FILE_DATA= 3 ;
 const uint8_t QOS_PRIORITY_RS_TURTLE_GENERIC_DATA     = 5 ;
+const uint8_t QOS_PRIORITY_RS_TURTLE_GENERIC_FAST_DATA= 7 ;
 
 // File transfer
 //

libretroshare/src/turtle/p3turtle.cc

@@ -88,7 +88,7 @@ static const rstime_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS   =   2 ;        ///
 static const uint32_t MAX_TUNNEL_REQS_PER_SECOND               =   1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before
 static const uint32_t MAX_ALLOWED_SR_IN_CACHE                  = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec.
 static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_FILES      =5000 ; /// maximum number of search results forwarded back to the source.
-static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT= 100 ;		/// default maximum number of search results forwarded back source.
+static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT    = 100 ; /// default maximum number of search results forwarded back source.
 
 static const float depth_peer_probability[7] = { 1.0f,0.99f,0.9f,0.7f,0.6f,0.5,0.4f } ;
 

libretroshare/src/turtle/rsturtleitem.cc

@@ -53,6 +53,7 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
 	case RS_TURTLE_SUBTYPE_OPEN_TUNNEL  			:	return new RsTurtleOpenTunnelItem();
 	case RS_TURTLE_SUBTYPE_TUNNEL_OK    			:	return new RsTurtleTunnelOkItem();
 	case RS_TURTLE_SUBTYPE_GENERIC_DATA 			:	return new RsTurtleGenericDataItem();
+	case RS_TURTLE_SUBTYPE_GENERIC_FAST_DATA 		:	return new RsTurtleGenericFastDataItem();
 	case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST	:	return new RsTurtleGenericSearchRequestItem();
 	case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT	:	return new RsTurtleGenericSearchResultItem();
 
@@ -244,8 +245,12 @@ void RsTurtleTunnelOkItem::serial_process(RsGenericSerializer::SerializeJob j,Rs
 void RsTurtleGenericDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
 {
     RsTypeSerializer::serial_process<uint32_t>(j,ctx,tunnel_id ,"tunnel_id") ;
-
     RsTypeSerializer::TlvMemBlock_proxy prox(data_bytes,data_size) ;
-
+    RsTypeSerializer::serial_process(j,ctx,prox,"data bytes") ;
+}
+void RsTurtleGenericFastDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
+{
+    RsTypeSerializer::serial_process<uint32_t>(j,ctx,tunnel_id ,"tunnel_id") ;
+    RsTypeSerializer::TlvMemBlock_proxy prox(data_bytes,data_size) ;
     RsTypeSerializer::serial_process(j,ctx,prox,"data bytes") ;
 }

libretroshare/src/turtle/rsturtleitem.h

@@ -52,6 +52,7 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST        = 0x11 ;
 // const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST        = 0x13 ;
 const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC               = 0x14 ;
 const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST       = 0x15 ;
+const uint8_t RS_TURTLE_SUBTYPE_GENERIC_FAST_DATA   	= 0x16 ;
 
 
 class TurtleSearchRequestInfo ;
@@ -331,6 +332,28 @@ class RsTurtleGenericDataItem: public RsTurtleGenericTunnelItem
 		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
 };
 
+// Same, but with a fact priority. Can rather be used for e.g. distant chat.
+//
+class RsTurtleGenericFastDataItem: public RsTurtleGenericTunnelItem
+{
+	public:
+        RsTurtleGenericFastDataItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_GENERIC_FAST_DATA), data_size(0), data_bytes(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_GENERIC_FAST_DATA);}
+		virtual ~RsTurtleGenericFastDataItem() { if(data_bytes != NULL) free(data_bytes) ; }
+
+		virtual bool shouldStampTunnel() const { return true ; }
+
+		uint32_t data_size ;
+		void *data_bytes ;
+
+        void clear()
+        {
+            free(data_bytes) ;
+            data_bytes = NULL ;
+            data_size = 0;
+        }
+	protected:
+		void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
+};
 /***********************************************************************************/
 /*                           Turtle Serialiser class                               */
 /***********************************************************************************/

retroshare.pri

@@ -235,6 +235,13 @@ isEmpty(RS_UPNP_LIB):RS_UPNP_LIB = upnp ixml threadutil
 #  	 Why: hasing twice is not per se a security issue, but it makes it harder to change the settings for hashing.
 #
 #  	 Backward compat: patched peers cannot connect to non patched peers older than Nov 2017.
+#
+#  V07_NON_BACKWARD_COMPATIBLE_CHANGE_004:
+#
+#    What: Do not probe that GXS tunnels accept fast items. Just assume they do.
+#    Why:  Avoids sending probe packets
+#    BackwardCompat: old RS before Mai 2019 will not be able to distant chat.
+#
 ###########################################################################################################################################################
 
 #CONFIG += rs_v07_changes
@@ -242,6 +249,7 @@ rs_v07_changes {
     DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_001
     DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_002
     DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_003
+    DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
     DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_UNNAMED
 }