Merge pull request #1522 from csoler/v0.6-Turtle

Add Fast Track items in Turtle and make GxsTunnel to use them. Improves distant chat.
This commit is contained in:
csoler 2019-04-14 20:52:35 +02:00 committed by GitHub
commit 051e2ee682
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 187 additions and 66 deletions

View File

@ -692,6 +692,20 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
(void) direction;
#endif
void *data_bytes;
uint32_t data_size ;
bool accept_fast_items = false;
const RsTurtleGenericFastDataItem *fitem = dynamic_cast<const RsTurtleGenericFastDataItem*>(gitem) ;
if(fitem != NULL)
{
data_bytes = fitem->data_bytes ;
data_size = fitem->data_size ;
accept_fast_items = true;
}
else
{
const RsTurtleGenericDataItem *item = dynamic_cast<const RsTurtleGenericDataItem*>(gitem) ;
if(item == NULL)
@ -699,15 +713,19 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
std::cerr << "(EE) item is not a data item. That is an error." << std::endl;
return ;
}
data_bytes = item->data_bytes ;
data_size = item->data_size ;
}
// Call the AES crypto module
// - the IV is the first 8 bytes of item->data_bytes
if(item->data_size < 8)
if(data_size < 8)
{
std::cerr << "(EE) item encrypted data stream is too small: size = " << item->data_size << std::endl;
std::cerr << "(EE) item encrypted data stream is too small: size = " << data_size << std::endl;
return ;
}
if(*((uint64_t*)item->data_bytes) != 0) // WTF?? we should use flags
if(*((uint64_t*)data_bytes) != 0) // WTF?? we should use flags
{
#ifdef DEBUG_GXS_TUNNEL
std::cerr << " Item is encrypted." << std::endl;
@ -715,7 +733,7 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
// if cannot decrypt, it means the key is wrong. We need to re-negociate a new key.
handleEncryptedData((uint8_t*)item->data_bytes,item->data_size,hash,virtual_peer_id) ;
handleEncryptedData((uint8_t*)data_bytes,data_size,hash,virtual_peer_id,accept_fast_items) ;
}
else
{
@ -725,8 +743,8 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
// Now try deserialise the decrypted data to make an RsItem out of it.
//
uint32_t pktsize = item->data_size-8;
RsItem *citem = RsGxsTunnelSerialiser().deserialise(&((uint8_t*)item->data_bytes)[8],&pktsize) ;
uint32_t pktsize = data_size-8;
RsItem *citem = RsGxsTunnelSerialiser().deserialise(&((uint8_t*)data_bytes)[8],&pktsize) ;
if(citem == NULL)
{
@ -752,7 +770,11 @@ void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gite
// This function encrypts the given data and adds a MAC and an IV into a serialised memory chunk that is then sent through the tunnel.
#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
bool p3GxsTunnelService::handleEncryptedData(const uint8_t *data_bytes,uint32_t data_size,const TurtleFileHash& hash,const RsPeerId& virtual_peer_id,bool accepts_fast_items)
#else
bool p3GxsTunnelService::handleEncryptedData(const uint8_t *data_bytes,uint32_t data_size,const TurtleFileHash& hash,const RsPeerId& virtual_peer_id)
#endif
{
#ifdef DEBUG_GXS_TUNNEL
std::cerr << "p3GxsTunnelService::handleEncryptedDataItem()" << std::endl;
@ -795,6 +817,16 @@ bool p3GxsTunnelService::handleEncryptedData(const uint8_t *data_bytes,uint32_t
std::cerr << "(EE) no tunnel data for tunnel ID=" << tunnel_id << ". This is a bug." << std::endl;
return false ;
}
#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
if(accepts_fast_items)
{
if(!it2->second.accepts_fast_turtle_items)
std::cerr << "(II) received probe for Fast track turtle items for tunnel VPID " << it2->second.virtual_peer_id << ": switching to Fast items mode." << std::endl;
it2->second.accepts_fast_turtle_items = true;
}
#endif
memcpy(aes_key,it2->second.aes_key,GXS_TUNNEL_AES_KEY_SIZE) ;
#ifdef DEBUG_GXS_TUNNEL
@ -1291,34 +1323,77 @@ bool p3GxsTunnelService::locked_sendEncryptedTunnelData(RsGxsTunnelItem *item)
// make a TurtleGenericData item out of it:
//
RsTurtleGenericDataItem *gitem = new RsTurtleGenericDataItem ;
gitem->data_size = encrypted_size + GXS_TUNNEL_ENCRYPTION_IV_SIZE + GXS_TUNNEL_ENCRYPTION_HMAC_SIZE ;
gitem->data_bytes = rs_malloc(gitem->data_size) ;
uint32_t data_size = encrypted_size + GXS_TUNNEL_ENCRYPTION_IV_SIZE + GXS_TUNNEL_ENCRYPTION_HMAC_SIZE ;
void *data_bytes = rs_malloc(data_size) ;
if(gitem->data_bytes == NULL)
if(data_bytes == NULL)
return false ;
memcpy(& ((uint8_t*)gitem->data_bytes)[0] ,&IV,8) ;
memcpy(& ((uint8_t*)data_bytes)[0] ,&IV,8) ;
unsigned int md_len = GXS_TUNNEL_ENCRYPTION_HMAC_SIZE ;
HMAC(EVP_sha1(),aes_key,GXS_TUNNEL_AES_KEY_SIZE,encrypted_data,encrypted_size,&(((uint8_t*)gitem->data_bytes)[GXS_TUNNEL_ENCRYPTION_IV_SIZE]),&md_len) ;
HMAC(EVP_sha1(),aes_key,GXS_TUNNEL_AES_KEY_SIZE,encrypted_data,encrypted_size,&(((uint8_t*)data_bytes)[GXS_TUNNEL_ENCRYPTION_IV_SIZE]),&md_len) ;
memcpy(& (((uint8_t*)gitem->data_bytes)[GXS_TUNNEL_ENCRYPTION_HMAC_SIZE+GXS_TUNNEL_ENCRYPTION_IV_SIZE]),encrypted_data,encrypted_size) ;
memcpy(& (((uint8_t*)data_bytes)[GXS_TUNNEL_ENCRYPTION_HMAC_SIZE+GXS_TUNNEL_ENCRYPTION_IV_SIZE]),encrypted_data,encrypted_size) ;
#ifdef DEBUG_GXS_TUNNEL
std::cerr << " Using IV: " << std::hex << IV << std::dec << std::endl;
std::cerr << " Using Key: " << RsUtil::BinToHex((char*)aes_key,GXS_TUNNEL_AES_KEY_SIZE) ; std::cerr << std::endl;
std::cerr << " hmac: " << RsUtil::BinToHex((char*)gitem->data_bytes,GXS_TUNNEL_ENCRYPTION_HMAC_SIZE) << std::endl;
std::cerr << " hmac: " << RsUtil::BinToHex((char*)data_bytes,GXS_TUNNEL_ENCRYPTION_HMAC_SIZE) << std::endl;
#endif
#ifdef DEBUG_GXS_TUNNEL
std::cerr << "GxsTunnelService::sendEncryptedTunnelData(): Sending encrypted data to virtual peer: " << virtual_peer_id << std::endl;
std::cerr << " gitem->data_size = " << gitem->data_size << std::endl;
std::cerr << " serialised data = " << RsUtil::BinToHex((unsigned char*)gitem->data_bytes,gitem->data_size,100) ;
std::cerr << " data_size = " << data_size << std::endl;
std::cerr << " serialised data = " << RsUtil::BinToHex((unsigned char*)data_bytes,data_size,100) ;
std::cerr << std::endl;
#endif
mTurtle->sendTurtleData(virtual_peer_id,gitem) ;
// We send the item through the turtle tunnel. We do that using the new 'fast' item type if the tunnel accepts it, and using the old slow one otherwise.
// Still if the tunnel hasn't been probed, we duplicate the packet using the new fast item format. The packet being received twice on the other side will
// be discarded with a warning.
// Note that because the data is deleted by sendTurtleData(), we need to send all packets at the end, and properly duplicate the data when needed.
#ifdef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
RsTurtleGenericFastDataItem *gitem = new RsTurtleGenericFastDataItem;
gitem->data_bytes = data_bytes;
gitem->data_size = data_size ;
#else
RsTurtleGenericDataItem *gitem = NULL;
RsTurtleGenericFastDataItem *gitem2 = NULL;
if(!it->second.accepts_fast_turtle_items)
{
std::cerr << "Sending old format (slow) item for packet IV=" << std::hex << IV << std::dec << " in tunnel VPID=" << it->second.virtual_peer_id << std::endl;
gitem = new RsTurtleGenericDataItem ;
gitem->data_bytes = data_bytes;
gitem->data_size = data_size ;
}
if(it->second.accepts_fast_turtle_items || !it->second.already_probed_for_fast_items)
{
std::cerr << "Sending new format (fast) item for packet IV=" << std::hex << IV << std::dec << " in tunnel VPID=" << it->second.virtual_peer_id << std::endl;
gitem2 = new RsTurtleGenericFastDataItem ;
if(gitem != NULL) // duplicate the data because it was already sent in gitem.
{
gitem2->data_bytes = rs_malloc(data_size);
gitem2->data_size = data_size ;
memcpy(gitem2->data_bytes,data_bytes,data_size);
}
else
{
gitem2->data_bytes = data_bytes;
gitem2->data_size = data_size ;
}
it->second.already_probed_for_fast_items = true;
}
if(gitem2) mTurtle->sendTurtleData(virtual_peer_id,gitem2) ;
#endif
if(gitem) mTurtle->sendTurtleData(virtual_peer_id,gitem) ;
return true ;
}

View File

@ -128,14 +128,12 @@ public:
// Creates the invite if the public key of the distant peer is available.
// Om success, stores the invite in the map above, so that we can respond to tunnel requests.
//
virtual bool requestSecuredTunnel(const RsGxsId& to_id,const RsGxsId& from_id,RsGxsTunnelId& tunnel_id,uint32_t service_id,uint32_t& error_code) ;
virtual bool closeExistingTunnel(const RsGxsTunnelId &tunnel_id,uint32_t service_id) ;
virtual bool getTunnelsInfo(std::vector<GxsTunnelInfo>& infos);
virtual bool getTunnelInfo(const RsGxsTunnelId& tunnel_id,GxsTunnelInfo& info);
virtual bool sendData(const RsGxsTunnelId& tunnel_id,uint32_t service_id,const uint8_t *data,uint32_t size) ;
virtual bool registerClientService(uint32_t service_id,RsGxsTunnelClientService *service) ;
virtual bool requestSecuredTunnel(const RsGxsId& to_id,const RsGxsId& from_id,RsGxsTunnelId& tunnel_id,uint32_t service_id,uint32_t& error_code) override ;
virtual bool closeExistingTunnel(const RsGxsTunnelId &tunnel_id,uint32_t service_id) override ;
virtual bool getTunnelsInfo(std::vector<GxsTunnelInfo>& infos) override ;
virtual bool getTunnelInfo(const RsGxsTunnelId& tunnel_id,GxsTunnelInfo& info) override ;
virtual bool sendData(const RsGxsTunnelId& tunnel_id,uint32_t service_id,const uint8_t *data,uint32_t size) override ;
virtual bool registerClientService(uint32_t service_id,RsGxsTunnelClientService *service) override ;
// derived from p3service
@ -150,6 +148,9 @@ private:
{
public:
GxsTunnelPeerInfo() : last_contact(0), last_keep_alive_sent(0), status(0), direction(0)
#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
,accepts_fast_turtle_items(false)
#endif
{
memset(aes_key, 0, GXS_TUNNEL_AES_KEY_SIZE);
@ -168,10 +169,14 @@ private:
RsGxsId own_gxs_id ; // gxs id we're using to talk.
RsTurtleGenericTunnelItem::Direction direction ; // specifiec wether we are client(managing the tunnel) or server.
TurtleFileHash hash ; // hash that is last used. This is necessary for handling tunnel establishment
std::set<uint32_t> client_services ;// services that used this tunnel
std::set<uint32_t> client_services ; // services that used this tunnel
std::map<uint64_t,rstime_t> received_data_prints ; // list of recently received messages, to avoid duplicates. Kept for 20 mins at most.
uint32_t total_sent ;
uint32_t total_received ;
uint32_t total_sent ; // total data sent to this peer
uint32_t total_received ; // total data received by this peer
#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
bool accepts_fast_turtle_items; // does the tunnel accept RsTurtleGenericFastDataItem type?
bool already_probed_for_fast_items; // has the tunnel been probed already? If not, a fast item will be sent
#endif
};
class GxsTunnelDHInfo
@ -243,7 +248,11 @@ private:
bool locked_sendEncryptedTunnelData(RsGxsTunnelItem *item) ;
bool locked_sendClearTunnelData(RsGxsTunnelDHPublicKeyItem *item); // this limits the usage to DH items. Others should be encrypted!
bool handleEncryptedData(const uint8_t *data_bytes,uint32_t data_size,const TurtleFileHash& hash,const RsPeerId& virtual_peer_id) ;
#ifndef V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
bool handleEncryptedData(const uint8_t *data_bytes, uint32_t data_size, const TurtleFileHash& hash, const RsPeerId& virtual_peer_id, bool accepts_fast_items) ;
#else
bool handleEncryptedData(const uint8_t *data_bytes, uint32_t data_size, const TurtleFileHash& hash, const RsPeerId& virtual_peer_id) ;
#endif
// local data

View File

@ -46,6 +46,7 @@ const uint8_t QOS_PRIORITY_RS_TURTLE_FILE_MAP = 3 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_GENERIC_ITEM = 3 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_FORWARD_FILE_DATA= 3 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_GENERIC_DATA = 5 ;
const uint8_t QOS_PRIORITY_RS_TURTLE_GENERIC_FAST_DATA= 7 ;
// File transfer
//

View File

@ -88,7 +88,7 @@ static const rstime_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 2 ; ///
static const uint32_t MAX_TUNNEL_REQS_PER_SECOND = 1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before
static const uint32_t MAX_ALLOWED_SR_IN_CACHE = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_FILES =5000 ; /// maximum number of search results forwarded back to the source.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT= 100 ; /// default maximum number of search results forwarded back source.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT = 100 ; /// default maximum number of search results forwarded back source.
static const float depth_peer_probability[7] = { 1.0f,0.99f,0.9f,0.7f,0.6f,0.5,0.4f } ;

View File

@ -53,6 +53,7 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : return new RsTurtleOpenTunnelItem();
case RS_TURTLE_SUBTYPE_TUNNEL_OK : return new RsTurtleTunnelOkItem();
case RS_TURTLE_SUBTYPE_GENERIC_DATA : return new RsTurtleGenericDataItem();
case RS_TURTLE_SUBTYPE_GENERIC_FAST_DATA : return new RsTurtleGenericFastDataItem();
case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST : return new RsTurtleGenericSearchRequestItem();
case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT : return new RsTurtleGenericSearchResultItem();
@ -244,8 +245,12 @@ void RsTurtleTunnelOkItem::serial_process(RsGenericSerializer::SerializeJob j,Rs
void RsTurtleGenericDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process<uint32_t>(j,ctx,tunnel_id ,"tunnel_id") ;
RsTypeSerializer::TlvMemBlock_proxy prox(data_bytes,data_size) ;
RsTypeSerializer::serial_process(j,ctx,prox,"data bytes") ;
}
void RsTurtleGenericFastDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process<uint32_t>(j,ctx,tunnel_id ,"tunnel_id") ;
RsTypeSerializer::TlvMemBlock_proxy prox(data_bytes,data_size) ;
RsTypeSerializer::serial_process(j,ctx,prox,"data bytes") ;
}

View File

@ -52,6 +52,7 @@ const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ;
// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ;
const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ;
const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST = 0x15 ;
const uint8_t RS_TURTLE_SUBTYPE_GENERIC_FAST_DATA = 0x16 ;
class TurtleSearchRequestInfo ;
@ -331,6 +332,28 @@ class RsTurtleGenericDataItem: public RsTurtleGenericTunnelItem
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
// Same, but with a fact priority. Can rather be used for e.g. distant chat.
//
class RsTurtleGenericFastDataItem: public RsTurtleGenericTunnelItem
{
public:
RsTurtleGenericFastDataItem() : RsTurtleGenericTunnelItem(RS_TURTLE_SUBTYPE_GENERIC_FAST_DATA), data_size(0), data_bytes(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_GENERIC_FAST_DATA);}
virtual ~RsTurtleGenericFastDataItem() { if(data_bytes != NULL) free(data_bytes) ; }
virtual bool shouldStampTunnel() const { return true ; }
uint32_t data_size ;
void *data_bytes ;
void clear()
{
free(data_bytes) ;
data_bytes = NULL ;
data_size = 0;
}
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
/***********************************************************************************/
/* Turtle Serialiser class */
/***********************************************************************************/

View File

@ -235,6 +235,13 @@ isEmpty(RS_UPNP_LIB):RS_UPNP_LIB = upnp ixml threadutil
# Why: hasing twice is not per se a security issue, but it makes it harder to change the settings for hashing.
#
# Backward compat: patched peers cannot connect to non patched peers older than Nov 2017.
#
# V07_NON_BACKWARD_COMPATIBLE_CHANGE_004:
#
# What: Do not probe that GXS tunnels accept fast items. Just assume they do.
# Why: Avoids sending probe packets
# BackwardCompat: old RS before Mai 2019 will not be able to distant chat.
#
###########################################################################################################################################################
#CONFIG += rs_v07_changes
@ -242,6 +249,7 @@ rs_v07_changes {
DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_001
DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_002
DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_003
DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_004
DEFINES += V07_NON_BACKWARD_COMPATIBLE_CHANGE_UNNAMED
}