Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-05-02 22:25:04 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into deep_search

Browse source 
Needed to make deep search and GXS search cooperate
This commit is contained in:
Gioacchino Mazzurco 2018-07-20 13:15:11 +02:00
commit a31b06ab60
No known key found for this signature in database
GPG key ID: A1FBCA3872E87051
103 changed files with 5004 additions and 989 deletions
Download patch file Download diff file Expand all files Collapse all files

212
libretroshare/src/crypto/rscrypto.cpp Normal file
View file

@ -0,0 +1,212 @@
/*******************************************************************************
* libretroshare/src/crypto: crypto.cc *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <csoler@users.sourceforge.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include <stdlib.h>
#include <string.h>
#include <assert.h>
#include "rscrypto.h"
#include "util/rsrandom.h"
//#define CRYPTO_DEBUG 1
namespace librs {
namespace crypto {
#define RSCRYPTO_DEBUG() std::cerr << time(NULL) << " : RSCRYPTO : " << __FUNCTION__ << " : "
#define RSCRYPTO_ERROR() std::cerr << "(EE) RSCRYPTO ERROR : "
static const uint32_t ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE = 12 ;
static const uint32_t ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE = 16 ;
static const uint32_t ENCRYPTED_MEMORY_HEADER_SIZE = 4 ;
static const uint32_t ENCRYPTED_MEMORY_EDATA_SIZE = 4 ;
static const uint8_t ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305 = 0x01 ;
static const uint8_t ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256 = 0x02 ;
bool encryptAuthenticateData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,unsigned char *& encrypted_data,uint32_t& encrypted_data_len)
{
uint8_t initialization_vector[ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE] ;
RSRandom::random_bytes(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) ;
#ifdef CRYPTO_DEBUG
RSCRYPTO_DEBUG() << "ftServer::Encrypting ft item." << std::endl;
RSCRYPTO_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) << std::endl;
#endif
uint32_t item_serialized_size = clear_data_size;//RsGenericSerializer().size(clear_item) ;
uint32_t total_data_size = ENCRYPTED_MEMORY_HEADER_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE + item_serialized_size + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE ;
#ifdef CRYPTO_DEBUG
RSCRYPTO_DEBUG() << " clear part size : " << size(clear_item) << std::endl;
RSCRYPTO_DEBUG() << " total item size : " << total_data_size << std::endl;
#endif
encrypted_data = (unsigned char*)rs_malloc( total_data_size ) ;
encrypted_data_len = total_data_size ;
if(encrypted_data == NULL)
return false ;
uint8_t *edata = (uint8_t*)encrypted_data;
uint32_t edata_size = item_serialized_size;
uint32_t offset = 0;
edata[0] = 0xae ;
edata[1] = 0xad ;
edata[2] = ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256 ; // means AEAD_chacha20_sha256
edata[3] = 0x01 ;
offset += ENCRYPTED_MEMORY_HEADER_SIZE;
uint32_t aad_offset = offset ;
uint32_t aad_size = ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE ;
memcpy(&edata[offset], initialization_vector, ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) ;
offset += ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ;
edata[offset+0] = (edata_size >> 0) & 0xff ;
edata[offset+1] = (edata_size >> 8) & 0xff ;
edata[offset+2] = (edata_size >> 16) & 0xff ;
edata[offset+3] = (edata_size >> 24) & 0xff ;
offset += ENCRYPTED_MEMORY_EDATA_SIZE ;
memcpy(&edata[offset],clear_data,clear_data_size);
#ifdef CRYPTO_DEBUG
RSCRYPTO_DEBUG() << " clear item : " << RsUtil::BinToHex(&edata[offset],std::min(50,(int)total_data_size-(int)offset)) << "(...)" << std::endl;
#endif
uint32_t clear_item_offset = offset ;
offset += edata_size ;
uint32_t authentication_tag_offset = offset ;
assert(ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + offset == total_data_size) ;
//uint8_t encryption_key[32] ;
//deriveEncryptionKey(hash,encryption_key) ;
if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305)
librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ;
else if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256)
librs::crypto::AEAD_chacha20_sha256 (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],true) ;
else
return false ;
#ifdef CRYPTO_DEBUG
RSCRYPTO_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl;
RSCRYPTO_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE) << std::endl;
RSCRYPTO_DEBUG() << " final item : " << RsUtil::BinToHex(&edata[0],std::min(50u,total_data_size)) << "(...)" << std::endl;
#endif
return true ;
}
// Decrypts the given item using aead-chacha20-poly1305
bool decryptAuthenticateData(const unsigned char *encrypted_data,uint32_t encrypted_data_len,uint8_t *encryption_master_key, unsigned char *& decrypted_data, uint32_t& decrypted_data_size)
{
//uint8_t encryption_key[32] ;
//deriveEncryptionKey(hash,encryption_key) ;
uint8_t *edata = (uint8_t*)encrypted_data;
uint32_t offset = 0;
if(encrypted_data_len < ENCRYPTED_MEMORY_HEADER_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_EDATA_SIZE) return false ;
if(edata[0] != 0xae) return false ;
if(edata[1] != 0xad) return false ;
if(edata[2] != ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305 && edata[2] != ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256) return false ;
if(edata[3] != 0x01) return false ;
offset += ENCRYPTED_MEMORY_HEADER_SIZE ;
uint32_t aad_offset = offset ;
uint32_t aad_size = ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ;
uint8_t *initialization_vector = &edata[offset] ;
#ifdef CRYPTO_DEBUG
RSCRYPTO_DEBUG() << "ftServer::decrypting ft item." << std::endl;
RSCRYPTO_DEBUG() << " item data : " << RsUtil::BinToHex(edata,std::min(50u,encrypted_data_len) << "(...)" << std::endl;
RSCRYPTO_DEBUG() << " hash : " << hash << std::endl;
RSCRYPTO_DEBUG() << " encryption key : " << RsUtil::BinToHex(encryption_key,32) << std::endl;
RSCRYPTO_DEBUG() << " random nonce : " << RsUtil::BinToHex(initialization_vector,ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE) << std::endl;
#endif
offset += ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE ;
uint32_t edata_size = 0 ;
edata_size += ((uint32_t)edata[offset+0]) << 0 ;
edata_size += ((uint32_t)edata[offset+1]) << 8 ;
edata_size += ((uint32_t)edata[offset+2]) << 16 ;
edata_size += ((uint32_t)edata[offset+3]) << 24 ;
if(edata_size + ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_HEADER_SIZE != encrypted_data_len)
{
RSCRYPTO_ERROR() << " ERROR: encrypted data size is " << edata_size << ", should be " << encrypted_data_len - (ENCRYPTED_MEMORY_EDATA_SIZE + ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE + ENCRYPTED_MEMORY_INITIALIZATION_VECTOR_SIZE + ENCRYPTED_MEMORY_HEADER_SIZE ) << std::endl;
return false ;
}
offset += ENCRYPTED_MEMORY_EDATA_SIZE ;
uint32_t clear_item_offset = offset ;
uint32_t authentication_tag_offset = offset + edata_size ;
#ifdef CRYPTO_DEBUG
RSCRYPTO_DEBUG() << " authen. tag : " << RsUtil::BinToHex(&edata[authentication_tag_offset],ENCRYPTED_MEMORY_AUTHENTICATION_TAG_SIZE) << std::endl;
#endif
bool result ;
if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_POLY1305)
result = librs::crypto::AEAD_chacha20_poly1305(encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ;
else if(edata[2] == ENCRYPTED_MEMORY_FORMAT_AEAD_CHACHA20_SHA256)
result = librs::crypto::AEAD_chacha20_sha256 (encryption_master_key,initialization_vector,&edata[clear_item_offset],edata_size, &edata[aad_offset],aad_size, &edata[authentication_tag_offset],false) ;
else
return false ;
#ifdef CRYPTO_DEBUG
RSCRYPTO_DEBUG() << " authen. result : " << result << std::endl;
RSCRYPTO_DEBUG() << " decrypted daya : " << RsUtil::BinToHex(&edata[clear_item_offset],std::min(50u,edata_size)) << "(...)" << std::endl;
#endif
if(!result)
{
RSCRYPTO_ERROR() << "(EE) decryption/authentication went wrong." << std::endl;
return false ;
}
decrypted_data_size = edata_size ;
decrypted_data = (unsigned char*)rs_malloc(edata_size) ;
if(decrypted_data == NULL)
{
std::cerr << "Failed to allocate memory for decrypted data chunk of size " << edata_size << std::endl;
return false ;
}
memcpy(decrypted_data,&edata[clear_item_offset],edata_size) ;
return true ;
}
}
}

59
libretroshare/src/crypto/rscrypto.h Normal file
View file

@ -0,0 +1,59 @@
/*******************************************************************************
* libretroshare/src/crypto: crypto.h *
* *
* libretroshare: retroshare core library *
* *
* Copyright 2018 by Cyril Soler <csoler@users.sourceforge.net> *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU Lesser General Public License as *
* published by the Free Software Foundation, either version 3 of the *
* License, or (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU Lesser General Public License for more details. *
* *
* You should have received a copy of the GNU Lesser General Public License *
* along with this program. If not, see <https://www.gnu.org/licenses/>. *
* *
*******************************************************************************/
#include "crypto/chacha20.h"
namespace librs
{
namespace crypto
{
/*!
* \brief encryptAuthenticateData
Encrypts/decrypts data, using a autenticated construction + chacha20, based on a given 32 bytes master key. The actual encryption using a randomized key
based on a 96 bits IV. Input values are not touched (memory is not released). Memory ownership of outputs is left to the client.
* \param clear_data input data to encrypt
* \param clear_data_size length of input data
* \param encryption_master_key encryption master key of length 32 bytes.
* \param encrypted_data encrypted data, allocated using malloc
* \param encrypted_data_len length of encrypted data
* \return
* true if everything went well.
*/
bool encryptAuthenticateData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,unsigned char *& encrypted_data,uint32_t& encrypted_data_size);
/*!
* \brief decryptAuthenticateData
Encrypts/decrypts data, using a autenticated construction + chacha20, based on a given 32 bytes master key. The actual encryption using a randomized key
based on a 96 bits IV. Input values are not touched (memory is not released). Memory ownership of outputs is left to the client.
* \param encrypted_data input encrypted data
* \param encrypted_data_size input encrypted data length
* \param encryption_master_key encryption master key of length 32 bytes.
* \param decrypted_data decrypted data, allocated using malloc.
* \param decrypted_data_size length of allocated decrypted data.
* \return
* true if decryption + authentication are ok.
*/
bool decryptAuthenticateData(const unsigned char *encrypted_data,uint32_t encrypted_data_size, uint8_t* encryption_master_key, unsigned char *& decrypted_data,uint32_t& decrypted_data_size);
}
}

81
libretroshare/src/ft/ftserver.cc
View file

@ -1234,6 +1234,21 @@ static const uint8_t ENCRYPTED_FT_FORMAT_AEAD_CHACHA20_SHA256 = 0x02 ;
bool ftServer::encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHash& hash,RsTurtleGenericDataItem *& encrypted_item)
{
#ifndef USE_NEW_METHOD
uint32_t item_serialized_size = size(clear_item) ;
RsTemporaryMemory data(item_serialized_size) ;
if(data == NULL)
return false ;
serialise(clear_item, data, &item_serialized_size);
uint8_t encryption_key[32] ;
deriveEncryptionKey(hash,encryption_key) ;
return p3turtle::encryptData(data,item_serialized_size,encryption_key,encrypted_item) ;
#else
uint8_t initialization_vector[ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE] ;
RSRandom::random_bytes(initialization_vector,ENCRYPTED_FT_INITIALIZATION_VECTOR_SIZE) ;
@ -1312,12 +1327,34 @@ bool ftServer::encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHas
#endif
return true ;
#endif
}
// Decrypts the given item using aead-chacha20-poly1305
bool ftServer::decryptItem(RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item)
bool ftServer::decryptItem(const RsTurtleGenericDataItem *encrypted_item,const RsFileHash& hash,RsTurtleGenericTunnelItem *& decrypted_item)
{
#ifndef USE_NEW_METHOD
unsigned char *data = NULL ;
uint32_t data_size = 0 ;
uint8_t encryption_key[32] ;
deriveEncryptionKey(hash,encryption_key) ;
if(!p3turtle::decryptItem(encrypted_item,encryption_key,data,data_size))
{
FTSERVER_ERROR() << "Cannot decrypt data!" << std::endl;
if(data)
free(data) ;
return false ;
}
decrypted_item = dynamic_cast<RsTurtleGenericTunnelItem*>(deserialise(data,&data_size)) ;
free(data);
return (decrypted_item != NULL);
#else
uint8_t encryption_key[32] ;
deriveEncryptionKey(hash,encryption_key) ;
@ -1393,6 +1430,7 @@ bool ftServer::decryptItem(RsTurtleGenericDataItem *encrypted_item,const RsFileH
return false ;
return true ;
#endif
}
bool ftServer::encryptHash(const RsFileHash& hash, RsFileHash& hash_of_hash)
@ -1430,9 +1468,34 @@ bool ftServer::findRealHash(const RsFileHash& hash, RsFileHash& real_hash)
return false ;
}
TurtleSearchRequestId ftServer::turtleSearch(const std::string& string_to_match)
{
return mTurtleRouter->turtleSearch(string_to_match) ;
}
TurtleSearchRequestId ftServer::turtleSearch(const RsRegularExpression::LinearizedExpression& expr)
{
return mTurtleRouter->turtleSearch(expr) ;
}
#warning we should do this here, but for now it is done by turtle router.
// // Dont delete the item. The client (p3turtle) is doing it after calling this.
// //
// void ftServer::receiveSearchResult(RsTurtleSearchResultItem *item)
// {
// RsTurtleFTSearchResultItem *ft_sr = dynamic_cast<RsTurtleFTSearchResultItem*>(item) ;
//
// if(ft_sr == NULL)
// {
// FTSERVER_ERROR() << "(EE) ftServer::receiveSearchResult(): item cannot be cast to a RsTurtleFTSearchResultItem" << std::endl;
// return ;
// }
//
// RsServer::notify()->notifyTurtleSearchResult(ft_sr->request_id,ft_sr->result) ;
// }
// Dont delete the item. The client (p3turtle) is doing it after calling this.
//
void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
void ftServer::receiveTurtleData(const RsTurtleGenericTunnelItem *i,
const RsFileHash& hash,
const RsPeerId& virtual_peer_id,
RsTurtleGenericTunnelItem::Direction direction)
@ -1452,7 +1515,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
}
RsTurtleGenericTunnelItem *decrypted_item ;
if(!decryptItem(dynamic_cast<RsTurtleGenericDataItem *>(i),real_hash,decrypted_item))
if(!decryptItem(dynamic_cast<const RsTurtleGenericDataItem *>(i),real_hash,decrypted_item))
{
FTSERVER_ERROR() << "(EE) decryption error." << std::endl;
return ;
@ -1468,7 +1531,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
{
case RS_TURTLE_SUBTYPE_FILE_REQUEST:
{
RsTurtleFileRequestItem *item = dynamic_cast<RsTurtleFileRequestItem *>(i) ;
const RsTurtleFileRequestItem *item = dynamic_cast<const RsTurtleFileRequestItem *>(i) ;
if (item)
{
#ifdef SERVER_DEBUG
@ -1481,7 +1544,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
case RS_TURTLE_SUBTYPE_FILE_DATA :
{
RsTurtleFileDataItem *item = dynamic_cast<RsTurtleFileDataItem *>(i) ;
const RsTurtleFileDataItem *item = dynamic_cast<const RsTurtleFileDataItem *>(i) ;
if (item)
{
#ifdef SERVER_DEBUG
@ -1489,7 +1552,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
#endif
getMultiplexer()->recvData(virtual_peer_id,hash,0,item->chunk_offset,item->chunk_size,item->chunk_data) ;
item->chunk_data = NULL ; // this prevents deletion in the destructor of RsFileDataItem, because data will be deleted
const_cast<RsTurtleFileDataItem*>(item)->chunk_data = NULL ; // this prevents deletion in the destructor of RsFileDataItem, because data will be deleted
// down _ft_server->getMultiplexer()->recvData()...in ftTransferModule::recvFileData
}
}
@ -1497,7 +1560,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
case RS_TURTLE_SUBTYPE_FILE_MAP :
{
RsTurtleFileMapItem *item = dynamic_cast<RsTurtleFileMapItem *>(i) ;
const RsTurtleFileMapItem *item = dynamic_cast<const RsTurtleFileMapItem *>(i) ;
if (item)
{
#ifdef SERVER_DEBUG
@ -1520,7 +1583,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
case RS_TURTLE_SUBTYPE_CHUNK_CRC :
{
RsTurtleChunkCrcItem *item = dynamic_cast<RsTurtleChunkCrcItem *>(i) ;
const RsTurtleChunkCrcItem *item = dynamic_cast<const RsTurtleChunkCrcItem *>(i) ;
if (item)
{
#ifdef SERVER_DEBUG
@ -1533,7 +1596,7 @@ void ftServer::receiveTurtleData(RsTurtleGenericTunnelItem *i,
case RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST:
{
RsTurtleChunkCrcRequestItem *item = dynamic_cast<RsTurtleChunkCrcRequestItem *>(i) ;
const RsTurtleChunkCrcRequestItem *item = dynamic_cast<const RsTurtleChunkCrcRequestItem *>(i) ;
if (item)
{
#ifdef SERVER_DEBUG

10
libretroshare/src/ft/ftserver.h
View file

@ -95,8 +95,11 @@ public:
// Implements RsTurtleClientService
//
uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
virtual bool handleTunnelRequest(const RsFileHash& hash,const RsPeerId& peer_id) ;
virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
//virtual void receiveSearchResult(RsTurtleSearchResultItem *item);// TODO
virtual RsItem *create_item(uint16_t service,uint8_t item_type) const ;
virtual RsServiceSerializer *serializer() { return this ; }
@ -143,6 +146,9 @@ public:
virtual void setFilePermDirectDL(uint32_t perm) ;
virtual uint32_t filePermDirectDL() ;
virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
/***
* Control of Downloads Priority.
***/
@ -250,7 +256,7 @@ public:
static void deriveEncryptionKey(const RsFileHash& hash, uint8_t *key);
bool encryptItem(RsTurtleGenericTunnelItem *clear_item,const RsFileHash& hash,RsTurtleGenericDataItem *& encrypted_item);
bool decryptItem(RsTurtleGenericDataItem *encrypted_item, const RsFileHash& hash, RsTurtleGenericTunnelItem *&decrypted_item);
bool decryptItem(const RsTurtleGenericDataItem *encrypted_item, const RsFileHash& hash, RsTurtleGenericTunnelItem *&decrypted_item);
/*************** Internal Transfer Fns *************************/
virtual int tick();

7
libretroshare/src/grouter/p3grouter.cc
View file

@ -482,7 +482,7 @@ void p3GRouter::handleLowLevelTransactionAckItem(RsGRouterTransactionAcknItem *t
#endif
}
void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/)
void p3GRouter::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/)
{
#ifdef GROUTER_DEBUG
std::cerr << "p3GRouter::receiveTurtleData() " << std::endl;
@ -496,7 +496,7 @@ void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFile
// - possibly packs multi-item blocks back together
// - converts it into a grouter generic item (by deserialising it)
RsTurtleGenericDataItem *item = dynamic_cast<RsTurtleGenericDataItem*>(gitem) ;
const RsTurtleGenericDataItem *item = dynamic_cast<const RsTurtleGenericDataItem*>(gitem) ;
if(item == NULL)
{
@ -510,7 +510,8 @@ void p3GRouter::receiveTurtleData(RsTurtleGenericTunnelItem *gitem, const RsFile
// Items come out of the pipe in order. We need to recover all chunks before we de-serialise the content and have it handled by handleIncoming()
RsItem *itm = RsGRouterSerialiser().deserialise(item->data_bytes,&item->data_size) ;
uint32_t size = item->data_size ;
RsItem *itm = RsGRouterSerialiser().deserialise(item->data_bytes,&size);
if(itm == NULL)
{

3
libretroshare/src/grouter/p3grouter.h
View file

@ -208,8 +208,9 @@ protected:
// Interaction with turtle router //
//===================================================//
uint16_t serviceId() const { return RS_SERVICE_TYPE_GROUTER; }
virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) ;
virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/);
virtual void receiveTurtleData(const RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/);
virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) ;
virtual void removeVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id) ;

51
libretroshare/src/gxs/rsgenexchange.cc
View file

@ -64,7 +64,7 @@ static const uint32_t INDEX_AUTHEN_ADMIN = 0x00000040; // admin key
#define GXS_MASK "GXS_MASK_HACK"
//#define GEN_EXCH_DEBUG 1
#define GEN_EXCH_DEBUG 1
static const uint32_t MSG_CLEANUP_PERIOD = 60*59; // 59 minutes
static const uint32_t INTEGRITY_CHECK_PERIOD = 60*2; // 31 minutes // TODO: Restore this line before merging deep_search
@ -1111,6 +1111,8 @@ void RsGenExchange::receiveChanges(std::vector<RsGxsNotify*>& changes)
RsGxsNotify* n = *vit;
RsGxsGroupChange* gc;
RsGxsMsgChange* mc;
RsGxsDistantSearchResultChange *gt;
if((mc = dynamic_cast<RsGxsMsgChange*>(n)) != NULL)
{
if (mc->metaChange())
@ -1133,6 +1135,10 @@ void RsGenExchange::receiveChanges(std::vector<RsGxsNotify*>& changes)
out.mGrps.splice(out.mGrps.end(), gc->mGrpIdList);
}
}
else if((gt = dynamic_cast<RsGxsDistantSearchResultChange*>(n)) != NULL)
{
out.mDistantSearchReqs.push_back(gt->mRequestId);
}
else
std::cerr << "(EE) Unknown changes type!!" << std::endl;
@ -1568,7 +1574,7 @@ bool RsGenExchange::setAuthenPolicyFlag(const uint8_t &msgFlag, uint32_t& authen
return true;
}
void RsGenExchange::notifyNewGroups(std::vector<RsNxsGrp *> &groups)
void RsGenExchange::receiveNewGroups(std::vector<RsNxsGrp *> &groups)
{
RS_STACK_MUTEX(mGenMtx) ;
@ -1600,7 +1606,7 @@ void RsGenExchange::notifyNewGroups(std::vector<RsNxsGrp *> &groups)
}
void RsGenExchange::notifyNewMessages(std::vector<RsNxsMsg *>& messages)
void RsGenExchange::receiveNewMessages(std::vector<RsNxsMsg *>& messages)
{
RS_STACK_MUTEX(mGenMtx) ;
@ -1635,11 +1641,22 @@ void RsGenExchange::notifyNewMessages(std::vector<RsNxsMsg *>& messages)
}
}
}
void RsGenExchange::receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId)
{
RS_STACK_MUTEX(mGenMtx);
RsGxsDistantSearchResultChange* gc = new RsGxsDistantSearchResultChange(id,grpId);
mNotifications.push_back(gc);
std::cerr << "RsGenExchange::receiveDistantSearchResults(): received result for request " << std::hex << id << std::dec << std::endl;
}
void RsGenExchange::notifyReceivePublishKey(const RsGxsGroupId &grpId)
{
RS_STACK_MUTEX(mGenMtx);
RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISHKEY, true);
RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY, true);
gc->mGrpIdList.push_back(grpId);
mNotifications.push_back(gc);
}
@ -2310,7 +2327,7 @@ void RsGenExchange::publishMsgs()
if(!msgChangeMap.empty())
{
RsGxsMsgChange* ch = new RsGxsMsgChange(RsGxsNotify::TYPE_PUBLISH, false);
RsGxsMsgChange* ch = new RsGxsMsgChange(RsGxsNotify::TYPE_PUBLISHED, false);
ch->msgChangeMap = msgChangeMap;
mNotifications.push_back(ch);
}
@ -2447,7 +2464,7 @@ void RsGenExchange::processGroupDelete()
if(!grpDeleted.empty())
{
RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISH, false);
RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_PUBLISHED, false);
gc->mGrpIdList = grpDeleted;
mNotifications.push_back(gc);
}
@ -2756,7 +2773,7 @@ void RsGenExchange::publishGrps()
if(!grpChanged.empty())
{
RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, true);
RsGxsGroupChange* gc = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, true);
gc->mGrpIdList = grpChanged;
mNotifications.push_back(gc);
#ifdef GEN_EXCH_DEBUG
@ -3022,7 +3039,7 @@ void RsGenExchange::processRecvdMessages()
#endif
mDataStore->storeMessage(msgs_to_store);
RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_RECEIVE, false);
RsGxsMsgChange* c = new RsGxsMsgChange(RsGxsNotify::TYPE_RECEIVED_NEW, false);
c->msgChangeMap = msgIds;
mNotifications.push_back(c);
}
@ -3155,7 +3172,7 @@ void RsGenExchange::processRecvdGroups()
if(!grpIds.empty())
{
RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, false);
RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, false);
c->mGrpIdList = grpIds;
mNotifications.push_back(c);
mDataStore->storeGroup(grps_to_store);
@ -3239,7 +3256,7 @@ void RsGenExchange::performUpdateValidation()
}
// notify the client
RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVE, true);
RsGxsGroupChange* c = new RsGxsGroupChange(RsGxsNotify::TYPE_RECEIVED_NEW, true);
for(uint32_t i=0;i<mGroupUpdates.size();++i)
if(mGroupUpdates[i].newGrp != NULL)
@ -3335,11 +3352,11 @@ void RsGenExchange::removeDeleteExistingMessages( std::list<RsNxsMsg*>& msgs, Gx
mDataStore->retrieveMsgIds(*it, msgIdReq[*it]);
#ifdef GEN_EXCH_DEBUG
const std::vector<RsGxsMessageId>& vec(msgIdReq[*it]) ;
const std::set<RsGxsMessageId>& vec(msgIdReq[*it]) ;
std::cerr << " retrieved " << vec.size() << " message ids for group " << *it << std::endl;
for(uint32_t i=0;i<vec.size();++i)
std::cerr << " " << vec[i] << std::endl;
for(auto it2(vec.begin());it2!=vec.end();++it2)
std::cerr << " " << *it2 << std::endl;
#endif
}
@ -3381,3 +3398,11 @@ void RsGenExchange::removeDeleteExistingMessages( std::list<RsNxsMsg*>& msgs, Gx
}
}
void RsGenExchange::turtleGroupRequest(const RsGxsGroupId& group_id)
{
mNetService->turtleGroupRequest(group_id) ;
}
void RsGenExchange::turtleSearchRequest(const std::string& match_string)
{
mNetService->turtleSearchRequest(match_string) ;
}

29
libretroshare/src/gxs/rsgenexchange.h
View file

@ -121,9 +121,9 @@ public:
virtual ~RsGenExchange();
// Convention that this is implemented here.
// Convention that this is implemented here.
// and passes to network service.
virtual RsServiceInfo getServiceInfo() = 0;
virtual RsServiceInfo getServiceInfo() = 0;
void setNetworkExchangeService(RsNetworkExchangeService *ns) ;
@ -132,18 +132,24 @@ public:
/*!
* @param messages messages are deleted after function returns
*/
virtual void notifyNewMessages(std::vector<RsNxsMsg*>& messages);
virtual void receiveNewMessages(std::vector<RsNxsMsg*>& messages);
/*!
* @param groups groups are deleted after function returns
*/
virtual void notifyNewGroups(std::vector<RsNxsGrp*>& groups);
virtual void receiveNewGroups(std::vector<RsNxsGrp*>& groups);
/*!
* @param grpId group id
*/
virtual void notifyReceivePublishKey(const RsGxsGroupId &grpId);
/*!
* \brief notifyReceiveDistantSearchResults
* Should be called when new search results arrive.
* \param grpId
*/
virtual void receiveDistantSearchResults(TurtleRequestId id,const RsGxsGroupId &grpId);
/*!
* @param grpId group id
*/
@ -287,6 +293,15 @@ public:
*/
bool getGroupStatistic(const uint32_t& token, GxsGroupStatistic& stats);
/*!
* \brief turtleGroupRequest
* Issues a browadcast group request using the turtle router generic search system. The request is obviously asynchroneous and will be
* handled in RsGenExchange when received.
* \param group_id
*/
void turtleGroupRequest(const RsGxsGroupId& group_id);
void turtleSearchRequest(const std::string& match_string);
protected:
bool messagePublicationTest(const RsGxsMsgMetaData&) ;
@ -566,7 +581,7 @@ public:
* @param msgs
*/
void deleteMsgs(uint32_t& token, const GxsMsgReq& msgs);
protected:
/*!
* This represents the group before its signature is calculated
@ -650,7 +665,7 @@ public:
*/
void shareGroupPublishKey(const RsGxsGroupId& grpId,const std::set<RsPeerId>& peers) ;
/*!
* Returns the local TS of the group as known by the network service.
* This is useful to allow various network services to sync their update TS
@ -755,6 +770,8 @@ protected:
*/
int createMessage(RsNxsMsg* msg);
RsNetworkExchangeService *netService() const { return mNetService ; }
private:
/*!
* convenience function to create sign

17
libretroshare/src/gxs/rsgxsdataaccess.cc
View file

@ -1807,6 +1807,23 @@ bool RsGxsDataAccess::updateGroupData(RsNxsGrp* grp) {
return mDataStore->updateGroup(grpM);
}
bool RsGxsDataAccess::getGroupData(const RsGxsGroupId& grpId, RsNxsGrp *& grp_data)
{
RsStackMutex stack(mDataMutex);
std::map<RsGxsGroupId, RsNxsGrp*> grps ;
grps[grpId] = NULL ;
if(mDataStore->retrieveNxsGrps(grps, false, true)) // the false here is very important: it removes the private key parts.
{
grp_data = grps.begin()->second;
return true;
}
else
return false ;
}
bool RsGxsDataAccess::addMsgData(RsNxsMsg* msg) {
RsStackMutex stack(mDataMutex);

8
libretroshare/src/gxs/rsgxsdataaccess.h
View file

@ -156,6 +156,14 @@ public:
*/
bool addMsgData(RsNxsMsg* msg);
/*!
* This retrieves a group from the gxs data base, this is a blocking call \n
* @param grp the group to add, memory ownership passed to the callee
* @return false if group cound not be retrieved
*/
bool getGroupData(const RsGxsGroupId& grpId,RsNxsGrp *& grp_data);
public:

658
libretroshare/src/gxs/rsgxsnetservice.cc
View file

@ -166,8 +166,6 @@
// | |
// (Only send if rand() < sendingProb()) +---comes from mClientMsgUpdateMap -----+
//
//
//
// Suggestions
// ===========
// * handleRecvSyncGroup should use mit->second.mLastPost to limit the sending of already known data
@ -194,6 +192,53 @@
// Problem: without msg, we cannot know the grpId!!
//
// * mClientMsgUpdateMap[peerid][grpId] is only updated when new msgs are received. Up to date groups will keep asking for lists!
//
// Distant sync
// ============
//
// Distant sync uses tunnels to sync subscribed GXS groups that are not supplied by friends. Peers can subscribe to a GXS group using a RS link
// which GXS uses to request updates through tunnels.
// * The whole exchange should be kept private and anonymous between the two distant peers, so we use the same trick than for FT: encrypt the data using the group ID.
// * The same node shouldn't be known as a common server for different GXS groups
//
// GXS net service:
// * talks to virtual peers, treated like normal peers
// * virtual peers only depend on the server ID, not on tunnel ID, and be kept constant accross time so that ClientGroupUpdateMap is kept consistent
// * does not use tunnels if friends can already supply the data (??) This causes issues with "islands".
//
// Tunnels:
// * a specific service named GxsSyncTunnelService handles the creation/management of sync tunnels:
// * tunnel data need to be encrypted.
//
// bool manageTunnels(const RsGxsGroupId&) ; // start managing tunnels for this group
// bool releaseTunnels(const RsGxsGroupId&) ; // stop managing tunnels for this group
// bool sendData(const unsigned char *data,uint32_t size,const RsPeerId& virtual_peer) ; // send data to this virtual peer
// bool getVirtualPeers(const RsGxsGroupId&, std::list<RsPeerId>& peers) ; // returns the virtual peers for this group
//
// Proposed protocol:
// * request tunnels based on H(GroupId)
// * encrypt tunnel data using chacha20+HMAC-SHA256 using AEAD( GroupId, 96bits IV, tunnel ID ) (similar to what FT does)
// * when tunnel is established, exchange virtual peer names: vpid = H( GroupID | Random bias )
// * when vpid is known, notify the client (GXS net service) which can use the virtual peer to sync
//
// * only use a single tunnel per virtual peer ID
//
// Client ------------------ TR(H(GroupId)) --------------> Server
//
// Client <-------------------- T OK ---------------------- Server
//
// [Encrypted traffic using H(GroupId, 96bits IV, tunnel ID)]
//
// Client <--------- VPID = H( Random IV | GroupId ) ------ Server
// | |
// +--------------> Mark the virtual peer active <-----------+
//
// Unsolved problems:
// * if we want to preserve anonymity, we cannot prevent GXS from duplicating the data from virtual/real peers that actually are the same peers.
// * ultimately we should only use tunnels to sync GXS. The mix between tunnels and real peers is not a problem but will cause unnecessary traffic.
//
// Notes:
// * given that GXS only talks to peers once every 2 mins, it's likely that keep-alive packets will be needed
#include <unistd.h>
@ -209,6 +254,7 @@
#include "retroshare/rsgxscircles.h"
#include "retroshare/rspeers.h"
#include "pgp/pgpauxutils.h"
#include "crypto/rscrypto.h"
#include "util/rsdir.h"
#include "util/rstime.h"
#include "util/rsmemory.h"
@ -224,6 +270,7 @@
NXS_NET_DEBUG_5 summary of transactions (useful to just know what comes in/out)
NXS_NET_DEBUG_6 group sync statistics (e.g. number of posts at nighbour nodes, etc)
NXS_NET_DEBUG_7 encryption/decryption of transactions
NXS_NET_DEBUG_8 gxs distant sync
***/
//#define NXS_NET_DEBUG_0 1
@ -234,6 +281,7 @@
//#define NXS_NET_DEBUG_5 1
//#define NXS_NET_DEBUG_6 1
//#define NXS_NET_DEBUG_7 1
#define NXS_NET_DEBUG_8 1
//#define NXS_FRAG
@ -256,6 +304,7 @@ static const uint32_t REJECTED_MESSAGE_RETRY_DELAY = 24*3600; //
static const uint32_t GROUP_STATS_UPDATE_DELAY = 240; // update unsubscribed group statistics every 3 mins
static const uint32_t GROUP_STATS_UPDATE_NB_PEERS = 2; // number of peers to which the group stats are asked
static const uint32_t MAX_ALLOWED_GXS_MESSAGE_SIZE = 199000; // 200,000 bytes including signature and headers
static const uint32_t MIN_DELAY_BETWEEN_GROUP_SEARCH = 40; // dont search same group more than every 40 secs.
static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_UNKNOWN = 0x00 ;
static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_NO_ERROR = 0x01 ;
@ -267,11 +316,12 @@ static const uint32_t RS_NXS_ITEM_ENCRYPTION_STATUS_GXS_KEY_MISSING = 0x05 ;
// Debug system to allow to print only for some IDs (group, Peer, etc)
#if defined(NXS_NET_DEBUG_0) || defined(NXS_NET_DEBUG_1) || defined(NXS_NET_DEBUG_2) || defined(NXS_NET_DEBUG_3) \
|| defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6) || defined(NXS_NET_DEBUG_7)
|| defined(NXS_NET_DEBUG_4) || defined(NXS_NET_DEBUG_5) || defined(NXS_NET_DEBUG_6) || defined(NXS_NET_DEBUG_7) \
|| defined(NXS_NET_DEBUG_8)
static const RsPeerId peer_to_print = RsPeerId(std::string("")) ;
static const RsGxsGroupId group_id_to_print = RsGxsGroupId(std::string("")) ; // use this to allow to this group id only, or "" for all IDs
static const uint32_t service_to_print = RS_SERVICE_TYPE_GXS_TRANS ; // use this to allow to this service id only, or 0 for all services
static const uint32_t service_to_print = RS_SERVICE_GXS_TYPE_CHANNELS ; // use this to allow to this service id only, or 0 for all services
// warning. Numbers should be SERVICE IDS (see serialiser/rsserviceids.h. E.g. 0x0215 for forums)
class nullstream: public std::ostream {};
@ -288,7 +338,6 @@ static std::string nice_time_stamp(time_t now,time_t TS)
}
}
static std::ostream& gxsnetdebug(const RsPeerId& peer_id,const RsGxsGroupId& grp_id,uint32_t service_type)
{
static nullstream null ;
@ -314,7 +363,8 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds,
RsNxsNetMgr *netMgr, RsNxsObserver *nxsObs,
const RsServiceInfo serviceInfo,
RsGixsReputation* reputations, RsGcxs* circles, RsGixs *gixs,
PgpAuxUtils *pgpUtils, bool grpAutoSync, bool msgAutoSync, uint32_t default_store_period, uint32_t default_sync_period)
PgpAuxUtils *pgpUtils, RsGxsNetTunnelService *mGxsNT,
bool grpAutoSync, bool msgAutoSync, bool distSync, uint32_t default_store_period, uint32_t default_sync_period)
: p3ThreadedService(), p3Config(), mTransactionN(0),
mObserver(nxsObs), mDataStore(gds),
mServType(servType), mTransactionTimeOut(TRANSAC_TIMEOUT),
@ -322,8 +372,8 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds,
mSyncTs(0), mLastKeyPublishTs(0),
mLastCleanRejectedMessages(0), mSYNC_PERIOD(SYNC_PERIOD),
mCircles(circles), mGixs(gixs),
mReputations(reputations), mPgpUtils(pgpUtils),
mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync),
mReputations(reputations), mPgpUtils(pgpUtils), mGxsNetTunnel(mGxsNT),
mGrpAutoSync(grpAutoSync), mAllowMsgSync(msgAutoSync),mAllowDistSync(distSync),
mServiceInfo(serviceInfo), mDefaultMsgStorePeriod(default_store_period),
mDefaultMsgSyncPeriod(default_sync_period)
{
@ -331,6 +381,8 @@ RsGxsNetService::RsGxsNetService(uint16_t servType, RsGeneralDataService *gds,
mOwnId = mNetMgr->getOwnId();
mUpdateCounter = 0;
mLastCacheReloadTS = 0;
// check the consistency
if(mDefaultMsgStorePeriod > 0 && mDefaultMsgSyncPeriod > mDefaultMsgStorePeriod)
@ -379,8 +431,7 @@ int RsGxsNetService::tick()
{
// always check for new items arriving
// from peers
if(receivedItems())
recvNxsItemQueue();
recvNxsItemQueue();
bool should_notify = false;
@ -403,6 +454,7 @@ int RsGxsNetService::tick()
{
syncWithPeers();
syncGrpStatistics();
checkDistantSyncState();
mSyncTs = now;
}
@ -450,8 +502,8 @@ void RsGxsNetService::processObserverNotifications()
mNewPublishKeysToNotify.clear() ;
}
if(!grps_copy.empty()) mObserver->notifyNewGroups (grps_copy);
if(!msgs_copy.empty()) mObserver->notifyNewMessages(msgs_copy);
if(!grps_copy.empty()) mObserver->receiveNewGroups (grps_copy);
if(!msgs_copy.empty()) mObserver->receiveNewMessages(msgs_copy);
for(std::set<RsGxsGroupId>::const_iterator it(keys_copy.begin());it!=keys_copy.end();++it)
mObserver->notifyReceivePublishKey(*it);
@ -520,6 +572,18 @@ void RsGxsNetService::syncWithPeers()
std::set<RsPeerId> peers;
mNetMgr->getOnlineList(mServiceInfo.mServiceType, peers);
if(mAllowDistSync && mGxsNetTunnel != NULL)
{
// Grab all online virtual peers of distant tunnels for the current service.
std::list<RsGxsNetTunnelVirtualPeerId> vpids ;
mGxsNetTunnel->getVirtualPeers(vpids);
for(auto it(vpids.begin());it!=vpids.end();++it)
peers.insert(RsPeerId(*it)) ;
}
if (peers.empty()) {
// nothing to do
return;
@ -549,7 +613,7 @@ void RsGxsNetService::syncWithPeers()
#ifdef NXS_NET_DEBUG_5
GXSNETDEBUG_P_(*sit) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global group TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) to himself" << std::endl;
#endif
sendItem(grp);
generic_sendItem(grp);
}
if(!mAllowMsgSync)
@ -665,7 +729,7 @@ void RsGxsNetService::syncWithPeers()
#ifdef NXS_NET_DEBUG_7
GXSNETDEBUG_PG(*sit,grpId) << " Service " << std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself - in clear " << std::endl;
#endif
sendItem(msg);
generic_sendItem(msg);
#ifdef NXS_NET_DEBUG_5
GXSNETDEBUG_PG(*sit,grpId) << "Service "<< std::hex << ((mServiceInfo.mServiceType >> 8)& 0xffff) << std::dec << " sending global message TS of peer id: " << *sit << " ts=" << nice_time_stamp(time(NULL),updateTS) << " (secs ago) for group " << grpId << " to himself" << std::endl;
@ -676,6 +740,91 @@ void RsGxsNetService::syncWithPeers()
#endif
}
void RsGxsNetService::generic_sendItem(RsNxsItem *si)
{
// check if the item is to be sent to a distant peer or not
RsGxsGroupId tmp_grpId;
if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer( static_cast<RsGxsNetTunnelVirtualPeerId>(si->PeerId()),tmp_grpId))
{
RsNxsSerialiser ser(mServType);
uint32_t size = ser.size(si);
unsigned char *mem = (unsigned char *)rs_malloc(size) ;
if(!mem)
return ;
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG_P_(si->PeerId()) << "Sending RsGxsNetTunnelService Item:" << (void*)si << " of type: " << std::hex << si->PacketId() << std::dec
<< " transaction " << si->transactionNumber << " to virtual peer " << si->PeerId() << std::endl ;
#endif
ser.serialise(si,mem,&size) ;
mGxsNetTunnel->sendTunnelData(mServType,mem,size,static_cast<RsGxsNetTunnelVirtualPeerId>(si->PeerId()));
}
else
sendItem(si) ;
}
void RsGxsNetService::checkDistantSyncState()
{
if(!mAllowDistSync || mGxsNetTunnel==NULL || !mGrpAutoSync)
return ;
RsGxsGrpMetaTemporaryMap grpMeta;
mDataStore->retrieveGxsGrpMetaData(grpMeta);
// Go through group statistics and groups without information are re-requested to random peers selected
// among the ones who provided the group info.
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___<< "Checking distant sync for all groups." << std::endl;
#endif
// get the list of online peers
std::set<RsPeerId> online_peers;
mNetMgr->getOnlineList(mServiceInfo.mServiceType , online_peers);
RS_STACK_MUTEX(mNxsMutex) ;
for(auto it(grpMeta.begin());it!=grpMeta.end();++it)
if(it->second->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED) // we only consider subscribed groups here.
{
#warning (cyril) We might need to also remove peers for recently unsubscribed groups
const RsGxsGroupId& grpId(it->first);
const RsGxsGrpConfig& rec = locked_getGrpConfig(grpId) ;
#ifdef NXS_NET_DEBUG_6
GXSNETDEBUG__G(it->first) << " group " << grpId;
#endif
bool at_least_one_friend_is_supplier = false ;
for(auto it2(rec.suppliers.ids.begin());it2!=rec.suppliers.ids.end() && !at_least_one_friend_is_supplier;++it2)
if(online_peers.find(*it2) != online_peers.end()) // check that the peer is online
at_least_one_friend_is_supplier = true ;
// That strategy is likely to create islands of friends connected to each other. There's no real way
// to decide what to do here, except maybe checking the last message TS remotely vs. locally.
if(at_least_one_friend_is_supplier)
{
mGxsNetTunnel->releaseDistantPeers(mServType,grpId);
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___<< " Group " << grpId << ": suppliers among friends. Releasing peers." << std::endl;
#endif
}
else
{
mGxsNetTunnel->requestDistantPeers(mServType,grpId);
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___<< " Group " << grpId << ": no suppliers among friends. Requesting peers." << std::endl;
#endif
}
}
}
void RsGxsNetService::syncGrpStatistics()
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -690,6 +839,17 @@ void RsGxsNetService::syncGrpStatistics()
std::set<RsPeerId> online_peers;
mNetMgr->getOnlineList(mServiceInfo.mServiceType, online_peers);
if(mAllowDistSync && mGxsNetTunnel != NULL)
{
// Grab all online virtual peers of distant tunnels for the current service.
std::list<RsGxsNetTunnelVirtualPeerId> vpids ;
mGxsNetTunnel->getVirtualPeers(vpids);
for(auto it(vpids.begin());it!=vpids.end();++it)
online_peers.insert(RsPeerId(*it)) ;
}
// Go through group statistics and groups without information are re-requested to random peers selected
// among the ones who provided the group info.
@ -704,44 +864,44 @@ void RsGxsNetService::syncGrpStatistics()
#endif
if(rec.statistics_update_TS + GROUP_STATS_UPDATE_DELAY < now && rec.suppliers.ids.size() > 0)
{
{
#ifdef NXS_NET_DEBUG_6
GXSNETDEBUG__G(it->first) << " needs update. Randomly asking to some friends" << std::endl;
GXSNETDEBUG__G(it->first) << " needs update. Randomly asking to some friends" << std::endl;
#endif
// randomly select GROUP_STATS_UPDATE_NB_PEERS friends among the suppliers of this group
// randomly select GROUP_STATS_UPDATE_NB_PEERS friends among the suppliers of this group
uint32_t n = RSRandom::random_u32() % rec.suppliers.ids.size() ;
uint32_t n = RSRandom::random_u32() % rec.suppliers.ids.size() ;
std::set<RsPeerId>::const_iterator rit = rec.suppliers.ids.begin();
for(uint32_t i=0;i<n;++i)
++rit ;
std::set<RsPeerId>::const_iterator rit = rec.suppliers.ids.begin();
for(uint32_t i=0;i<n;++i)
++rit ;
for(uint32_t i=0;i<std::min(rec.suppliers.ids.size(),(size_t)GROUP_STATS_UPDATE_NB_PEERS);++i)
{
// we started at a random position in the set, wrap around if the end is reached
if(rit == rec.suppliers.ids.end())
rit = rec.suppliers.ids.begin() ;
RsPeerId peer_id = *rit ;
++rit ;
if(online_peers.find(peer_id) != online_peers.end()) // check that the peer is online
for(uint32_t i=0;i<std::min(rec.suppliers.ids.size(),(size_t)GROUP_STATS_UPDATE_NB_PEERS);++i)
{
// we started at a random position in the set, wrap around if the end is reached
if(rit == rec.suppliers.ids.end())
rit = rec.suppliers.ids.begin() ;
RsPeerId peer_id = *rit ;
++rit ;
if(online_peers.find(peer_id) != online_peers.end()) // check that the peer is online
{
#ifdef NXS_NET_DEBUG_6
GXSNETDEBUG_PG(peer_id,it->first) << " asking friend " << peer_id << " for an update of stats for group " << it->first << std::endl;
GXSNETDEBUG_PG(peer_id,it->first) << " asking friend " << peer_id << " for an update of stats for group " << it->first << std::endl;
#endif
RsNxsSyncGrpStatsItem *grs = new RsNxsSyncGrpStatsItem(mServType) ;
RsNxsSyncGrpStatsItem *grs = new RsNxsSyncGrpStatsItem(mServType) ;
grs->request_type = RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_REQUEST ;
grs->request_type = RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_REQUEST ;
grs->grpId = it->first ;
grs->PeerId(peer_id) ;
grs->grpId = it->first ;
grs->PeerId(peer_id) ;
sendItem(grs) ;
generic_sendItem(grs) ;
}
}
}
}
}
#ifdef NXS_NET_DEBUG_6
else
GXSNETDEBUG__G(it->first) << " up to date." << std::endl;
@ -819,7 +979,7 @@ void RsGxsNetService::handleRecvSyncGrpStatistics(RsNxsSyncGrpStatsItem *grs)
GXSNETDEBUG_PG(grs->PeerId(),grs->grpId) << " sending back statistics item with " << vec.size() << " elements." << std::endl;
#endif
sendItem(grs_resp) ;
generic_sendItem(grs_resp) ;
}
else if(grs->request_type == RsNxsSyncGrpStatsItem::GROUP_INFO_TYPE_RESPONSE)
{
@ -1391,11 +1551,11 @@ public:
void operator() (RsItem* item)
{
RsGxsMsgUpdateItem* mui;
RsGxsGrpUpdateItem* gui;
RsGxsServerGrpUpdateItem* gsui;
RsGxsServerMsgUpdateItem* msui;
RsGxsGrpConfigItem* mgci;
RsGxsMsgUpdateItem *mui;
RsGxsGrpUpdateItem *gui;
RsGxsServerGrpUpdateItem *gsui;
RsGxsServerMsgUpdateItem *msui;
RsGxsGrpConfigItem *mgci;
if((mui = dynamic_cast<RsGxsMsgUpdateItem*>(item)) != NULL)
check_store(mui->peerID,mClientMsgMap,*mui);
@ -1421,7 +1581,6 @@ private:
RsGxsNetService::GrpConfigMap& mGrpConfigMap;
RsGxsServerGrpUpdate& mServerGrpUpdate;
};
bool RsGxsNetService::loadList(std::list<RsItem *> &load)
@ -1431,11 +1590,10 @@ bool RsGxsNetService::loadList(std::list<RsItem *> &load)
// The delete is done in StoreHere, if necessary
std::for_each(load.begin(), load.end(), StoreHere(mClientGrpUpdateMap, mClientMsgUpdateMap, mServerMsgUpdateMap, mServerGrpConfigMap, mGrpServerUpdate));
time_t now = time(NULL);
// We reset group statistics here. This is the best place since we know at this point which are all unsubscribed groups.
time_t now = time(NULL);
for(GrpConfigMap::iterator it(mServerGrpConfigMap.begin());it!=mServerGrpConfigMap.end();++it)
{
// At each reload, we reset the count of visible messages. It will be rapidely restored to its real value from friends.
@ -1489,6 +1647,7 @@ struct get_second : public std::unary_function<typename UpdateMap::value_type, R
typename UpdateMap::key_type ItemClass::*ID_member ;
};
bool RsGxsNetService::saveList(bool& cleanup, std::list<RsItem*>& save)
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -1520,11 +1679,44 @@ RsSerialiser *RsGxsNetService::setupSerialiser()
return rss;
}
RsItem *RsGxsNetService::generic_recvItem()
{
{
RsItem *item ;
if(NULL != (item=recvItem()))
return item ;
}
unsigned char *data = NULL ;
uint32_t size = 0 ;
RsGxsNetTunnelVirtualPeerId virtual_peer_id ;
while(mAllowDistSync && mGxsNetTunnel!=NULL && mGxsNetTunnel->receiveTunnelData(mServType,data,size,virtual_peer_id))
{
RsNxsItem *item = dynamic_cast<RsNxsItem*>(RsNxsSerialiser(mServType).deserialise(data,&size)) ;
item->PeerId(virtual_peer_id) ;
free(data) ;
if(!item)
continue ;
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetTunnelService Item:" << (void*)item << " of type: " << std::hex << item->PacketId() << std::dec
<< " transaction " << item->transactionNumber << " from virtual peer " << item->PeerId() << std::endl ;
#endif
return item ;
}
return NULL ;
}
void RsGxsNetService::recvNxsItemQueue()
{
RsItem *item ;
while(NULL != (item=recvItem()))
while(NULL != (item=generic_recvItem()))
{
#ifdef NXS_NET_DEBUG_1
GXSNETDEBUG_P_(item->PeerId()) << "Received RsGxsNetService Item:" << (void*)item << " type=" << std::hex << item->PacketId() << std::dec << std::endl ;
@ -1829,7 +2021,7 @@ void RsGxsNetService::debugDump()
GXSNETDEBUG___<< "RsGxsNetService::debugDump():" << std::endl;
RsGxsMetaDataTemporaryMap<RsGxsGrpMetaData> grpMetas;
RsGxsGrpMetaTemporaryMap grpMetas;
if(!group_id_to_print.isNull())
grpMetas[group_id_to_print] = NULL ;
@ -1842,7 +2034,7 @@ void RsGxsNetService::debugDump()
for(ServerMsgMap::const_iterator it(mServerMsgUpdateMap.begin());it!=mServerMsgUpdateMap.end();++it)
{
RsGxsMetaDataTemporaryMap<RsGxsGrpMetaData>::const_iterator it2 = grpMetas.find(it->first) ;
RsGxsGrpMetaTemporaryMap::const_iterator it2 = grpMetas.find(it->first) ;
RsGxsGrpMetaData *grpMeta = (it2 != grpMetas.end())? it2->second : NULL;
std::string subscribe_string = (grpMeta==NULL)?"Unknown" : ((grpMeta->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED)?" Subscribed":" NOT Subscribed") ;
@ -2125,7 +2317,7 @@ void RsGxsNetService::processTransactions()
lit_end = tr->mItems.end();
for(; lit != lit_end; ++lit){
sendItem(*lit);
generic_sendItem(*lit);
}
tr->mItems.clear(); // clear so they don't get deleted in trans cleaning
@ -2234,7 +2426,7 @@ void RsGxsNetService::processTransactions()
trans->transactFlag = RsNxsTransacItem::FLAG_END_SUCCESS;
trans->transactionNumber = transN;
trans->PeerId(tr->mTransaction->PeerId());
sendItem(trans);
generic_sendItem(trans);
// move to completed transactions
@ -2277,7 +2469,7 @@ void RsGxsNetService::processTransactions()
(tr->mTransaction->transactFlag & RsNxsTransacItem::FLAG_TYPE_MASK);
trans->transactionNumber = transN;
trans->PeerId(tr->mTransaction->PeerId());
sendItem(trans);
generic_sendItem(trans);
tr->mFlag = NxsTransaction::FLAG_STATE_RECEIVING;
}
@ -2654,7 +2846,7 @@ void RsGxsNetService::locked_pushMsgTransactionFromList(std::list<RsNxsItem*>& r
newTrans->mTransaction->PeerId(mOwnId);
if (locked_addTransaction(newTrans))
sendItem(transac);
generic_sendItem(transac);
else
{
delete newTrans;
@ -2950,7 +3142,7 @@ void RsGxsNetService::locked_pushGrpTransactionFromList( std::list<RsNxsItem*>&
newTrans->mTransaction->PeerId(mOwnId);
if (locked_addTransaction(newTrans))
sendItem(transac);
generic_sendItem(transac);
else
{
delete newTrans;
@ -3084,7 +3276,7 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
{
#ifdef NXS_NET_DEBUG_1
GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendGrpsTransaction() Generating Grp data send fron TransN: " << tr->mTransaction->transactionNumber << std::endl;
GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "locked_genSendGrpsTransaction() Generating Grp data send from TransN: " << tr->mTransaction->transactionNumber << std::endl;
#endif
// go groups requested in transaction tr
@ -3097,7 +3289,12 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
{
RsNxsSyncGrpItem* item = dynamic_cast<RsNxsSyncGrpItem*>(*lit);
if (item)
{
#ifdef NXS_NET_DEBUG_1
GXSNETDEBUG_PG(tr->mTransaction->PeerId(),item->grpId) << "locked_genSendGrpsTransaction() retrieving data for group \"" << item->grpId << "\"" << std::endl;
#endif
grps[item->grpId] = NULL;
}
else
{
#ifdef NXS_NET_DEBUG_1
@ -3109,7 +3306,12 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
if(!grps.empty())
mDataStore->retrieveNxsGrps(grps, false, false);
else
{
#ifdef NXS_NET_DEBUG_1
GXSNETDEBUG_P_(tr->mTransaction->PeerId()) << "RsGxsNetService::locked_genSendGrpsTransaction(): no group to request! This is unexpected" << std::endl;
#endif
return;
}
NxsTransaction* newTr = new NxsTransaction();
newTr->mFlag = NxsTransaction::FLAG_STATE_WAITING_CONFIRM;
@ -3126,6 +3328,9 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
mit->second->transactionNumber = transN;
newTr->mItems.push_back(mit->second);
mit->second = NULL ; // avoids deletion
#ifdef NXS_NET_DEBUG_1
GXSNETDEBUG_PG(tr->mTransaction->PeerId(),mit->first) << "RsGxsNetService::locked_genSendGrpsTransaction(): adding grp data of group \"" << mit->first << "\" to transaction" << std::endl;
#endif
}
if(newTr->mItems.empty()){
@ -3154,8 +3359,8 @@ void RsGxsNetService::locked_genSendGrpsTransaction(NxsTransaction* tr)
ntr->PeerId(tr->mTransaction->PeerId());
if(locked_addTransaction(newTr))
sendItem(ntr);
else
generic_sendItem(ntr);
else
{
delete ntr ;
delete newTr;
@ -3449,7 +3654,7 @@ void RsGxsNetService::locked_genSendMsgsTransaction(NxsTransaction* tr)
ntr->PeerId(tr->mTransaction->PeerId());
if(locked_addTransaction(newTr))
sendItem(ntr);
generic_sendItem(ntr);
else
{
delete ntr ;
@ -3768,7 +3973,7 @@ void RsGxsNetService::locked_pushGrpRespFromList(std::list<RsNxsItem*>& respList
<< peer << " with " << respList.size() << " groups " << std::endl;
#endif
if(locked_addTransaction(tr))
sendItem(trItem);
generic_sendItem(trItem);
else
{
delete tr ;
@ -3910,6 +4115,18 @@ bool RsGxsNetService::canSendGrpId(const RsPeerId& sslId, const RsGxsGrpMetaData
#ifdef NXS_NET_DEBUG_4
GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendGrpId()"<< std::endl;
#endif
// check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for
RsGxsGroupId peer_grp ;
if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId)
{
#warning (cyril) make sure that this is not a problem for cross-service sending of items
#ifdef NXS_NET_DEBUG_4
GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl;
#endif
return false ;
}
// first do the simple checks
uint8_t circleType = grpMeta.mCircleType;
@ -3963,6 +4180,17 @@ bool RsGxsNetService::checkCanRecvMsgFromPeer(const RsPeerId& sslId, const RsGxs
GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::checkCanRecvMsgFromPeer()";
GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " peer Id = " << sslId << ", grpId=" << grpMeta.mGroupId <<std::endl;
#endif
// check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for
RsGxsGroupId peer_grp ;
if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId)
{
#ifdef NXS_NET_DEBUG_4
GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl;
#endif
return false ;
}
// first do the simple checks
uint8_t circleType = grpMeta.mCircleType;
should_encrypt_id.clear() ;
@ -4293,7 +4521,7 @@ void RsGxsNetService::locked_pushMsgRespFromList(std::list<RsNxsItem*>& itemL, c
#endif
// signal peer to prepare for transaction
if(locked_addTransaction(tr))
sendItem(trItem);
generic_sendItem(trItem);
else
{
delete tr ;
@ -4306,6 +4534,16 @@ bool RsGxsNetService::canSendMsgIds(std::vector<RsGxsMsgMetaData*>& msgMetas, co
#ifdef NXS_NET_DEBUG_4
GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << "RsGxsNetService::canSendMsgIds() CIRCLE VETTING" << std::endl;
#endif
// check if that peer is a virtual peer id, in which case we only send/recv data to/from it items for the group it's requested for
RsGxsGroupId peer_grp ;
if(mAllowDistSync && mGxsNetTunnel != NULL && mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(sslId),peer_grp) && peer_grp != grpMeta.mGroupId)
{
#ifdef NXS_NET_DEBUG_4
GXSNETDEBUG_PG(sslId,grpMeta.mGroupId) << " Distant peer designed for group " << peer_grp << ": cannot request sync for different group." << std::endl;
#endif
return false ;
}
// first do the simple checks
uint8_t circleType = grpMeta.mCircleType;
@ -4546,7 +4784,7 @@ void RsGxsNetService::processExplicitGroupRequests()
for(; git != groupIdList.end(); ++git)
{
#ifdef NXS_NET_DEBUG_0
GXSNETDEBUG_PG(peerId,*git) << " group request for grp ID " << *git << " to peer " << peerId << std::endl;
GXSNETDEBUG_P_(peerId) << " group request for grp ID " << *git << " to peer " << peerId << std::endl;
#endif
RsNxsSyncGrpItem* item = new RsNxsSyncGrpItem(mServType);
item->grpId = *git;
@ -4680,7 +4918,7 @@ void RsGxsNetService::sharePublishKeysPending()
publishKeyItem->private_key = publishKey ;
publishKeyItem->PeerId(*it);
sendItem(publishKeyItem);
generic_sendItem(publishKeyItem);
#ifdef NXS_NET_DEBUG_3
GXSNETDEBUG_PG(*it,grpMeta->mGroupId) << " sent key item to " << *it << std::endl;
#endif
@ -4841,6 +5079,18 @@ bool RsGxsNetService::removeGroups(const std::list<RsGxsGroupId>& groups)
return true ;
}
bool RsGxsNetService::isDistantPeer(const RsPeerId& pid)
{
RS_STACK_MUTEX(mNxsMutex) ;
if(!mAllowDistSync || mGxsNetTunnel == NULL)
return false ;
RsGxsGroupId group_id ;
return mGxsNetTunnel->isDistantPeer(RsGxsNetTunnelVirtualPeerId(pid),group_id);
}
bool RsGxsNetService::stampMsgServerUpdateTS(const RsGxsGroupId& gid)
{
RS_STACK_MUTEX(mNxsMutex) ;
@ -4856,3 +5106,283 @@ bool RsGxsNetService::locked_stampMsgServerUpdateTS(const RsGxsGroupId& gid)
return true;
}
TurtleRequestId RsGxsNetService::turtleGroupRequest(const RsGxsGroupId& group_id)
{
RS_STACK_MUTEX(mNxsMutex) ;
time_t now = time(NULL);
auto it = mSearchedGroups.find(group_id) ;
if(mSearchedGroups.end() != it && (it->second.ts + MIN_DELAY_BETWEEN_GROUP_SEARCH > now))
{
std::cerr << "(WW) Last turtle request was " << now - it->second.ts << " secs ago. Not searching again." << std::endl;
return it->second.request_id;
}
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG__G(group_id) << " requesting group id " << group_id << " using turtle" << std::endl;
#endif
TurtleRequestId req = mGxsNetTunnel->turtleGroupRequest(group_id,this) ;
GroupRequestRecord& rec(mSearchedGroups[group_id]) ;
rec.request_id = req;
rec.ts = now;
mSearchRequests[req] = group_id;
return req;
}
TurtleRequestId RsGxsNetService::turtleSearchRequest(const std::string& match_string)
{
return mGxsNetTunnel->turtleSearchRequest(match_string,this) ;
}
static bool termSearch(const std::string& src, const std::string& substring)
{
/* always ignore case */
return src.end() != std::search( src.begin(), src.end(), substring.begin(), substring.end(), RsRegularExpression::CompareCharIC() );
}
bool RsGxsNetService::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSummary>& group_infos)
{
RS_STACK_MUTEX(mNxsMutex) ;
auto it = mDistantSearchResults.find(req) ;
if(it == mDistantSearchResults.end())
return false ;
group_infos = it->second;
return true ;
}
bool RsGxsNetService::retrieveDistantGroupSummary(const RsGxsGroupId& group_id,RsGxsGroupSummary& gs)
{
RS_STACK_MUTEX(mNxsMutex) ;
for(auto it(mDistantSearchResults.begin());it!=mDistantSearchResults.end();++it)
{
auto it2 = it->second.find(group_id) ;
if(it2 != it->second.end())
{
gs = it2->second;
return true ;
}
}
return false ;
}
bool RsGxsNetService::clearDistantSearchResults(const TurtleRequestId& id)
{
RS_STACK_MUTEX(mNxsMutex) ;
mDistantSearchResults.erase(id);
return true ;
}
void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req, const std::list<RsGxsGroupSummary>& group_infos)
{
RS_STACK_MUTEX(mNxsMutex) ;
RsGxsGrpMetaTemporaryMap grpMeta;
std::map<RsGxsGroupId,RsGxsGroupSummary>& search_results_map(mDistantSearchResults[req]) ;
for(auto it(group_infos.begin());it!=group_infos.end();++it)
if(search_results_map.find((*it).group_id) == search_results_map.end())
grpMeta[(*it).group_id] = NULL;
mDataStore->retrieveGxsGrpMetaData(grpMeta);
std::list<RsGxsGroupSummary> filtered_results ;
// only keep groups that are not locally known, and groups that are not already in the mDistantSearchResults structure
for(auto it(group_infos.begin());it!=group_infos.end();++it)
if(grpMeta[(*it).group_id] == NULL)
{
filtered_results.push_back(*it) ;
auto it2 = search_results_map.find((*it).group_id) ;
if(it2 != search_results_map.end())
{
// update existing data
it2->second.popularity++ ;
it2->second.number_of_messages = std::max(it2->second.number_of_messages,(*it).number_of_messages) ;
}
else
{
search_results_map[(*it).group_id] = *it;
search_results_map[(*it).group_id].popularity = 1; // number of results so far
}
mObserver->receiveDistantSearchResults(req,(*it).group_id) ;
}
}
void RsGxsNetService::receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)
{
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " received encrypted group data for turtle search request " << std::hex << req << std::dec << ": " << RsUtil::BinToHex(encrypted_group_data,encrypted_group_data_len,50) << std::endl;
#endif
auto it = mSearchRequests.find(req);
if(mSearchRequests.end() == it)
{
std::cerr << "(EE) received search results for unknown request " << std::hex << req << std::dec ;
return;
}
RsGxsGroupId grpId = it->second;
uint8_t encryption_master_key[32];
Sha256CheckSum s = RsDirUtil::sha256sum(grpId.toByteArray(),grpId.SIZE_IN_BYTES);
memcpy(encryption_master_key,s.toByteArray(),32);
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " attempting data decryption with master key " << RsUtil::BinToHex(encryption_master_key,32) << std::endl;
#endif
unsigned char *clear_group_data = NULL;
uint32_t clear_group_data_len ;
if(!librs::crypto::decryptAuthenticateData(encrypted_group_data,encrypted_group_data_len,encryption_master_key,clear_group_data,clear_group_data_len))
{
std::cerr << "(EE) Could not decrypt data. Something went wrong. Wrong key??" << std::endl;
return ;
}
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " successfuly decrypted data : " << RsUtil::BinToHex(clear_group_data,clear_group_data_len,50) << std::endl;
#endif
RsItem *item = RsNxsSerialiser(mServType).deserialise(clear_group_data,&clear_group_data_len) ;
free(clear_group_data);
clear_group_data = NULL ;
RsNxsGrp *nxs_grp = dynamic_cast<RsNxsGrp*>(item) ;
if(nxs_grp == NULL)
{
std::cerr << "(EE) decrypted item is not a RsNxsGrp. Weird!" << std::endl;
return ;
}
std::vector<RsNxsGrp*> new_grps(1,nxs_grp);
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " passing the grp data to observer." << std::endl;
#endif
mObserver->receiveNewGroups(new_grps);
}
bool RsGxsNetService::search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos)
{
RsGxsGrpMetaTemporaryMap grpMetaMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
mDataStore->retrieveGxsGrpMetaData(grpMetaMap);
}
RsGroupNetworkStats stats ;
for(auto it(grpMetaMap.begin());it!=grpMetaMap.end();++it)
if(termSearch(it->second->mGroupName,substring))
{
getGroupNetworkStats(it->first,stats) ;
RsGxsGroupSummary s ;
s.group_id = it->first ;
s.group_name = it->second->mGroupName ;
s.group_description = it->second->mGroupName ; // to be filled with something better when we use the real search
s.search_context = it->second->mGroupName ;
s.sign_flags = it->second->mSignFlags;
s.publish_ts = it->second->mPublishTs;
s.author_id = it->second->mAuthorId;
s.number_of_messages = stats.mMaxVisibleCount ;
s.last_message_ts = stats.mLastGroupModificationTS ;
s.popularity = it->second->mPop;
group_infos.push_back(s) ;
}
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " performing local substring search in response to distant request. Found " << group_infos.size() << " responses." << std::endl;
#endif
return !group_infos.empty();
}
bool RsGxsNetService::search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)
{
// First look into the grp hash cache
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " Received group data request for hash " << hashed_group_id << std::endl;
#endif
auto it = mGroupHashCache.find(hashed_group_id) ;
RsNxsGrp *grp_data = NULL ;
if(mGroupHashCache.end() != it)
{
grp_data = it->second;
}
else
{
// Now check if the last request was too close in time, in which case, we dont retrieve data.
if(mLastCacheReloadTS + 60 > time(NULL))
{
std::cerr << "(WW) Not found in cache, and last cache reload less than 60 secs ago. Returning false. " << std::endl;
return false ;
}
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " reloading group cache information" << std::endl;
#endif
RsNxsGrpDataTemporaryMap grpDataMap;
{
RS_STACK_MUTEX(mNxsMutex) ;
mDataStore->retrieveNxsGrps(grpDataMap, true, true);
mLastCacheReloadTS = time(NULL);
}
for(auto it(grpDataMap.begin());it!=grpDataMap.end();++it)
if(it->second->metaData->mSubscribeFlags & GXS_SERV::GROUP_SUBSCRIBE_SUBSCRIBED ) // only cache subscribed groups
{
RsNxsGrp *grp = it->second ;
delete grp->metaData ; // clean private keys
grp->metaData = NULL ;
Sha1CheckSum hash(RsDirUtil::sha1sum(it->first.toByteArray(),it->first.SIZE_IN_BYTES));
mGroupHashCache[hash] = grp ;
it->second = NULL ; // prevents deletion
if(hash == hashed_group_id)
grp_data = grp ;
}
}
if(!grp_data)
{
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " no group found for hash " << hashed_group_id << ": returning false." << std::endl;
#endif
return false ;
}
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " found corresponding group data group id in cache group_id=" << grp_data->grpId << std::endl;
#endif
// Finally, serialize and encrypt the grp data
uint32_t size = RsNxsSerialiser(mServType).size(grp_data);
RsTemporaryMemory mem(size) ;
RsNxsSerialiser(mServType).serialise(grp_data,mem,&size) ;
uint8_t encryption_master_key[32];
Sha256CheckSum s = RsDirUtil::sha256sum(grp_data->grpId.toByteArray(),grp_data->grpId.SIZE_IN_BYTES);
memcpy(encryption_master_key,s.toByteArray(),32);
#ifdef NXS_NET_DEBUG_8
GXSNETDEBUG___ << " sending data encrypted with master key " << RsUtil::BinToHex(encryption_master_key,32) << std::endl;
#endif
return librs::crypto::encryptAuthenticateData(mem,size,encryption_master_key,encrypted_group_data,encrypted_group_data_len);
}

48
libretroshare/src/gxs/rsgxsnetservice.h
View file

@ -35,6 +35,7 @@
#include "pqi/p3linkmgr.h"
#include "rsitems/rsnxsitems.h"
#include "rsitems/rsgxsupdateitems.h"
#include "rsgxsnettunnel.h"
#include "rsgxsnetutils.h"
#include "pqi/p3cfgmgr.h"
#include "rsgixs.h"
@ -58,6 +59,14 @@ class RsGroupNetworkStatsRecord
time_t update_TS ;
};
struct GroupRequestRecord
{
GroupRequestRecord(): ts(0), request_id(0) {}
time_t ts ;
TurtleRequestId request_id;
};
/*!
* This class implements the RsNetWorkExchangeService
* using transactions to handle synchrnisation of Nxs items between
@ -89,8 +98,8 @@ public:
RsNxsObserver *nxsObs, // used to be = NULL.
const RsServiceInfo serviceInfo,
RsGixsReputation* reputations = NULL, RsGcxs* circles = NULL, RsGixs *gixs=NULL,
PgpAuxUtils *pgpUtils = NULL,
bool grpAutoSync = true, bool msgAutoSync = true,
PgpAuxUtils *pgpUtils = NULL, RsGxsNetTunnelService *mGxsNT = NULL,
bool grpAutoSync = true, bool msgAutoSync = true,bool distSync=false,
uint32_t default_store_period = RS_GXS_DEFAULT_MSG_STORE_PERIOD,
uint32_t default_sync_period = RS_GXS_DEFAULT_MSG_REQ_PERIOD);
@ -103,6 +112,8 @@ public:
public:
virtual uint16_t serviceType() const { return mServType ; }
/*!
* Use this to set how far back synchronisation and storage of messages should take place
* @param age the max age a sync/storage item can to be allowed in a synchronisation
@ -119,6 +130,23 @@ public:
virtual void setDefaultKeepAge(uint32_t t) { mDefaultMsgStorePeriod = t ; }
virtual void setDefaultSyncAge(uint32_t t) { mDefaultMsgSyncPeriod = t ; }
/*!
* \brief Search methods.
* These four methods are used to request distant search and receive the results.
* \param group_id
*/
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id);
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string);
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) ;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len);
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos);
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len);
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &group_infos);
virtual bool clearDistantSearchResults(const TurtleRequestId& id);
virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&);
/*!
* pauses synchronisation of subscribed groups and request for group id
* from peers
@ -167,6 +195,7 @@ public:
virtual bool getGroupServerUpdateTS(const RsGxsGroupId& gid,time_t& grp_server_update_TS,time_t& msg_server_update_TS) ;
virtual bool stampMsgServerUpdateTS(const RsGxsGroupId& gid) ;
virtual bool removeGroups(const std::list<RsGxsGroupId>& groups);
virtual bool isDistantPeer(const RsPeerId& pid);
/* p3Config methods */
public:
@ -394,6 +423,7 @@ private:
void locked_pushGrpRespFromList(std::list<RsNxsItem*>& respList, const RsPeerId& peer, const uint32_t& transN);
void locked_pushMsgRespFromList(std::list<RsNxsItem*>& itemL, const RsPeerId& sslId, const RsGxsGroupId &grp_id, const uint32_t& transN);
void checkDistantSyncState();
void syncWithPeers();
void syncGrpStatistics();
void addGroupItemToList(NxsTransaction*& tr,
@ -493,6 +523,9 @@ private:
void cleanRejectedMessages();
void processObserverNotifications();
void generic_sendItem(RsNxsItem *si);
RsItem *generic_recvItem();
private:
static void locked_checkDelay(uint32_t& time_in_secs);
@ -541,8 +574,11 @@ private:
RsGixs *mGixs;
RsGixsReputation* mReputations;
PgpAuxUtils *mPgpUtils;
RsGxsNetTunnelService *mGxsNetTunnel;
bool mGrpAutoSync;
bool mAllowMsgSync;
bool mAllowDistSync;
// need to be verfied
std::vector<AuthorPending*> mPendingResp;
@ -577,10 +613,18 @@ private:
std::set<RsGxsGroupId> mNewStatsToNotify ;
std::set<RsGxsGroupId> mNewPublishKeysToNotify ;
// Distant search result map
std::map<TurtleRequestId,std::map<RsGxsGroupId,RsGxsGroupSummary> > mDistantSearchResults ;
void debugDump();
uint32_t mDefaultMsgStorePeriod ;
uint32_t mDefaultMsgSyncPeriod ;
std::map<Sha1CheckSum, RsNxsGrp*> mGroupHashCache;
std::map<TurtleRequestId,RsGxsGroupId> mSearchRequests;
std::map<RsGxsGroupId,GroupRequestRecord> mSearchedGroups ;
time_t mLastCacheReloadTS ;
};
#endif // RSGXSNETSERVICE_H

1174
libretroshare/src/gxs/rsgxsnettunnel.cc Normal file
View file

File diff suppressed because it is too large Load diff

277
libretroshare/src/gxs/rsgxsnettunnel.h Normal file
View file

@ -0,0 +1,277 @@
/*
* libretroshare/src/gxs: rsgxsnettunnel.h
*
* General Data service, interface for RetroShare.
*
* Copyright 2018-2018 by Cyril Soler
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare.project@gmail.com"
*
*/
#pragma once
#include <map>
#include "turtle/p3turtle.h"
#include "retroshare/rsgxsdistsync.h"
/*!
* \brief The RsGxsNetTunnelService class takes care of requesting tunnels to the turtle router, through which it is possible to sync
* a particular GXS group. For each group, a set of virtual peers, corresponding to active tunnels will be made available
* to RsGxsNetService.
*
* It is the responsibility of RsGxsNetService to activate/desactivate tunnels for each particular group, depending on wether the group
* is already available at regular friends or not.
*
* Tunnel management is done at the level of groups rather than services, because we would like to keep the possibility to not
* request tunnels for some groups which do not need it, and only request tunnels for specific groups that cannot be provided
* by direct connections.
*/
// Protocol:
// * request tunnels based on H(GroupId)
// * encrypt tunnel data using chacha20+HMAC-SHA256 using AEAD( GroupId, 96bits IV, tunnel ID ) (similar to what FT does)
// * when tunnel is established, exchange virtual peer names: vpid = H( GroupID | Random bias )
// * when vpid is known, notify the client (GXS net service) which can use the virtual peer to sync
// * only use a single tunnel per virtual peer ID
// -
// Client ------------------ TR(H(GroupId)) --------------> Server |
// | Turtle
// Client <-------------------- T OK ---------------------- Server |
// -
// Here, a turtle vpid is known | [ addVirtualPeer() called by turtle ]
// -
// [Encrypted traffic using H(GroupId | Tunnel ID, 96bits IV)] |
// |
// Client <--------- VPID = H( Random IV | GroupId ) ------ Server |
// | | |
// +--------------> Mark the virtual peer active <-----------+ | Encrypted traffic decoded locally and sorted
// |
// Here, a consistent virtual peer ID is known |
// |
// Client <------------------- GXS Data ------------------> Server |
// -
// Notes:
// * tunnels are established symmetrically. If a distant peers wants to sync the same group, they'll have to open a single tunnel, with a different ID.
// Groups therefore have two states:
// - managed : the group can be used to answer tunnel requests. If server tunnels are established, the group will be synced with these peers
// - tunneled: the group will actively request tunnels. If tunnels are established both ways, the same virtual peers will be used so the tunnels are "merged".
// * In practice, that means one of the two tunnels will not be used and therefore die.
// * If a tunneled group already has enough virtual peers, it will not request for tunnels itself.
//
// Group policy | Request tunnels | SyncWithPeers | Item receipt
// --------------------+-------------------+-----------------------+----------------
// Passive | no | If peers present | If peers present
// Active | yes, if no peers | If peers present | If peers present
// | | |
//
// * when a service has the DistSync flag set, groups to sync are communicated passively to the GxsNetTunnel service when requesting distant peers.
// However, a call should be made to set a particular group policy to "ACTIVE" for group that do not have peers and need some.
//
// * services also need to retrieve GXS data items that come out of tunnels. These will be available as (data,len) type, since they are not de-serialized.
//
// * GxsNetService stores data information (such as update maps) per peerId, so it makes sense to use the same PeerId for all groups of a given service
// Therefore, virtual peers are stored separately from groups, because each one can sync multiple groups.
//
// * virtual peers are also shared among services. This reduces the required amount of tunnels and tunnel requests to send.
//
// * tunnels for a given service may also be used by the net service of p3IdService in order to explicitely request missing GxsIds.
// So GxsNetTunnel somehow allows different services to use the same tunnel. However we make sure that this traffic is limited to only p3IdService.
//
// How do we know that a group needs distant sync?
// * look into GrpConfigMap for suppliers. Suppliers is cleared at load.
// * last_update_TS in GrpConfigMap is randomised so it cannot be used
// * we need a way to know that there's no suppliers for good reasons (not that we just started)
//
// Security
// * the question of sync-ing with distant anonymous peers is a bit tricky because these peers can try to generate fake requests or change which messages are available
// and there is no way to prevent it. We therefore rely on GXS data integrity system to prevent this to happen.
//
class RsGxsNetTunnelItem ;
class RsNetworkExchangeService ;
class RsGxsNetTunnelService: public RsTurtleClientService, public RsTickingThread, public p3Config, public RsGxsDistSync
{
public:
RsGxsNetTunnelService() ;
virtual ~RsGxsNetTunnelService() ;
/*!
* \brief registerSearchableService
* Adds the network exchange service as a possible search source. This is used to allow distant search on the corresponding
* GXS service.
* \return
* always returns true.
*/
bool registerSearchableService(RsNetworkExchangeService *) ;
/*!
* \brief Manage tunnels for this group
* @param group_id group for which tunnels should be released
*/
bool requestDistantPeers(uint16_t service_id,const RsGxsGroupId&group_id) ;
/*!
* \brief Stop managing tunnels for this group
* @param group_id group for which tunnels should be released
*/
bool releaseDistantPeers(uint16_t service_id, const RsGxsGroupId&group_id) ;
/*!
* \brief Get the list of active virtual peers for a given group. This implies that a tunnel is up and
* alive. This function also "registers" the group which allows to handle tunnel requests in the server side.
*/
bool getVirtualPeers(std::list<RsGxsNetTunnelVirtualPeerId>& peers) ; // returns the virtual peers for this service
/*!
* \brief serviceId
* Overloads the method in RsTurtleClientService.
* \return
* The service id for RsGxsNetTunnel.
*/
uint16_t serviceId() const ;
/*!
* \brief sendData
* send data to this virtual peer, and takes memory ownership (deletes the item)
* \param item item to send
* \param virtual_peer destination virtual peer
* \return
* true if succeeded.
*/
bool sendTunnelData(uint16_t service_id,unsigned char *& data, uint32_t data_len, const RsGxsNetTunnelVirtualPeerId& virtual_peer) ;
/*!
* \brief receiveData
* returns the next piece of data received, and the virtual GXS peer that sended it.
* \param data memory check containing the data. Memory ownership belongs to the client.
* \param data_len length of memory chunk
* \param virtual_peer peer who sent the data
* \return
* true if something is returned. If not, data is set to NULL, data_len to 0.
*/
bool receiveTunnelData(uint16_t service_id, unsigned char *& data, uint32_t& data_len, RsGxsNetTunnelVirtualPeerId& virtual_peer) ;
/*!
* \brief isDistantPeer
* returns wether the peer is in the list of available distant peers or not
* \param group_id returned by the service to indicate which group this peer id is designed for.
* \return true if the peer is a distant GXS peer.
*/
bool isDistantPeer(const RsGxsNetTunnelVirtualPeerId& virtual_peer,RsGxsGroupId& group_id) ;
/*!
* \brief dumps all information about monitored groups.
*/
void dump() const;
/*!
* \brief connectToTurtleRouter
* Should be called after allocating a RsGxsNetTunnelService
* \param tr turtle router object
*/
virtual void connectToTurtleRouter(p3turtle *tr) ;
TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id, RsNetworkExchangeService *client_service) ;
TurtleRequestId turtleSearchRequest(const std::string& match_string,RsNetworkExchangeService *client_service) ;
/*!
* \brief receiveSearchRequest
* See RsTurtleClientService::@
*/
virtual bool receiveSearchRequest(unsigned char *search_request_data, uint32_t search_request_data_len, unsigned char *& search_result_data, uint32_t& search_result_data_len, uint32_t &max_allowed_hits);
virtual void receiveSearchResult(TurtleSearchRequestId request_id,unsigned char *search_result_data,uint32_t search_result_data_len);
// Overloaded from RsTickingThread
void data_tick() ;
// Overloads p3Config
RsSerialiser *setupSerialiser();
bool saveList(bool& cleanup, std::list<RsItem*>& save);
bool loadList(std::list<RsItem *> &load);
// Overloads RsGxsDistSync
void getStatistics(std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, // groups on the client and server side
std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, // current virtual peers, which group they provide, and how to talk to them through turtle
std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
Bias20Bytes& bias) const;
protected:
// interaction with turtle router
virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ;
virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ;
void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ;
const Bias20Bytes& locked_randomBias() ;
p3turtle *mTurtle ;
static const uint32_t RS_GXS_TUNNEL_CONST_RANDOM_BIAS_SIZE = 20 ;
static const uint32_t RS_GXS_TUNNEL_CONST_EKEY_SIZE = 32 ;
mutable Bias20Bytes mRandomBias ; // constant accross reboots. Allows to disguise the real SSL id while providing a consistent value accross time.
private:
void autowash() ;
void sendKeepAlivePackets() ;
void handleIncoming(RsGxsNetTunnelItem *item) ;
void flush_pending_items();
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo> mGroups ; // groups on the client and server side
std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo> mVirtualPeers ; // current virtual peers, which group they provide, and how to talk to them through turtle
std::map<RsFileHash, RsGxsGroupId> mHandledHashes ; // hashes asked to turtle. Used to answer tunnel requests
std::map<TurtleVirtualPeerId, RsGxsNetTunnelVirtualPeerId> mTurtle2GxsPeer ; // convertion table to find GXS peer id from turtle
std::list<std::pair<TurtleVirtualPeerId,RsTurtleGenericDataItem*> > mPendingTurtleItems ; // items that need to be sent off-turtle Mutex.
std::map<uint16_t, std::list<std::pair<RsGxsNetTunnelVirtualPeerId,RsTlvBinaryData *> > > mIncomingData; // list of incoming data items
std::map<uint16_t,RsNetworkExchangeService *> mSearchableServices ;
/*!
* \brief Generates the hash to request tunnels for this group. This hash is only used by turtle, and is used to
* hide the real group id.
*/
RsFileHash calculateGroupHash(const RsGxsGroupId&group_id) const ;
/*!
* \brief makeVirtualPeerIdForGroup creates a virtual peer id that can be used and that will be constant accross time, whatever the
* tunnel ID and turtle virtual peer id. This allows RsGxsNetService to keep sync-ing the data consistently.
*/
RsGxsNetTunnelVirtualPeerId locked_makeVirtualPeerId(const RsGxsGroupId& group_id) ;
static void generateEncryptionKey(const RsGxsGroupId& group_id,const TurtleVirtualPeerId& vpid,unsigned char key[RS_GXS_TUNNEL_CONST_EKEY_SIZE]) ;
mutable RsMutex mGxsNetTunnelMtx;
friend class RsGxsTunnelRandomBiasItem ;
friend class StoreHere ;
time_t mLastKeepAlive ;
time_t mLastAutoWash ;
time_t mLastDump ;
};

73
libretroshare/src/gxs/rsnxs.h
View file

@ -36,6 +36,7 @@
#include "services/p3service.h"
#include "retroshare/rsreputations.h"
#include "retroshare/rsidentity.h"
#include "retroshare/rsturtle.h"
#include "rsgds.h"
/*!
@ -58,6 +59,7 @@
* 2 transfers only between group
* - the also group matrix settings which is by default everyone can transfer to each other
*/
class RsNetworkExchangeService
{
public:
@ -65,6 +67,7 @@ public:
RsNetworkExchangeService(){ return;}
virtual ~RsNetworkExchangeService() {}
virtual uint16_t serviceType() const =0;
/*!
* Use this to set how far back synchronisation of messages should take place
* @param age in seconds the max age a sync/store item can to be allowed in a synchronisation
@ -81,6 +84,65 @@ public:
virtual uint32_t getDefaultSyncAge() =0;
virtual uint32_t getDefaultKeepAge() =0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// DISTANT SEARCH FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/*!
* \brief turtleGroupRequest
* Requests a particular group meta data. The request protects the group ID.
* \param group_id
* \return
* returns the turtle request ID that might be associated to some results.
*/
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0;
/*!
* \brief turtleSearchRequest
* Uses distant search to match the substring to the group meta data.
* \param match_string
* \return
* returns the turtle request ID that might be associated to some results.
*/
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new results are received
* \param req Turtle search request ID associated with this result
* \param group_infos Group summary information for the groups returned by the search
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const std::list<RsGxsGroupSummary>& group_infos)=0;
/*!
* \brief receiveTurtleSearchResults
* Called by turtle (through RsGxsNetTunnel) when new data is received
* \param req Turtle search request ID associated with this result
* \param encrypted_group_data Group data
*/
virtual void receiveTurtleSearchResults(TurtleRequestId req,const unsigned char *encrypted_group_data,uint32_t encrypted_group_data_len)=0;
/*!
* \brief retrieveTurtleSearchResults
* To be used to retrieve the search results that have been notified (or not)
* \param req request that match the results to retrieve
* \param group_infos results to retrieve.
* \return
* false when the request is unknown.
*/
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &group_infos)=0;
/*!
* \brief getDistantSearchResults
* \param id
* \param group_infos
* \return
*/
virtual bool clearDistantSearchResults(const TurtleRequestId& id)=0;
virtual bool retrieveDistantGroupSummary(const RsGxsGroupId&,RsGxsGroupSummary&)=0;
virtual bool search(const std::string& substring,std::list<RsGxsGroupSummary>& group_infos) =0;
virtual bool search(const Sha1CheckSum& hashed_group_id,unsigned char *& encrypted_group_data,uint32_t& encrypted_group_data_len)=0;
/*!
* Initiates a search through the network
* This returns messages which contains the search terms set in RsGxsSearch
@ -99,6 +161,9 @@ public:
*/
//virtual int searchGrps(RsGxsSearch* search, uint8_t hops = 1, bool retrieve = 0) = 0;
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/// DATA ACCESS FUNCTIONS ///
///////////////////////////////////////////////////////////////////////////////////////////////////////////////////
/*!
* pauses synchronisation of subscribed groups and request for group id
@ -166,6 +231,14 @@ public:
*/
virtual bool stampMsgServerUpdateTS(const RsGxsGroupId& gid) =0;
/*!
* \brief isDistantPeer
* \param pid peer that is a virtual peer provided by GxsNetTunnel
* \return
* true if the peer exists (adn therefore is online)
*/
virtual bool isDistantPeer(const RsPeerId& pid)=0;
/*!
* \brief removeGroups
* Removes time stamp information from the list of groups. This allows to re-sync them if suppliers are present.

15
libretroshare/src/gxs/rsnxsobserver.h
View file

@ -29,6 +29,7 @@
#include <set>
#include "rsitems/rsnxsitems.h"
typedef uint32_t TurtleRequestId ;
class RsNxsObserver
{
@ -42,12 +43,22 @@ public:
/*!
* @param messages messages are deleted after function returns
*/
virtual void notifyNewMessages(std::vector<RsNxsMsg*>& messages) = 0;
virtual void receiveNewMessages(std::vector<RsNxsMsg*>& messages) = 0;
/*!
* @param groups groups are deleted after function returns
*/
virtual void notifyNewGroups(std::vector<RsNxsGrp*>& groups) = 0;
virtual void receiveNewGroups(std::vector<RsNxsGrp*>& groups) = 0;
/*!
* \brief receiveDistantSearchResults
* Called when new distant search result arrive.
* \param grpId
*/
virtual void receiveDistantSearchResults(TurtleRequestId /*id*/,const RsGxsGroupId& /*grpId*/)
{
std::cerr << __PRETTY_FUNCTION__ << ": not overloaded but still called. Nothing will happen." << std::endl;
}
/*!
* @param grpId group id

2
libretroshare/src/gxstrans/p3gxstrans.cc
View file

@ -839,7 +839,7 @@ bool p3GxsTrans::dispatchDecryptedMail( const RsGxsId& authorId,
#endif
std::vector<RsNxsMsg*> rcct; rcct.push_back(receipt);
RsGenExchange::notifyNewMessages(rcct);
RsGenExchange::receiveNewMessages(rcct);
GxsTransClient* recipientService = NULL;
{

4
libretroshare/src/gxstunnel/p3gxstunnel.cc
View file

@ -685,7 +685,7 @@ void p3GxsTunnelService::removeVirtualPeer(const TurtleFileHash& hash,const Turt
}
}
void p3GxsTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *gitem,const RsFileHash& hash, const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction)
void p3GxsTunnelService::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash& hash, const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem::Direction direction)
{
#ifdef DEBUG_GXS_TUNNEL
std::cerr << "GxsTunnelService::receiveTurtleData(): Received turtle data. " << std::endl;
@ -697,7 +697,7 @@ void p3GxsTunnelService::receiveTurtleData(RsTurtleGenericTunnelItem *gitem,cons
(void) direction;
#endif
RsTurtleGenericDataItem *item = dynamic_cast<RsTurtleGenericDataItem*>(gitem) ;
const RsTurtleGenericDataItem *item = dynamic_cast<const RsTurtleGenericDataItem*>(gitem) ;
if(item == NULL)
{

6
libretroshare/src/gxstunnel/p3gxstunnel.h
View file

@ -95,7 +95,7 @@
// by a mix between our own GXS id and the GXS id we're talking to. That is what the TunnelVirtualPeer is.
//
//
// RequestTunnel(source_own_id,destination_id) -
// RequestTunnel(source_own_id,destination_id) -
// | |
// +---------------------------> p3Turtle::monitorTunnels( hash(destination_id) ) |
// | |
@ -127,6 +127,8 @@ public:
explicit p3GxsTunnelService(RsGixs *pids) ;
virtual void connectToTurtleRouter(p3turtle *) ;
uint16_t serviceId() const { return RS_SERVICE_TYPE_GXS_TUNNEL ; }
// Creates the invite if the public key of the distant peer is available.
// Om success, stores the invite in the map above, so that we can respond to tunnel requests.
//
@ -211,7 +213,7 @@ private:
// Overloaded from RsTurtleClientService
virtual bool handleTunnelRequest(const RsFileHash &hash,const RsPeerId& peer_id) ;
virtual void receiveTurtleData(RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
virtual void receiveTurtleData(const RsTurtleGenericTunnelItem *item,const RsFileHash& hash,const RsPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction direction) ;
void addVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&,RsTurtleGenericTunnelItem::Direction dir) ;
void removeVirtualPeer(const TurtleFileHash&, const TurtleVirtualPeerId&) ;

9
libretroshare/src/libretroshare.pro
View file

@ -137,6 +137,7 @@ PUBLIC_HEADERS = retroshare/rsdisc.h \
retroshare/rsconfig.h \
retroshare/rsversion.h \
retroshare/rsservicecontrol.h \
retroshare/rsgxsdistsync.h
HEADERS += plugins/pluginmanager.h \
plugins/dlfcn_win32.h \
@ -351,7 +352,8 @@ HEADERS += ft/ftchunkmap.h \
ft/ftturtlefiletransferitem.h
HEADERS += crypto/chacha20.h \
crypto/hashstream.h
crypto/hashstream.h \
crypto/rscrypto.h
HEADERS += directory_updater.h \
directory_list.h \
@ -519,7 +521,8 @@ SOURCES += ft/ftchunkmap.cc \
ft/ftturtlefiletransferitem.cc
SOURCES += crypto/chacha20.cpp \
crypto/hashstream.cc
crypto/hashstream.cc \
crypto/rscrypto.cpp
SOURCES += chat/distantchat.cc \
chat/p3chatservice.cc \
@ -707,6 +710,7 @@ HEADERS += rsitems/rsnxsitems.h \
gxs/rsgxs.h \
gxs/rsdataservice.h \
gxs/rsgxsnetservice.h \
gxs/rsgxsnettunnel.h \
retroshare/rsgxsflags.h \
retroshare/rsgxsifacetypes.h \
gxs/rsgenexchange.h \
@ -732,6 +736,7 @@ SOURCES += rsitems/rsnxsitems.cc \
gxs/rsdataservice.cc \
gxs/rsgenexchange.cc \
gxs/rsgxsnetservice.cc \
gxs/rsgxsnettunnel.cc \
gxs/rsgxsdata.cc \
rsitems/rsgxsitems.cc \
gxs/rsgxsdataaccess.cc \

6
libretroshare/src/pqi/p3cfgmgr.cc
View file

@ -65,7 +65,7 @@ void p3ConfigMgr::tick()
#ifdef CONFIG_DEBUG
std::cerr << "p3ConfigMgr::tick() Config Changed - Element: ";
std::cerr << it->first;
std::cerr << *it;
std::cerr << std::endl;
#endif
@ -111,7 +111,7 @@ void p3ConfigMgr::saveConfig()
{
#ifdef CONFIG_DEBUG
std::cerr << "p3ConfigMgr::globalSaveConfig() Saving Element: ";
std::cerr << it->first;
std::cerr << *it;
std::cerr << std::endl;
#endif
ok &= (*it)->saveConfiguration();
@ -137,7 +137,7 @@ void p3ConfigMgr::loadConfig()
{
#ifdef CONFIG_DEBUG
std::cerr << "p3ConfigMgr::loadConfig() Element: ";
std::cerr << cit->first <<"Dummy Hash: " << dummyHash;
std::cerr << *cit <<" Dummy Hash: " << dummyHash;
std::cerr << std::endl;
#endif

4
libretroshare/src/pqi/p3notify.cc
View file

@ -231,7 +231,9 @@ void p3Notify::notifyChatLobbyTimeShift (int time_shift)
void p3Notify::notifyCustomState (const std::string& peer_id , const std::string& status_string ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyCustomState (peer_id,status_string) ; }
void p3Notify::notifyHashingInfo (uint32_t type , const std::string& fileinfo ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyHashingInfo (type,fileinfo) ; }
void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list<TurtleFileInfo>& files ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,files) ; }
void p3Notify::notifyPeerHasNewAvatar (std::string peer_id ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; }
#warning MISSING CODE HERE
//void p3Notify::notifyTurtleSearchResult (uint32_t search_id , const std::list<TurtleGxsInfo>& groups ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyTurtleSearchResult(search_id,groups) ; }
void p3Notify::notifyPeerHasNewAvatar (std::string peer_id ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyPeerHasNewAvatar(peer_id) ; }
void p3Notify::notifyOwnAvatarChanged () { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnAvatarChanged() ; }
void p3Notify::notifyOwnStatusMessageChanged() { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyOwnStatusMessageChanged() ; }
void p3Notify::notifyDiskFull (uint32_t location , uint32_t size_limit_in_MB ) { FOR_ALL_NOTIFY_CLIENTS (*it)->notifyDiskFull (location,size_limit_in_MB) ; }

2
libretroshare/src/pqi/p3notify.h
View file

@ -106,6 +106,8 @@ class p3Notify: public RsNotify
void notifyCustomState (const std::string& /* peer_id */, const std::string& /* status_string */) ;
void notifyHashingInfo (uint32_t /* type */, const std::string& /* fileinfo */) ;
void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list<TurtleFileInfo>& /* files */) ;
#warning MISSING CODE HERE
// void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list<TurtleGxsInfo >& /* groups */) ;
void notifyPeerHasNewAvatar (std::string /* peer_id */) ;
void notifyOwnAvatarChanged () ;
void notifyOwnStatusMessageChanged () ;

174
libretroshare/src/pqi/p3peermgr.cc
View file

@ -89,11 +89,11 @@ static const std::string kConfigKeyProxyServerIpAddrTor = "PROXY_SERVER_IPADDR";
static const std::string kConfigKeyProxyServerPortTor = "PROXY_SERVER_PORT";
static const std::string kConfigKeyProxyServerIpAddrI2P = "PROXY_SERVER_IPADDR_I2P";
static const std::string kConfigKeyProxyServerPortI2P = "PROXY_SERVER_PORT_I2P";
void printConnectState(std::ostream &out, peerState &peer);
peerState::peerState()
:netMode(RS_NET_MODE_UNKNOWN), vs_disc(RS_VS_DISC_FULL), vs_dht(RS_VS_DHT_FULL), lastcontact(0),
:netMode(RS_NET_MODE_UNKNOWN), vs_disc(RS_VS_DISC_FULL), vs_dht(RS_VS_DHT_FULL), lastcontact(0),
hiddenNode(false), hiddenPort(0), hiddenType(RS_HIDDEN_TYPE_NONE)
{
sockaddr_storage_clear(localaddr);
@ -107,13 +107,13 @@ std::string textPeerConnectState(peerState &state)
std::string out = "Id: " + state.id.toStdString() + "\n";
rs_sprintf_append(out, "NetMode: %lu\n", state.netMode);
rs_sprintf_append(out, "VisState: Disc: %u Dht: %u\n", state.vs_disc, state.vs_dht);
out += "laddr: ";
out += sockaddr_storage_tostring(state.localaddr);
out += "\neaddr: ";
out += sockaddr_storage_tostring(state.serveraddr);
out += "\n";
return out;
}
@ -136,7 +136,7 @@ p3PeerMgrIMPL::p3PeerMgrIMPL(const RsPeerId& ssl_own_id, const RsPgpId& gpg_own_
mOwnState.netMode = RS_NET_MODE_UPNP; // Default to UPNP.
mOwnState.vs_disc = RS_VS_DISC_FULL;
mOwnState.vs_dht = RS_VS_DHT_FULL;
// setup default ProxyServerAddress.
// Tor
sockaddr_storage_clear(mProxyServerAddressTor);
@ -154,7 +154,7 @@ p3PeerMgrIMPL::p3PeerMgrIMPL(const RsPeerId& ssl_own_id, const RsPgpId& gpg_own_
mProxyServerStatusTor = RS_NET_PROXY_STATUS_UNKNOWN ;
mProxyServerStatusI2P = RS_NET_PROXY_STATUS_UNKNOWN;
}
#ifdef PEER_DEBUG
std::cerr << "p3PeerMgr() Startup" << std::endl;
#endif
@ -222,8 +222,8 @@ bool p3PeerMgrIMPL::forceHiddenNode()
struct sockaddr_storage loopback;
sockaddr_storage_clear(loopback);
sockaddr_storage_ipv4_aton(loopback, "127.0.0.1");
uint16_t port = sockaddr_storage_port(mOwnState.localaddr);
sockaddr_storage_ipv4_setport(loopback, port);
uint16_t port = sockaddr_storage_port(mOwnState.localaddr);
sockaddr_storage_ipv4_setport(loopback, port);
setLocalAddress(AuthSSL::getAuthSSL()->OwnId(), loopback);
@ -254,7 +254,7 @@ bool p3PeerMgrIMPL::setOwnNetworkMode(uint32_t netMode)
changed = true;
}
}
// Pass on Flags to NetMgr.
mNetMgr->setNetworkMode((netMode & RS_NET_MODE_ACTUAL));
return changed;
@ -267,7 +267,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht)
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
std::string out;
rs_sprintf(out, "p3PeerMgr::setOwnVisState() Existing vis: %u/%u Input vis: %u/%u",
rs_sprintf(out, "p3PeerMgr::setOwnVisState() Existing vis: %u/%u Input vis: %u/%u",
mOwnState.vs_disc, mOwnState.vs_dht, vs_disc, vs_dht);
rslog(RSL_WARNING, p3peermgrzone, out);
@ -275,7 +275,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht)
std::cerr << out.c_str() << std::endl;
#endif
if (mOwnState.vs_disc != vs_disc || mOwnState.vs_dht != vs_dht)
if (mOwnState.vs_disc != vs_disc || mOwnState.vs_dht != vs_dht)
{
mOwnState.vs_disc = vs_disc;
mOwnState.vs_dht = vs_dht;
@ -283,7 +283,7 @@ bool p3PeerMgrIMPL::setOwnVisState(uint16_t vs_disc, uint16_t vs_dht)
IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
}
}
// Pass on Flags to NetMgr.
mNetMgr->setVisState(vs_disc, vs_dht);
@ -575,7 +575,7 @@ bool p3PeerMgrIMPL::setHiddenDomainPort(const RsPeerId &ssl_id, const std::strin
IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
if (ssl_id == AuthSSL::getAuthSSL()->OwnId())
if (ssl_id == AuthSSL::getAuthSSL()->OwnId())
{
mOwnState.hiddenNode = true;
mOwnState.hiddenDomain = domain;
@ -917,7 +917,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
if (id == AuthSSL::getAuthSSL()->OwnId())
if (id == AuthSSL::getAuthSSL()->OwnId())
{
#ifdef PEER_DEBUG
std::cerr << "p3PeerMgrIMPL::addFriend() cannot add own id as a friend." << std::endl;
@ -973,7 +973,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
/* setup connectivity parameters */
it->second.vs_disc = vs_disc;
it->second.vs_dht = vs_dht;
it->second.netMode = netMode;
it->second.lastcontact = lastContact;
@ -995,7 +995,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
pstate.id = id;
pstate.gpg_id = gpg_id;
pstate.name = AuthGPG::getAuthGPG()->getGPGName(gpg_id);
pstate.vs_disc = vs_disc;
pstate.vs_dht = vs_dht;
pstate.netMode = netMode;
@ -1018,7 +1018,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
mLinkMgr->addFriend(id, vs_dht != RS_VS_DHT_OFF);
}
service_flags &= servicePermissionFlags(gpg_id) ; // Always reduce the permissions.
service_flags &= servicePermissionFlags(gpg_id) ; // Always reduce the permissions.
#ifdef RS_CHATSERVER //Defined by chatserver
setServicePermissionFlags(gpg_id,RS_NODE_PERM_NONE) ;
#else
@ -1029,7 +1029,7 @@ bool p3PeerMgrIMPL::addFriend(const RsPeerId& input_id, const RsPgpId& input_gpg
printPeerLists(std::cerr);
mLinkMgr->printPeerLists(std::cerr);
#endif
return true;
}
@ -1071,10 +1071,10 @@ bool p3PeerMgrIMPL::removeFriend(const RsPgpId &id)
}
for(std::list<RsPeerId>::iterator rit = sslid_toRemove.begin(); rit != sslid_toRemove.end(); ++rit)
if (mFriendList.end() != (it = mFriendList.find(*rit)))
if (mFriendList.end() != (it = mFriendList.find(*rit)))
mFriendList.erase(it);
std::map<RsPgpId,ServicePermissionFlags>::iterator it2 = mFriendsPermissionFlags.find(id) ;
std::map<RsPgpId,ServicePermissionFlags>::iterator it2 = mFriendsPermissionFlags.find(id) ;
if(it2 != mFriendsPermissionFlags.end())
mFriendsPermissionFlags.erase(it2);
@ -1146,13 +1146,13 @@ bool p3PeerMgrIMPL::removeFriend(const RsPeerId &id, bool removePgpId)
}
for(std::list<RsPeerId>::iterator rit = sslid_toRemove.begin(); rit != sslid_toRemove.end(); ++rit)
if (mFriendList.end() != (it = mFriendList.find(*rit)))
if (mFriendList.end() != (it = mFriendList.find(*rit)))
mFriendList.erase(it);
std::map<RsPgpId,ServicePermissionFlags>::iterator it2 ;
for(std::list<RsPgpId>::iterator rit = pgpid_toRemove.begin(); rit != pgpid_toRemove.end(); ++rit)
if (mFriendsPermissionFlags.end() != (it2 = mFriendsPermissionFlags.find(*rit)))
if (mFriendsPermissionFlags.end() != (it2 = mFriendsPermissionFlags.find(*rit)))
mFriendsPermissionFlags.erase(it2);
#ifdef PEER_DEBUG
@ -1432,7 +1432,7 @@ bool p3PeerMgrIMPL::setLocalAddress( const RsPeerId &id,
if (changed)
{
IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
mNetMgr->setLocalAddress(addr);
mLinkMgr->setLocalAddress(addr);
}
@ -1499,9 +1499,9 @@ bool p3PeerMgrIMPL::setExtAddress( const RsPeerId &id,
changed = true;
}
}
mNetMgr->setExtAddress(addr);
return changed;
}
@ -1584,12 +1584,16 @@ bool p3PeerMgrIMPL::setDynDNS(const RsPeerId &id, const std::string &dyndns)
return changed;
}
namespace pqi {
struct ZeroedInt
{
ZeroedInt() { n=0 ;}
int n ;
};
}
bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, const sockaddr_storage &addr)
{
// The algorithm is the following:
@ -1617,10 +1621,10 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons
// Update a list of own IPs:
// - remove old values for that same peer
// - remove values for non connected peers
{
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
mReportedOwnAddresses[from] = addr_filtered ;
for(std::map<RsPeerId,sockaddr_storage>::iterator it(mReportedOwnAddresses.begin());it!=mReportedOwnAddresses.end();)
@ -1641,8 +1645,8 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons
std::cerr << "p3PeerMgr:: Current external address is calculated to be: " << sockaddr_storage_iptostring(current_best_ext_address_guess) << " (simultaneously reported by " << count << " peers)." << std::endl;
}
// now current
// now current
sockaddr_storage own_addr ;
@ -1665,7 +1669,7 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons
RsServer::notify()->AddFeedItem(RS_FEED_ITEM_SEC_IP_WRONG_EXTERNAL_IP_REPORTED, from.toStdString(), sockaddr_storage_iptostring(own_addr), sockaddr_storage_iptostring(addr));
}
// we could also sweep over all connected friends and see if some report a different address.
return true ;
@ -1673,46 +1677,46 @@ bool p3PeerMgrIMPL::addCandidateForOwnExternalAddress(const RsPeerId &from, cons
bool p3PeerMgrIMPL::locked_computeCurrentBestOwnExtAddressCandidate(sockaddr_storage& addr, uint32_t& count)
{
std::map<sockaddr_storage,ZeroedInt> addr_counts ;
std::map<sockaddr_storage, pqi::ZeroedInt> addr_counts ;
for(std::map<RsPeerId,sockaddr_storage>::iterator it(mReportedOwnAddresses.begin());it!=mReportedOwnAddresses.end();++it)
++addr_counts[it->second].n ;
#ifdef PEER_DEBUG
std::cerr << "Current ext addr statistics:" << std::endl;
#endif
count = 0 ;
for(std::map<sockaddr_storage,ZeroedInt>::const_iterator it(addr_counts.begin());it!=addr_counts.end();++it)
for(std::map<sockaddr_storage, pqi::ZeroedInt>::const_iterator it(addr_counts.begin());it!=addr_counts.end();++it)
{
if(uint32_t(it->second.n) > count)
{
addr = it->first ;
count = it->second.n ;
}
#ifdef PEER_DEBUG
std::cerr << sockaddr_storage_iptostring(it->first) << " : " << it->second.n << std::endl;
#endif
}
return true ;
}
bool p3PeerMgrIMPL::getExtAddressReportedByFriends(sockaddr_storage &addr, uint8_t& /*isstable*/)
{
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
uint32_t count ;
locked_computeCurrentBestOwnExtAddressCandidate(addr,count) ;
#ifdef PEER_DEBUG
std::cerr << "Estimation count = " << count << ". Trusted? = " << (count>=2) << std::endl;
#endif
return count >= 2 ;// 2 is not conservative enough. 3 should be probably better.
return count >= 2 ;// 2 is not conservative enough. 3 should be probably better.
}
static bool cleanIpList(std::list<pqiIpAddress>& lst,const RsPeerId& pid,p3LinkMgr *link_mgr)
@ -1765,7 +1769,7 @@ bool p3PeerMgrIMPL::updateAddressList(const RsPeerId& id, const pqiIpAddrSet
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
/* check if it is our own ip */
if (id == getOwnId())
if (id == getOwnId())
{
mOwnState.ipAddrs.updateAddrs(clean_set);
return true;
@ -1811,11 +1815,11 @@ bool p3PeerMgrIMPL::updateCurrentAddress(const RsPeerId& id, const pqiIpAddre
#ifdef PEER_DEBUG
std::cerr << "p3PeerMgrIMPL::updateCurrentAddress() called for id : " << id << std::endl;
#endif
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
/* cannot be own id */
/* check if it is a friend */
std::map<RsPeerId, peerState>::iterator it;
if (mFriendList.end() == (it = mFriendList.find(id)))
@ -1846,23 +1850,23 @@ bool p3PeerMgrIMPL::updateCurrentAddress(const RsPeerId& id, const pqiIpAddre
std::cerr << addrstr;
std::cerr << std::endl;
#endif
IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
return true;
}
bool p3PeerMgrIMPL::updateLastContact(const RsPeerId& id)
{
#ifdef PEER_DEBUG
std::cerr << "p3PeerMgrIMPL::updateLastContact() called for id : " << id << std::endl;
#endif
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
/* cannot be own id */
/* check if it is a friend */
std::map<RsPeerId, peerState>::iterator it;
if (mFriendList.end() == (it = mFriendList.find(id)))
@ -1877,7 +1881,7 @@ bool p3PeerMgrIMPL::updateLastContact(const RsPeerId& id)
it->second.lastcontact = time(NULL);
IndicateConfigChanged(); /**** INDICATE MSG CONFIG CHANGED! *****/
return true;
}
@ -2061,7 +2065,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
getProxyServerAddress(RS_HIDDEN_TYPE_TOR, proxy_addr_tor);
getProxyServerAddress(RS_HIDDEN_TYPE_I2P, proxy_addr_i2p);
mPeerMtx.lock(); /****** MUTEX LOCKED *******/
mPeerMtx.lock(); /****** MUTEX LOCKED *******/
RsPeerNetItem *item = new RsPeerNetItem();
item->clear();
@ -2088,14 +2092,14 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
item->vs_disc = mOwnState.vs_disc;
item->vs_dht = mOwnState.vs_dht;
item->lastContact = mOwnState.lastcontact;
item->localAddrV4.addr = mOwnState.localaddr;
item->extAddrV4.addr = mOwnState.serveraddr;
sockaddr_storage_clear(item->localAddrV6.addr);
sockaddr_storage_clear(item->extAddrV6.addr);
item->dyndns = mOwnState.dyndns;
mOwnState.ipAddrs.mLocal.loadTlv(item->localAddrList);
mOwnState.ipAddrs.mExt.loadTlv(item->extAddrList);
@ -2125,20 +2129,20 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
item->vs_dht = (it->second).vs_dht;
item->lastContact = (it->second).lastcontact;
item->localAddrV4.addr = (it->second).localaddr;
item->extAddrV4.addr = (it->second).serveraddr;
sockaddr_storage_clear(item->localAddrV6.addr);
sockaddr_storage_clear(item->extAddrV6.addr);
item->dyndns = (it->second).dyndns;
(it->second).ipAddrs.mLocal.loadTlv(item->localAddrList);
(it->second).ipAddrs.mExt.loadTlv(item->extAddrList);
item->domain_addr = (it->second).hiddenDomain;
item->domain_port = (it->second).hiddenPort;
saveData.push_back(item);
#ifdef PEER_DEBUG
std::cerr << "p3PeerMgrIMPL::saveList() Peer Config Item:" << std::endl;
@ -2150,7 +2154,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
RsPeerBandwidthLimitsItem *pblitem = new RsPeerBandwidthLimitsItem ;
pblitem->peers = mPeerBandwidthLimits ;
saveData.push_back(pblitem) ;
RsPeerServicePermissionItem *sitem = new RsPeerServicePermissionItem ;
for(std::map<RsPgpId,ServicePermissionFlags>::const_iterator it(mFriendsPermissionFlags.begin());it!=mFriendsPermissionFlags.end();++it)
@ -2158,11 +2162,11 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
sitem->pgp_ids.push_back(it->first) ;
sitem->service_flags.push_back(it->second) ;
}
saveData.push_back(sitem) ;
// Now save config for network digging strategies
RsConfigKeyValueSet *vitem = new RsConfigKeyValueSet ;
RsTlvKeyValue kv;
@ -2199,7 +2203,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
kv.key = kConfigKeyProxyServerPortI2P;
kv.value = sockaddr_storage_porttostring(proxy_addr_i2p);
vitem->tlvkvs.pairs.push_back(kv) ;
saveData.push_back(vitem);
/* save groups */
@ -2213,7 +2217,7 @@ bool p3PeerMgrIMPL::saveList(bool &cleanup, std::list<RsItem *>& saveData)
return true;
}
bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& maxDn)
bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& maxDn)
{
RsPgpId pgp_id ;
@ -2235,7 +2239,7 @@ bool p3PeerMgrIMPL::getMaxRates(const RsPeerId& pid,uint32_t& maxUp,uint32_t& ma
return getMaxRates(pgp_id,maxUp,maxDn) ;
}
bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& maxDn)
bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& maxDn)
{
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
@ -2254,25 +2258,25 @@ bool p3PeerMgrIMPL::getMaxRates(const RsPgpId& pid,uint32_t& maxUp,uint32_t& max
return false ;
}
}
bool p3PeerMgrIMPL::setMaxRates(const RsPgpId& pid,uint32_t maxUp,uint32_t maxDn)
bool p3PeerMgrIMPL::setMaxRates(const RsPgpId& pid,uint32_t maxUp,uint32_t maxDn)
{
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
PeerBandwidthLimits& p(mPeerBandwidthLimits[pid]) ;
if(maxUp == p.max_up_rate_kbs && maxDn == p.max_dl_rate_kbs)
return true ;
std::cerr << "Updating max rates for peer " << pid << " to " << maxUp << " kB/s (up), " << maxDn << " kB/s (dn)" << std::endl;
p.max_up_rate_kbs = maxUp ;
p.max_dl_rate_kbs = maxDn ;
IndicateConfigChanged();
return true ;
}
void p3PeerMgrIMPL::saveDone()
{
/* clean up the save List */
@ -2396,7 +2400,7 @@ bool p3PeerMgrIMPL::loadList(std::list<RsItem *>& load)
#ifdef PEER_DEBUG
std::cerr << "setting use_extr_addr_finder to " << useExtAddrFinder << std::endl ;
#endif
}
}
// Tor
else if (kit->key == kConfigKeyProxyServerIpAddrTor)
{
@ -2859,13 +2863,13 @@ bool p3PeerMgrIMPL::removeAllFriendLocations(const RsPgpId &gpgid)
{
return false;
}
std::list<RsPeerId>::iterator it;
for(it = sslIds.begin(); it != sslIds.end(); ++it)
{
removeFriend(*it, true);
}
return true;
}
@ -2877,7 +2881,7 @@ bool p3PeerMgrIMPL::getAssociatedPeers(const RsPgpId &gpg_id, std::list<RsPeerId
#ifdef PEER_DEBUG
std::cerr << "p3PeerMgr::getAssociatedPeers() for id : " << gpg_id << std::endl;
#endif
int count = 0;
std::map<RsPeerId, peerState>::iterator it;
for(it = mFriendList.begin(); it != mFriendList.end(); ++it)
@ -2890,10 +2894,10 @@ bool p3PeerMgrIMPL::getAssociatedPeers(const RsPgpId &gpg_id, std::list<RsPeerId
#ifdef PEER_DEBUG
std::cerr << "p3PeerMgr::getAssociatedPeers() found ssl id : " << it->first << std::endl;
#endif
}
}
return (count > 0);
}
@ -2930,10 +2934,10 @@ bool p3PeerMgrIMPL::removeBannedIps()
return true ;
}
// /* This only removes SSL certs, that are old... Can end up with no Certs per GPG Id
// /* This only removes SSL certs, that are old... Can end up with no Certs per GPG Id
// * We are removing the concept of a "DummyId" - There is no need for it.
// */
//
//
// bool isDummyFriend(RsPeerId id)
// {
// bool ret = (id.substr(0,5) == "dummy");
@ -2974,7 +2978,7 @@ bool p3PeerMgrIMPL::removeUnusedLocations()
{
RsStackMutex stack(mPeerMtx); /****** STACK LOCK MUTEX *******/
// First put a sensible number in all PGP ids
for(std::list<RsPgpId>::const_iterator it = pgpList.begin(); it != pgpList.end(); ++it)
@ -2984,7 +2988,7 @@ bool p3PeerMgrIMPL::removeUnusedLocations()
std::cerr << "p3PeerMgr::removeUnusedLocations()" << std::endl;
#endif
// Then compute the most recently used location for all PGP ids
for( std::map<RsPeerId, peerState>::iterator it = mFriendList.begin(); it != mFriendList.end(); ++it)
{
time_t& bst(mostRecentTime[it->second.gpg_id]) ;
@ -3010,4 +3014,4 @@ bool p3PeerMgrIMPL::removeUnusedLocations()
return true;
}

3
libretroshare/src/retroshare/rsfiles.h
View file

@ -32,6 +32,7 @@
#include <string>
#include "rstypes.h"
#include "rsturtle.h"
class RsFiles;
extern RsFiles *rsFiles;
@ -212,6 +213,8 @@ public:
virtual uint32_t getMaxUploadSlotsPerFriend()=0;
virtual void setFilePermDirectDL(uint32_t perm)=0;
virtual uint32_t filePermDirectDL()=0;
virtual TurtleRequestId turtleSearch(const std::string& string_to_match) =0;
virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) =0;
/***
* Control of Downloads Priority.

65
libretroshare/src/retroshare/rsgxschannels.h
View file

@ -33,8 +33,7 @@
#include "retroshare/rstokenservice.h"
#include "retroshare/rsgxsifacehelper.h"
#include "retroshare/rsgxscommon.h"
#include "retroshare/rsturtle.h"
/* The Main Interface Class - for information about your Peers */
class RsGxsChannels;
@ -77,55 +76,49 @@ std::ostream &operator<<(std::ostream &out, const RsGxsChannelPost &post);
class RsGxsChannels: public RsGxsIfaceHelper, public RsGxsCommentService
{
public:
public:
explicit RsGxsChannels(RsGxsIface *gxs)
:RsGxsIfaceHelper(gxs) {}
:RsGxsIfaceHelper(gxs) {}
virtual ~RsGxsChannels() {}
/* Specific Service Data */
virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups) = 0;
virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts, std::vector<RsGxsComment> &cmts) = 0;
virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts) = 0;
//Not currently used
//virtual bool getRelatedPosts(const uint32_t &token, std::vector<RsGxsChannelPost> &posts) = 0;
virtual bool getGroupData(const uint32_t &token, std::vector<RsGxsChannelGroup> &groups) = 0;
virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts, std::vector<RsGxsComment> &cmts) = 0;
virtual bool getPostData(const uint32_t &token, std::vector<RsGxsChannelPost> &posts) = 0;
/* From RsGxsCommentService */
//virtual bool getCommentData(const uint32_t &token, std::vector<RsGxsComment> &comments) = 0;
//virtual bool getRelatedComments(const uint32_t &token, std::vector<RsGxsComment> &comments) = 0;
//virtual bool createComment(uint32_t &token, RsGxsComment &comment) = 0;
//virtual bool createVote(uint32_t &token, RsGxsVote &vote) = 0;
//////////////////////////////////////////////////////////////////////////////
/// Distant synchronisation methods ///
//////////////////////////////////////////////////////////////////////////////
///
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id)=0;
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string)=0;
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &results) =0;
virtual bool clearDistantSearchResults(TurtleRequestId req)=0;
virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)=0;
//////////////////////////////////////////////////////////////////////////////
virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0;
//////////////////////////////////////////////////////////////////////////////
virtual void setMessageReadStatus(uint32_t& token, const RsGxsGrpMsgIdPair& msgId, bool read) = 0;
virtual bool setChannelAutoDownload(const RsGxsGroupId &groupId, bool enabled) = 0;
virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled) = 0;
virtual bool setChannelAutoDownload(const RsGxsGroupId &groupId, bool enabled) = 0;
virtual bool getChannelAutoDownload(const RsGxsGroupId &groupid, bool& enabled) = 0;
virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory)=0;
virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory)=0;
virtual bool setChannelDownloadDirectory(const RsGxsGroupId &groupId, const std::string& directory)=0;
virtual bool getChannelDownloadDirectory(const RsGxsGroupId &groupId, std::string& directory)=0;
//virtual void setChannelAutoDownload(uint32_t& token, const RsGxsGroupId& groupId, bool autoDownload) = 0;
//virtual bool setMessageStatus(const std::string &msgId, const uint32_t status, const uint32_t statusMask);
//virtual bool setGroupSubscribeFlags(const std::string &groupId, uint32_t subscribeFlags, uint32_t subscribeMask);
//virtual bool groupRestoreKeys(const std::string &groupId);
virtual bool groupShareKeys(const RsGxsGroupId &groupId, std::set<RsPeerId>& peers)=0;
virtual bool groupShareKeys(const RsGxsGroupId &groupId, std::set<RsPeerId>& peers)=0;
// Overloaded subscribe fn.
virtual bool subscribeToGroup(uint32_t &token, const RsGxsGroupId &groupId, bool subscribe) = 0;
virtual bool subscribeToGroup(uint32_t &token, const RsGxsGroupId &groupId, bool subscribe) = 0;
virtual bool createGroup(uint32_t &token, RsGxsChannelGroup &group) = 0;
virtual bool createPost(uint32_t &token, RsGxsChannelPost &post) = 0;
virtual bool updateGroup(uint32_t &token, RsGxsChannelGroup &group) = 0;
// File Interface
virtual bool ExtraFileHash(const std::string &path, std::string filename) = 0;
virtual bool ExtraFileRemove(const RsFileHash &hash) = 0;
virtual bool createGroup(uint32_t &token, RsGxsChannelGroup &group) = 0;
virtual bool createPost(uint32_t &token, RsGxsChannelPost &post) = 0;
virtual bool updateGroup(uint32_t &token, RsGxsChannelGroup &group) = 0;
// File Interface
virtual bool ExtraFileHash(const std::string &path, std::string filename) = 0;
virtual bool ExtraFileRemove(const RsFileHash &hash) = 0;
};

92
libretroshare/src/retroshare/rsgxsdistsync.h Normal file
View file

@ -0,0 +1,92 @@
/*
* RetroShare C++ Interface.
*
* Copyright 2018 by Cyril Soler
*
* This library is free software; you can redistribute it and/or
* modify it under the terms of the GNU Library General Public
* License Version 2 as published by the Free Software Foundation.
*
* This library is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Library General Public License for more details.
*
* You should have received a copy of the GNU Library General Public
* License along with this library; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
* USA.
*
* Please report all bugs and problems to "retroshare@lunamutt.com".
*
*/
#pragma once
#include "retroshare/rsfiles.h"
#include "retroshare/rsturtle.h"
typedef RsPeerId RsGxsNetTunnelVirtualPeerId ;
struct RsGxsNetTunnelVirtualPeerInfo
{
enum { RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN = 0x00, // unknown status.
RS_GXS_NET_TUNNEL_VP_STATUS_TUNNEL_OK = 0x01, // tunnel has been established and we're waiting for virtual peer id
RS_GXS_NET_TUNNEL_VP_STATUS_ACTIVE = 0x02 // virtual peer id is known. Data can transfer.
};
RsGxsNetTunnelVirtualPeerInfo() : vpid_status(RS_GXS_NET_TUNNEL_VP_STATUS_UNKNOWN), last_contact(0),side(0) { memset(encryption_master_key,0,32) ; }
virtual ~RsGxsNetTunnelVirtualPeerInfo(){}
uint8_t vpid_status ; // status of the peer
time_t last_contact ; // last time some data was sent/recvd
uint8_t side ; // client/server
uint8_t encryption_master_key[32];
TurtleVirtualPeerId turtle_virtual_peer_id ; // turtle peer to use when sending data to this vpid.
RsGxsGroupId group_id ; // group that virtual peer is providing
uint16_t service_id ; // this is used for checkng consistency of the incoming data
};
struct RsGxsNetTunnelGroupInfo
{
enum GroupStatus {
RS_GXS_NET_TUNNEL_GRP_STATUS_UNKNOWN = 0x00, // unknown status
RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE = 0x01, // no virtual peers requested, just waiting
RS_GXS_NET_TUNNEL_GRP_STATUS_VPIDS_AVAILABLE = 0x02 // some virtual peers are available. Data can be read/written
};
enum GroupPolicy {
RS_GXS_NET_TUNNEL_GRP_POLICY_UNKNOWN = 0x00, // nothing has been set
RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE = 0x01, // group is available for server side tunnels, but does not explicitely request tunnels
RS_GXS_NET_TUNNEL_GRP_POLICY_ACTIVE = 0x02, // group will only explicitely request tunnels if none available
RS_GXS_NET_TUNNEL_GRP_POLICY_REQUESTING = 0x03, // group explicitely requests tunnels
};
RsGxsNetTunnelGroupInfo() : group_policy(RS_GXS_NET_TUNNEL_GRP_POLICY_PASSIVE),group_status(RS_GXS_NET_TUNNEL_GRP_STATUS_IDLE),last_contact(0) {}
GroupPolicy group_policy ;
GroupStatus group_status ;
time_t last_contact ;
RsFileHash hash ;
uint16_t service_id ;
std::set<TurtleVirtualPeerId> virtual_peers ; // list of which virtual peers provide this group. Can me more than 1.
};
// This class is here to provide statistics about GXS dist sync internals. It
//
class RsGxsDistSync
{
public:
virtual void getStatistics(
std::map<RsGxsGroupId,RsGxsNetTunnelGroupInfo>& groups, // groups on the client and server side
std::map<RsGxsNetTunnelVirtualPeerId, RsGxsNetTunnelVirtualPeerInfo>& virtual_peers, // current virtual peers, which group they provide, and how to talk to them through turtle
std::map<TurtleVirtualPeerId,RsGxsNetTunnelVirtualPeerId>& turtle_vpid_to_net_tunnel_vpid,
Bias20Bytes& bias
) const =0;
};
extern RsGxsDistSync *rsGxsDistSync ;

25
libretroshare/src/retroshare/rsgxsiface.h
View file

@ -32,6 +32,30 @@
#include "gxs/rsgxsdata.h"
#include "retroshare/rsgxsifacetypes.h"
/*!
* \brief The RsGxsGroupSymmary struct
* This structure is used to transport group summary information when a GXS service is searched. It contains the group information
* as well as a context string to tell where the information was found. It is more compact than a GroupMeta object, so as to make
* search responses as light as possible.
*/
struct RsGxsGroupSummary
{
RsGxsGroupSummary() : publish_ts(0), number_of_messages(0),last_message_ts(0),sign_flags(0),popularity(0) {}
RsGxsGroupId group_id ;
std::string group_name ;
std::string group_description ;
std::string search_context ;
RsGxsId author_id ;
time_t publish_ts ;
uint32_t number_of_messages ;
time_t last_message_ts ;
uint32_t sign_flags ;
uint32_t popularity ;
};
/*!
* Stores ids of changed gxs groups and messages. It is used to notify the GUI about changes.
*/
@ -44,6 +68,7 @@ public:
std::map<RsGxsGroupId, std::set<RsGxsMessageId> > mMsgsMeta;
std::list<RsGxsGroupId> mGrps;
std::list<RsGxsGroupId> mGrpsMeta;
std::list<TurtleRequestId> mDistantSearchReqs;
};
/*!

12
libretroshare/src/retroshare/rsgxsifacetypes.h
View file

@ -69,18 +69,18 @@ struct RsGroupMetaData : RsSerializable
RsGxsGroupId mGroupId;
std::string mGroupName;
uint32_t mGroupFlags; // Combination of FLAG_PRIVACY_PRIVATE | FLAG_PRIVACY_RESTRICTED | FLAG_PRIVACY_PUBLIC
uint32_t mSignFlags; // Combination of RSGXS_GROUP_SIGN_PUBLISH_MASK & RSGXS_GROUP_SIGN_AUTHOR_MASK.
uint32_t mGroupFlags; // Combination of FLAG_PRIVACY_PRIVATE | FLAG_PRIVACY_RESTRICTED | FLAG_PRIVACY_PUBLIC: diffusion
uint32_t mSignFlags; // Combination of RSGXS_GROUP_SIGN_PUBLISH_MASK & RSGXS_GROUP_SIGN_AUTHOR_MASK, i.e. what signatures are required for parent and child msgs
time_t mPublishTs; // Mandatory.
RsGxsId mAuthorId; // Optional.
RsGxsId mAuthorId; // Author of the group. Left to "000....0" if anonymous
// for circles
RsGxsCircleId mCircleId;
uint32_t mCircleType;
RsGxsCircleId mCircleId; // Id of the circle to which the group is restricted
uint32_t mCircleType; // combination of CIRCLE_TYPE_{ PUBLIC,EXTERNAL,YOUR_FRIENDS_ONLY,LOCAL,EXT_SELF,YOUR_EYES_ONLY }
// other stuff.
uint32_t mAuthenFlags;
uint32_t mAuthenFlags; // Actually not used yet.
RsGxsGroupId mParentGrpId;
// BELOW HERE IS LOCAL DATA, THAT IS NOT FROM MSG.

14
libretroshare/src/retroshare/rsgxsservice.h
View file

@ -6,6 +6,7 @@
#include "retroshare/rstokenservice.h"
struct RsMsgMetaData ;
typedef uint32_t TurtleRequestId;
typedef std::map<RsGxsGroupId, std::vector<RsMsgMetaData> > GxsMsgMetaMap;
typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsMsgMetaData> > GxsMsgRelatedMetaMap;
@ -18,7 +19,7 @@ typedef std::map<RsGxsGrpMsgIdPair, std::vector<RsMsgMetaData> > GxsMsgRelatedMe
struct RsGxsNotify
{
enum NotifyType
{ TYPE_PUBLISH, TYPE_RECEIVE, TYPE_PROCESSED, TYPE_PUBLISHKEY };
{ TYPE_PUBLISHED, TYPE_RECEIVED_NEW, TYPE_PROCESSED, TYPE_RECEIVED_PUBLISHKEY, TYPE_RECEIVED_DISTANT_SEARCH_RESULTS };
virtual ~RsGxsNotify() {}
virtual NotifyType getType() = 0;
@ -39,6 +40,17 @@ private:
bool mMetaChange;
};
class RsGxsDistantSearchResultChange: public RsGxsNotify
{
public:
RsGxsDistantSearchResultChange(TurtleRequestId id,const RsGxsGroupId& group_id) : mRequestId(id),mGroupId(group_id){}
NotifyType getType() { return TYPE_RECEIVED_DISTANT_SEARCH_RESULTS ; }
TurtleRequestId mRequestId ;
RsGxsGroupId mGroupId;
};
/*!
* Relevant to message changes
*/

2
libretroshare/src/retroshare/rsids.h
View file

@ -238,12 +238,14 @@ static const uint32_t RS_GENERIC_ID_GXS_TUNNEL_ID_TYPE = 0x0010 ;
static const uint32_t RS_GENERIC_ID_GXS_DISTANT_CHAT_ID_TYPE = 0x0011 ;
static const uint32_t RS_GENERIC_ID_NODE_GROUP_ID_TYPE = 0x0012 ;
static const uint32_t RS_GENERIC_ID_SHA256_ID_TYPE = 0x0013 ;
static const uint32_t RS_GENERIC_ID_20_BYTES_UNTYPED = 0x0014 ;
typedef t_RsGenericIdType< SSL_ID_SIZE , false, RS_GENERIC_ID_SSL_ID_TYPE> SSLIdType ;
typedef t_RsGenericIdType< PGP_KEY_ID_SIZE , true, RS_GENERIC_ID_PGP_ID_TYPE> PGPIdType ;
typedef t_RsGenericIdType< SHA1_SIZE , false, RS_GENERIC_ID_SHA1_ID_TYPE> Sha1CheckSum ;
typedef t_RsGenericIdType< SHA256_SIZE , false, RS_GENERIC_ID_SHA256_ID_TYPE> Sha256CheckSum ;
typedef t_RsGenericIdType< PGP_KEY_FINGERPRINT_SIZE, true, RS_GENERIC_ID_PGP_FINGERPRINT_TYPE> PGPFingerprintType ;
typedef t_RsGenericIdType< SHA1_SIZE , true, RS_GENERIC_ID_20_BYTES_UNTYPED> Bias20Bytes ;
typedef t_RsGenericIdType< CERT_SIGN_LEN , false, RS_GENERIC_ID_GXS_GROUP_ID_TYPE > GXSGroupId ;
typedef t_RsGenericIdType< CERT_SIGN_LEN , false, RS_GENERIC_ID_GXS_ID_TYPE > GXSId ;

2
libretroshare/src/retroshare/rsnotify.h
View file

@ -228,6 +228,8 @@ class NotifyClient
virtual void notifyCustomState (const std::string& /* peer_id */, const std::string& /* status_string */) {}
virtual void notifyHashingInfo (uint32_t /* type */, const std::string& /* fileinfo */) {}
virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list<TurtleFileInfo>& /* files */) {}
#warning MISSING CODE HERE
// virtual void notifyTurtleSearchResult (uint32_t /* search_id */, const std::list<TurtleGxsInfo >& /* groups */) {}
virtual void notifyPeerHasNewAvatar (std::string /* peer_id */) {}
virtual void notifyOwnAvatarChanged () {}
virtual void notifyOwnStatusMessageChanged () {}

7
libretroshare/src/retroshare/rsturtle.h
View file

@ -33,7 +33,9 @@
#include <list>
#include <vector>
#include "serialiser/rstlvbinary.h"
#include "retroshare/rstypes.h"
#include "retroshare/rsgxsifacetypes.h"
namespace RsRegularExpression { class LinearizedExpression ; }
class RsTurtleClientService ;
@ -42,6 +44,7 @@ class RsTurtle;
extern RsTurtle *rsTurtle ;
typedef uint32_t TurtleRequestId ;
typedef RsPeerId TurtleVirtualPeerId;
// This is the structure used to send back results of the turtle search
// to the notifyBase class, or send info to the GUI.
@ -52,6 +55,7 @@ struct TurtleFileInfo
std::string name ;
uint64_t size ;
};
struct TurtleTunnelRequestDisplayInfo
{
uint32_t request_id ; // Id of the request
@ -109,8 +113,7 @@ class RsTurtle
// the request id, which will be further used by the gui to store results
// as they come back.
//
virtual TurtleRequestId turtleSearch(const std::string& match_string) = 0 ;
virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) = 0 ;
virtual TurtleRequestId turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) =0;
// Initiates tunnel handling for the given file hash. tunnels. Launches
// an exception if an error occurs during the initialization process. The

60
libretroshare/src/rsitems/rsgxsitems.cc
View file

@ -6,6 +6,8 @@
*/
#include "serialiser/rstypeserializer.h"
#include "serialiser/rsbaseserial.h"
#include "rsgxsitems.h"
#include "gxs/rsgxsdata.h"
#include <iostream>
@ -70,4 +72,62 @@ std::ostream &operator<<(std::ostream &out, const RsMsgMetaData &meta)
return out;
}
template<> uint32_t RsTypeSerializer::serial_size(const TurtleGxsInfo& i)
{
uint32_t s = 0 ;
s += 2 ; // service_id
s += i.group_id.SIZE_IN_BYTES ;
s += GetTlvStringSize(i.name) ;
return s;
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[],uint32_t size,uint32_t& offset,TurtleGxsInfo& i)
{
uint32_t saved_offset = offset ;
bool ok = true ;
ok &= getRawUInt16(data, size, &offset, &i.service_id); // service_id
ok &= i.group_id.deserialise(data, size, offset); // group_id
ok &= GetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name); // group name
if(!ok)
offset = saved_offset ;
return ok;
}
template<> bool RsTypeSerializer::serialize(uint8_t data[],uint32_t size,uint32_t& offset,const TurtleGxsInfo& i)
{
uint32_t saved_offset = offset ;
bool ok = true ;
ok &= setRawUInt16(data, size, &offset, i.service_id); // service_id
ok &= i.group_id.serialise(data, size, offset); // group_id
ok &= SetTlvString(data, size, &offset, TLV_TYPE_STR_NAME, i.name); // group name
if(!ok)
offset = saved_offset ;
return ok;
}
template<> void RsTypeSerializer::print_data(const std::string& n, const TurtleGxsInfo& i)
{
std::cerr << " [GXS Info ] " << n << " group_id=" << i.group_id << " service=" << std::hex << i.service_id << std::dec << ", name=" << i.name << std::endl;
}
void RsTurtleGxsSearchResultGroupSummaryItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process(j,ctx,result,"result") ;
}
void RsTurtleGxsSearchResultGroupDataItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::TlvMemBlock_proxy prox(encrypted_nxs_group_data,encrypted_nxs_group_data_len) ;
RsTypeSerializer::serial_process(j,ctx,prox,"encrypted_nxs_data") ;
}
RS_TYPE_SERIALIZER_FROM_JSON_NOT_IMPLEMENTED_DEF(TurtleGxsInfo)
RS_TYPE_SERIALIZER_TO_JSON_NOT_IMPLEMENTED_DEF(TurtleGxsInfo)

76
libretroshare/src/rsitems/rsgxsitems.h
View file

@ -57,6 +57,82 @@ public:
RsMsgMetaData meta;
};
// We should make these items templates or generic classes so that each GXS service will handle them on its own.
static const uint8_t RS_PKT_SUBTYPE_GXS_SUBSTRING_SEARCH_ITEM = 0x20 ;
static const uint8_t RS_PKT_SUBTYPE_GXS_GROUP_SEARCH_ITEM = 0x21 ;
static const uint8_t RS_PKT_SUBTYPE_GXS_GROUP_SUMMARY_ITEM = 0x22 ;
static const uint8_t RS_PKT_SUBTYPE_GXS_GROUP_DATA_ITEM = 0x23 ;
class RsGxsTurtleSubStringSearchItem: public RsItem
{
public:
RsGxsTurtleSubStringSearchItem(uint16_t service): RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_SUBSTRING_SEARCH_ITEM) {}
virtual ~RsGxsTurtleSubStringSearchItem() {}
std::string match_string ; // string to match
std::string GetKeywords() { return match_string; }
void clear() { match_string.clear() ; }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
class RsGxsTurtleGroupSearchItem: public RsItem
{
public:
RsGxsTurtleGroupSearchItem(uint16_t service): RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_SEARCH_ITEM) {}
virtual ~RsGxsTurtleGroupSearchItem() {}
uint16_t service_id ; // searvice to search
Sha1CheckSum hashed_group_id ; // the group ID is hashed in order to keep it private.
std::string GetKeywords() { return std::string("Group request for [hashed] ")+hashed_group_id.toStdString() ; }
void clear() { hashed_group_id.clear() ; }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
struct TurtleGxsInfo
{
uint16_t service_id ;
RsGxsGroupId group_id ;
RsGxsId author;
std::string name ;
std::string description ;
time_t last_post ;
uint32_t number_of_posts ;
};
class RsTurtleGxsSearchResultGroupSummaryItem: public RsItem
{
public:
RsTurtleGxsSearchResultGroupSummaryItem(uint16_t service) : RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_SUMMARY_ITEM){}
virtual ~RsTurtleGxsSearchResultGroupSummaryItem() {}
std::list<TurtleGxsInfo> result ;
void clear() { result.clear() ; }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
class RsTurtleGxsSearchResultGroupDataItem: public RsItem
{
public:
RsTurtleGxsSearchResultGroupDataItem(uint16_t service) : RsItem(RS_PKT_VERSION_SERVICE,service,RS_PKT_SUBTYPE_GXS_GROUP_DATA_ITEM){}
virtual ~RsTurtleGxsSearchResultGroupDataItem() {}
unsigned char *encrypted_nxs_group_data; // data is encrypted with group ID. Only the requester, or anyone who already know the group id can decrypt.
uint32_t encrypted_nxs_group_data_len ;
void clear() { free(encrypted_nxs_group_data); encrypted_nxs_group_data=NULL; encrypted_nxs_group_data_len=0; }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};

26
libretroshare/src/rsitems/rsgxsupdateitems.cc
View file

@ -97,6 +97,31 @@ void RsGxsMsgUpdateItem::serial_process(RsGenericSerializer::SerializeJob j,RsGe
RsTypeSerializer::serial_process(j,ctx,msgUpdateInfos,"msgUpdateInfos");
}
template<> bool RsTypeSerializer::serialize(uint8_t data[], uint32_t size, uint32_t &offset, const RsGxsMsgUpdateItem::MsgUpdateInfo& info)
{
bool ok = true ;
ok = ok && setRawUInt32(data,size,&offset,info.time_stamp);
ok = ok && setRawUInt32(data,size,&offset,info.message_count);
return ok;
}
template<> bool RsTypeSerializer::deserialize(const uint8_t data[], uint32_t size, uint32_t &offset, RsGxsMsgUpdateItem::MsgUpdateInfo& info)
{
bool ok = true ;
ok = ok && getRawUInt32(data,size,&offset,&info.time_stamp);
ok = ok && getRawUInt32(data,size,&offset,&info.message_count);
return ok;
}
template<> uint32_t RsTypeSerializer::serial_size(const RsGxsMsgUpdateItem::MsgUpdateInfo& /* info */) { return 8; }
template<> void RsTypeSerializer::print_data(const std::string& name,const RsGxsMsgUpdateItem::MsgUpdateInfo& info)
{
std::cerr << "[MsgUpdateInfo]: " << name << ": " << info.time_stamp << ", " << info.message_count << std::endl;
}
void RsGxsServerMsgUpdateItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process (j,ctx,grpId,"grpId");
@ -109,3 +134,4 @@ void RsGxsGrpConfigItem::serial_process(RsGenericSerializer::SerializeJob j,RsGe
RsTypeSerializer::serial_process<uint32_t>(j,ctx,msg_send_delay,"msg_send_delay") ;
RsTypeSerializer::serial_process<uint32_t>(j,ctx,msg_req_delay,"msg_req_delay") ;
}

3
libretroshare/src/rsitems/rsgxsupdateitems.h
View file

@ -30,6 +30,7 @@
#include "gxs/rsgxs.h"
#include "gxs/rsgxsdata.h"
#include "gxs/rsgxsnettunnel.h"
#include "serialiser/rstlvidset.h"
#include "serialiser/rstypeserializer.h"
#include "serialiser/rsserializable.h"
@ -41,6 +42,7 @@ const uint8_t RS_PKT_SUBTYPE_GXS_MSG_UPDATE = 0x03;
const uint8_t RS_PKT_SUBTYPE_GXS_SERVER_GRP_UPDATE = 0x04;
const uint8_t RS_PKT_SUBTYPE_GXS_SERVER_MSG_UPDATE = 0x08;
const uint8_t RS_PKT_SUBTYPE_GXS_GRP_CONFIG = 0x09;
const uint8_t RS_PKT_SUBTYPE_GXS_RANDOM_BIAS = 0x0a;
class RsGxsNetServiceItem: public RsItem
{
@ -188,7 +190,6 @@ public:
RsGxsGroupId grpId;
};
class RsGxsUpdateSerialiser : public RsServiceSerializer
{
public:

88
libretroshare/src/rsserver/rsinit.cc
View file

@ -62,6 +62,7 @@
#include <fcntl.h>
#include "gxstunnel/p3gxstunnel.h"
#include "retroshare/rsgxsdistsync.h"
#include "file_sharing/p3filelists.h"
#define ENABLE_GROUTER
@ -1279,6 +1280,15 @@ int RsServer::StartupRetroShare()
RsNxsNetMgr* nxsMgr = new RsNxsNetMgrImpl(serviceCtrl);
/**** GXS Dist sync service ****/
#ifdef RS_USE_GXS_DISTANT_SYNC
RsGxsNetTunnelService *mGxsNetTunnel = new RsGxsNetTunnelService ;
rsGxsDistSync = mGxsNetTunnel ;
#else
RsGxsNetTunnelService *mGxsNetTunnel = NULL ;
#endif
/**** Identity service ****/
RsGeneralDataService* gxsid_ds = new RsDataService(currGxsDir + "/", "gxsid_db",
@ -1300,9 +1310,10 @@ int RsServer::StartupRetroShare()
RS_SERVICE_GXS_TYPE_GXSID, gxsid_ds, nxsMgr,
mGxsIdService, mGxsIdService->getServiceInfo(),
mReputations, mGxsCircles,mGxsIdService,
pgpAuxUtils,
false,false); // don't synchronise group automatic (need explicit group request)
pgpAuxUtils,mGxsNetTunnel,
false,false,true); // don't synchronise group automatic (need explicit group request)
// don't sync messages at all.
// allow distsync, so that we can grab GXS id requests for other services
// Normally we wouldn't need this (we do in other service):
// mGxsIdService->setNetworkExchangeService(gxsid_ns) ;
@ -1319,9 +1330,7 @@ int RsServer::StartupRetroShare()
RS_SERVICE_GXS_TYPE_GXSCIRCLE, gxscircles_ds, nxsMgr,
mGxsCircles, mGxsCircles->getServiceInfo(),
mReputations, mGxsCircles,mGxsIdService,
pgpAuxUtils,
true, // synchronise group automatic
true); // sync messages automatic, since they contain subscription requests.
pgpAuxUtils);
mGxsCircles->setNetworkExchangeService(gxscircles_ns) ;
@ -1373,7 +1382,7 @@ int RsServer::StartupRetroShare()
RS_SERVICE_GXS_TYPE_FORUMS, gxsforums_ds, nxsMgr,
mGxsForums, mGxsForums->getServiceInfo(),
mReputations, mGxsCircles,mGxsIdService,
pgpAuxUtils);
pgpAuxUtils);//,mGxsNetTunnel,true,true,true);
mGxsForums->setNetworkExchangeService(gxsforums_ns) ;
@ -1386,10 +1395,10 @@ int RsServer::StartupRetroShare()
// create GXS photo service
RsGxsNetService* gxschannels_ns = new RsGxsNetService(
RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr,
mGxsChannels, mGxsChannels->getServiceInfo(),
mReputations, mGxsCircles,mGxsIdService,
pgpAuxUtils);
RS_SERVICE_GXS_TYPE_CHANNELS, gxschannels_ds, nxsMgr,
mGxsChannels, mGxsChannels->getServiceInfo(),
mReputations, mGxsCircles,mGxsIdService,
pgpAuxUtils,mGxsNetTunnel,true,true,true);
mGxsChannels->setNetworkExchangeService(gxschannels_ns) ;
@ -1444,7 +1453,7 @@ int RsServer::StartupRetroShare()
RsGxsNetService* gxstrans_ns = new RsGxsNetService(
RS_SERVICE_TYPE_GXS_TRANS, gxstrans_ds, nxsMgr, mGxsTrans,
mGxsTrans->getServiceInfo(), mReputations, mGxsCircles,
mGxsIdService, pgpAuxUtils,true,true,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD);
mGxsIdService, pgpAuxUtils,NULL,true,true,false,p3GxsTrans::GXS_STORAGE_PERIOD,p3GxsTrans::GXS_SYNC_PERIOD);
mGxsTrans->setNetworkExchangeService(gxstrans_ns);
pqih->addService(gxstrans_ns, true);
@ -1477,9 +1486,11 @@ int RsServer::StartupRetroShare()
pqih -> addService(fdb,true);
pqih -> addService(ftserver,true);
mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ;
mGxsTunnels->connectToTurtleRouter(tr) ;
rsGxsTunnel = mGxsTunnels;
mGxsTunnels = new p3GxsTunnelService(mGxsIdService) ;
mGxsTunnels->connectToTurtleRouter(tr) ;
rsGxsTunnel = mGxsTunnels;
mGxsNetTunnel->connectToTurtleRouter(tr) ;
rsDisc = mDisc;
rsMsgs = new p3Msgs(msgSrv, chatSrv);
@ -1612,48 +1623,51 @@ int RsServer::StartupRetroShare()
serviceCtrl->registerServiceMonitor(mBwCtrl, mBwCtrl->getServiceInfo().mServiceType);
/**************************************************************************/
// Turtle search for GXS services
mGxsNetTunnel->registerSearchableService(gxschannels_ns) ;
/**************************************************************************/
//mConfigMgr->addConfiguration("ftserver.cfg", ftserver);
//
mConfigMgr->addConfiguration("gpg_prefs.cfg", AuthGPG::getAuthGPG());
mConfigMgr->loadConfiguration();
mConfigMgr->addConfiguration("peers.cfg", mPeerMgr);
mConfigMgr->addConfiguration("general.cfg", mGeneralConfig);
mConfigMgr->addConfiguration("msgs.cfg", msgSrv);
mConfigMgr->addConfiguration("chat.cfg", chatSrv);
mConfigMgr->addConfiguration("p3History.cfg", mHistoryMgr);
mConfigMgr->addConfiguration("p3Status.cfg", mStatusSrv);
mConfigMgr->addConfiguration("turtle.cfg", tr);
mConfigMgr->addConfiguration("gpg_prefs.cfg" , AuthGPG::getAuthGPG());
mConfigMgr->addConfiguration("gxsnettunnel.cfg", mGxsNetTunnel);
mConfigMgr->addConfiguration("peers.cfg" , mPeerMgr);
mConfigMgr->addConfiguration("general.cfg" , mGeneralConfig);
mConfigMgr->addConfiguration("msgs.cfg" , msgSrv);
mConfigMgr->addConfiguration("chat.cfg" , chatSrv);
mConfigMgr->addConfiguration("p3History.cfg" , mHistoryMgr);
mConfigMgr->addConfiguration("p3Status.cfg" , mStatusSrv);
mConfigMgr->addConfiguration("turtle.cfg" , tr);
#ifndef RETROTOR
mConfigMgr->addConfiguration("banlist.cfg", mBanList);
mConfigMgr->addConfiguration("banlist.cfg" , mBanList);
#endif
mConfigMgr->addConfiguration("servicecontrol.cfg", serviceCtrl);
mConfigMgr->addConfiguration("reputations.cfg", mReputations);
mConfigMgr->addConfiguration("reputations.cfg" , mReputations);
#ifdef ENABLE_GROUTER
mConfigMgr->addConfiguration("grouter.cfg", gr);
mConfigMgr->addConfiguration("grouter.cfg" , gr);
#endif
#ifdef RS_USE_BITDHT
mConfigMgr->addConfiguration("bitdht.cfg", mBitDht);
mConfigMgr->addConfiguration("bitdht.cfg" , mBitDht);
#endif
#ifdef RS_ENABLE_GXS
# ifdef RS_GXS_TRANS
mConfigMgr->addConfiguration("gxs_trans_ns.cfg", gxstrans_ns);
mConfigMgr->addConfiguration("gxs_trans.cfg", mGxsTrans);
mConfigMgr->addConfiguration("gxs_trans.cfg" , mGxsTrans);
# endif // RS_GXS_TRANS
mConfigMgr->addConfiguration("p3identity.cfg", mGxsIdService);
mConfigMgr->addConfiguration("identity.cfg", gxsid_ns);
mConfigMgr->addConfiguration("gxsforums.cfg", gxsforums_ns);
mConfigMgr->addConfiguration("p3identity.cfg" , mGxsIdService);
mConfigMgr->addConfiguration("identity.cfg" , gxsid_ns);
mConfigMgr->addConfiguration("gxsforums.cfg" , gxsforums_ns);
mConfigMgr->addConfiguration("gxsforums_srv.cfg", mGxsForums);
mConfigMgr->addConfiguration("gxschannels.cfg", gxschannels_ns);
mConfigMgr->addConfiguration("gxschannels.cfg" , gxschannels_ns);
mConfigMgr->addConfiguration("gxschannels_srv.cfg", mGxsChannels);
mConfigMgr->addConfiguration("gxscircles.cfg", gxscircles_ns);
mConfigMgr->addConfiguration("posted.cfg", posted_ns);
mConfigMgr->addConfiguration("gxscircles.cfg" , gxscircles_ns);
mConfigMgr->addConfiguration("posted.cfg" , posted_ns);
#ifdef RS_USE_WIKI
mConfigMgr->addConfiguration("wiki.cfg", wiki_ns);
#endif
@ -1824,6 +1838,8 @@ int RsServer::StartupRetroShare()
//rsWire = mWire;
/*** start up GXS core runner ***/
startServiceThread(mGxsNetTunnel, "gxs net tunnel");
startServiceThread(mGxsIdService, "gxs id");
startServiceThread(mGxsCircles, "gxs circle");
startServiceThread(mPosted, "gxs posted");

4
libretroshare/src/serialiser/rsserializer.h
View file

@ -154,7 +154,11 @@
#include <string.h>
#include <iostream>
#include <string>
#ifdef HAS_RAPIDJSON
#include <rapidjson/document.h>
#else
#include <rapid_json/document.h>
#endif // HAS_RAPIDJSON
#include "retroshare/rsflags.h"
#include "serialiser/rsserial.h"

5
libretroshare/src/serialiser/rstypeserializer.cc
View file

@ -36,8 +36,11 @@
#include <string>
#include <typeinfo> // for typeid
#include <rapid_json/document.h>
#ifdef HAS_RAPIDJSON
#include <rapidjson/prettywriter.h>
#else
#include <rapid_json/prettywriter.h>
#endif // HAS_RAPIDJSON
//static const uint32_t MAX_SERIALIZED_ARRAY_SIZE = 500 ;
static const uint32_t MAX_SERIALIZED_CHUNK_SIZE = 10*1024*1024 ; // 10 MB.

4
libretroshare/src/serialiser/rstypeserializer.h
View file

@ -35,7 +35,11 @@
#include "serialiser/rsserializer.h"
#include "serialiser/rsserializable.h"
#ifdef HAS_RAPIDJSON
#include <rapidjson/document.h>
#else
#include <rapid_json/document.h>
#endif // HAS_RAPIDJSON
#include <typeinfo> // for typeid
#include <type_traits>
#include <errno.h>

30
libretroshare/src/services/p3banlist.cc
View file

@ -55,7 +55,7 @@
#define RSBANLIST_DELAY_BETWEEN_TALK_TO_DHT 240 // every 4 mins.
/************ IMPLEMENTATION NOTES *********************************
*
*
* Get Bad Peers passed to us (from DHT mainly).
* we distribute and track the network list of bad peers.
*
@ -113,6 +113,8 @@ void p3BanList::setAutoRangeLimit(int n)
IndicateConfigChanged();
}
namespace services {
class ZeroedInt
{
public:
@ -120,6 +122,8 @@ class ZeroedInt
uint32_t n ;
};
}
BanListPeer::BanListPeer()
{
memset(&addr, 0, sizeof(addr));
@ -220,14 +224,14 @@ void p3BanList::autoFigureOutBanRanges()
std::cerr << "Automatically figuring out IP ranges from banned IPs." << std::endl;
#endif
std::map<sockaddr_storage,ZeroedInt> range_map ;
std::map<sockaddr_storage, services::ZeroedInt> range_map ;
for(std::map<sockaddr_storage,BanListPeer>::iterator it(mBanSet.begin());it!=mBanSet.end();++it)
++range_map[makeBitsRange(it->first,1)].n ;
time_t now = time(NULL) ;
for(std::map<sockaddr_storage,ZeroedInt>::const_iterator it=range_map.begin();it!=range_map.end();++it)
for(std::map<sockaddr_storage, services::ZeroedInt>::const_iterator it=range_map.begin();it!=range_map.end();++it)
{
#ifdef DEBUG_BANLIST
std::cerr << "Ban range: " << sockaddr_storage_iptostring(it->first) << " : " << it->second.n << std::endl;
@ -646,7 +650,7 @@ bool p3BanList::processIncoming()
break;
case RS_PKT_SUBTYPE_BANLIST_ITEM:
{
// Order is important!.
// Order is important!.
updated = (recvBanItem((RsBanListItem *) item) || updated);
}
break;
@ -669,8 +673,8 @@ bool p3BanList::processIncoming()
}
return true ;
}
}
bool p3BanList::recvBanItem(RsBanListItem *item)
{
@ -681,7 +685,7 @@ bool p3BanList::recvBanItem(RsBanListItem *item)
for(it = item->peerList.mList.begin(); it != item->peerList.mList.end(); ++it)
{
// Order is important!.
// Order is important!.
updated = (addBanEntry(item->PeerId(), it->addr.addr, it->level, it->reason, now - it->age) || updated);
}
return updated;
@ -961,7 +965,7 @@ bool p3BanList::addBanEntry( const RsPeerId &peerId,
it = mBanSources.find(peerId);
updated = true;
}
// index is FAMILY + IP - the rest should be Zeros..
struct sockaddr_storage bannedaddr;
sockaddr_storage_clear(bannedaddr);
@ -980,7 +984,7 @@ bool p3BanList::addBanEntry( const RsPeerId &peerId,
blp.level = level;
blp.mTs = time_stamp ;
blp.masked_bytes = 0 ;
it->second.mBanPeers[bannedaddr] = blp;
it->second.mLastUpdate = now;
updated = true;
@ -1036,7 +1040,7 @@ int p3BanList::condenseBanSources_locked()
time_t now = time(NULL);
RsPeerId ownId = mServiceCtrl->getOwnId();
#ifdef DEBUG_BANLIST
std::cerr << "p3BanList::condenseBanSources_locked()";
std::cerr << std::endl;
@ -1062,7 +1066,7 @@ int p3BanList::condenseBanSources_locked()
std::cerr << " Condensing Info from peer: " << it->first;
std::cerr << std::endl;
#endif
std::map<struct sockaddr_storage, BanListPeer>::const_iterator lit;
for(lit = it->second.mBanPeers.begin(); lit != it->second.mBanPeers.end(); ++lit)
{
@ -1135,7 +1139,7 @@ int p3BanList::condenseBanSources_locked()
}
}
#ifdef DEBUG_BANLIST
std::cerr << "p3BanList::condenseBanSources_locked() Printing New Set:";
std::cerr << std::endl;
@ -1280,7 +1284,7 @@ int p3BanList::printBanSet_locked(std::ostream &out)
int p3BanList::printBanSources_locked(std::ostream &out)
{
time_t now = time(NULL);
std::map<RsPeerId, BanList>::const_iterator it;
for(it = mBanSources.begin(); it != mBanSources.end(); ++it)
{

63
libretroshare/src/services/p3gxschannels.cc
View file

@ -236,7 +236,7 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange *>(*it);
if (msgChange)
{
if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVE)
if (msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW)
{
/* message received */
if (notify)
@ -289,11 +289,12 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
{
switch (grpChange->getType())
{
default:
case RsGxsNotify::TYPE_PROCESSED:
case RsGxsNotify::TYPE_PUBLISH:
case RsGxsNotify::TYPE_PUBLISHED:
break;
case RsGxsNotify::TYPE_RECEIVE:
case RsGxsNotify::TYPE_RECEIVED_NEW:
{
/* group received */
std::list<RsGxsGroupId> &grpList = grpChange->mGrpIdList;
@ -311,7 +312,7 @@ void p3GxsChannels::notifyChanges(std::vector<RsGxsNotify *> &changes)
break;
}
case RsGxsNotify::TYPE_PUBLISHKEY:
case RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY:
{
/* group received */
std::list<RsGxsGroupId> &grpList = grpChange->mGrpIdList;
@ -1686,3 +1687,57 @@ void p3GxsChannels::handle_event(uint32_t event_type, const std::string &elabel)
}
}
TurtleRequestId p3GxsChannels::turtleGroupRequest(const RsGxsGroupId& group_id)
{
return netService()->turtleGroupRequest(group_id) ;
}
TurtleRequestId p3GxsChannels::turtleSearchRequest(const std::string& match_string)
{
return netService()->turtleSearchRequest(match_string) ;
}
bool p3GxsChannels::clearDistantSearchResults(TurtleRequestId req)
{
return netService()->clearDistantSearchResults(req);
}
bool p3GxsChannels::retrieveDistantSearchResults(TurtleRequestId req,std::map<RsGxsGroupId,RsGxsGroupSummary>& results)
{
return netService()->retrieveDistantSearchResults(req,results);
}
bool p3GxsChannels::retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group)
{
RsGxsGroupSummary gs ;
if(netService()->retrieveDistantGroupSummary(group_id,gs))
{
// This is a placeholder information by the time we receive the full group meta data.
distant_group.mDescription = gs.group_description;
distant_group.mMeta.mGroupId = gs.group_id ;
distant_group.mMeta.mGroupName = gs.group_name;
distant_group.mMeta.mGroupFlags = GXS_SERV::FLAG_PRIVACY_PUBLIC ;
distant_group.mMeta.mSignFlags = gs.sign_flags;
distant_group.mMeta.mPublishTs = gs.publish_ts;
distant_group.mMeta.mAuthorId = gs.author_id;
distant_group.mMeta.mCircleType = GXS_CIRCLE_TYPE_PUBLIC ;// guessed, otherwise the group would not be search-able.
// other stuff.
distant_group.mMeta.mAuthenFlags = 0; // wild guess...
distant_group.mMeta.mSubscribeFlags = GXS_SERV::GROUP_SUBSCRIBE_NOT_SUBSCRIBED ;
distant_group.mMeta.mPop = gs.popularity; // Popularity = number of friend subscribers
distant_group.mMeta.mVisibleMsgCount = gs.number_of_messages; // Max messages reported by friends
distant_group.mMeta.mLastPost = gs.last_message_ts; // Timestamp for last message. Not used yet.
return true ;
}
else
return false ;
}

6
libretroshare/src/services/p3gxschannels.h
View file

@ -72,6 +72,12 @@ virtual void service_tick();
virtual bool saveList(bool &cleanup, std::list<RsItem *>&saveList); // @see p3Config::saveList(bool &cleanup, std::list<RsItem *>&)
virtual bool loadList(std::list<RsItem *>& loadList); // @see p3Config::loadList(std::list<RsItem *>&)
virtual TurtleRequestId turtleGroupRequest(const RsGxsGroupId& group_id);
virtual TurtleRequestId turtleSearchRequest(const std::string& match_string);
virtual bool retrieveDistantSearchResults(TurtleRequestId req, std::map<RsGxsGroupId, RsGxsGroupSummary> &results) ;
virtual bool clearDistantSearchResults(TurtleRequestId req);
virtual bool retrieveDistantGroup(const RsGxsGroupId& group_id,RsGxsChannelGroup& distant_group);
// Overloaded to cache new groups.
virtual RsGenExchange::ServiceCreate_Return service_CreateGroup(RsGxsGrpItem* grpItem, RsTlvSecurityKeySet& keySet);

5
libretroshare/src/services/p3gxscircles.cc
View file

@ -218,7 +218,8 @@ void p3GxsCircles::notifyChanges(std::vector<RsGxsNotify *> &changes)
std::cerr << " Msgs for Group: " << mit->first << std::endl;
#endif
force_cache_reload(RsGxsCircleId(mit->first));
if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVE) )
if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) )
for (auto msgIdIt(mit->second.begin()), end(mit->second.end()); msgIdIt != end; ++msgIdIt)
{
const RsGxsMessageId& msgId = *msgIdIt;
@ -261,7 +262,7 @@ void p3GxsCircles::notifyChanges(std::vector<RsGxsNotify *> &changes)
std::cerr << " forcing cache loading for circle " << *git << " in order to trigger subscribe update." << std::endl;
#endif
force_cache_reload(RsGxsCircleId(*git)) ;
if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVE) )
if (notify && (c->getType() == RsGxsNotify::TYPE_RECEIVED_NEW) )
notify->AddFeedItem(RS_FEED_ITEM_CIRCLE_INVIT_REC,RsGxsCircleId(*git).toStdString(),"");
}

7
libretroshare/src/services/p3gxsforums.cc
View file

@ -196,11 +196,12 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
switch (c->getType())
{
default:
case RsGxsNotify::TYPE_PROCESSED:
case RsGxsNotify::TYPE_PUBLISH:
case RsGxsNotify::TYPE_PUBLISHED:
break;
case RsGxsNotify::TYPE_RECEIVE:
case RsGxsNotify::TYPE_RECEIVED_NEW:
{
RsGxsMsgChange *msgChange = dynamic_cast<RsGxsMsgChange*>(c);
if (msgChange)
@ -239,7 +240,7 @@ void p3GxsForums::notifyChanges(std::vector<RsGxsNotify *> &changes)
break;
}
case RsGxsNotify::TYPE_PUBLISHKEY:
case RsGxsNotify::TYPE_RECEIVED_PUBLISHKEY:
{
RsGxsGroupChange *grpChange = dynamic_cast<RsGxsGroupChange *>(*it);
if (grpChange)

4
libretroshare/src/services/p3idservice.cc
View file

@ -278,7 +278,7 @@ void p3IdService::timeStampKey(const RsGxsId& gxs_id, const RsIdentityUsage& rea
return ;
}
#ifdef DEBUG_IDS
std::cerr << "(II) time stamping key " << gxs_id << " for the following reason: " << reason << std::endl;
std::cerr << "(II) time stamping key " << gxs_id << " for the following reason: " << reason.mUsageCode << std::endl;
#endif
RS_STACK_MUTEX(mIdMtx) ;
@ -2700,7 +2700,7 @@ void p3IdService::requestIdsFromNet()
bool request_can_proceed = false ;
for(cit2 = peers.begin(); cit2 != peers.end(); ++cit2)
if(rsPeers->isOnline(*cit2)) // make sure that the peer in online, so that we know that the request has some chance to succeed.
if(rsPeers->isOnline(*cit2) || mNes->isDistantPeer(*cit2)) // make sure that the peer in online, so that we know that the request has some chance to succeed.
{
requests[*cit2].push_back(cit->first);
request_can_proceed = true ;

4
libretroshare/src/services/p3postbase.cc
View file

@ -122,7 +122,7 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
// It could be taken a step further and directly request these msgs for an update.
addGroupForProcessing(mit->first);
if (notify && msgChange->getType() == RsGxsNotify::TYPE_RECEIVE)
if (notify && msgChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW)
{
for (auto mit1 = mit->second.begin(); mit1 != mit->second.end(); ++mit1)
{
@ -149,7 +149,7 @@ void p3PostBase::notifyChanges(std::vector<RsGxsNotify *> &changes)
std::cerr << std::endl;
#endif
if (notify && groupChange->getType() == RsGxsNotify::TYPE_RECEIVE)
if (notify && groupChange->getType() == RsGxsNotify::TYPE_RECEIVED_NEW)
{
notify->AddFeedItem(RS_FEED_ITEM_POSTED_NEW, git->toStdString());
}

527
libretroshare/src/turtle/p3turtle.cc
View file

@ -34,6 +34,7 @@
#endif
#include "rsserver/p3face.h"
#include "crypto/rscrypto.h"
#include "pqi/authssl.h"
#include "pqi/p3linkmgr.h"
@ -63,11 +64,14 @@ static std::map<TurtleTunnelRequestId, std::vector<time_t> > TS_request_bounces
void TS_dumpState() ;
#endif
#define TURTLE_DEBUG() std::cerr << time(NULL) << " : TURTLE : " << __FUNCTION__ << " : "
#define TURTLE_ERROR() std::cerr << "(EE) TURTLE ERROR : "
// These number may be quite important. I setup them with sensible values, but
// an in-depth test would be better to get an idea of what the ideal values
// could ever be.
//
// update of 14-03-11:
// update of 14-03-11:
// - I raised the cache time for tunnel requests. This avoids inconsistencies such as:
// * tunnel requests bouncing back while the original request is not in the cache anymore
// * special case of this for own file transfer: an outgoing tunnel is built with no end.
@ -79,17 +83,18 @@ void TS_dumpState() ;
// - The total number of TR per second emmited from self will be MAX_TUNNEL_REQS_PER_SECOND / TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 0.5
// - I updated forward probabilities to higher values, and min them to 1/nb_connected_friends to prevent blocking tunnels.
//
static const time_t TUNNEL_REQUESTS_LIFE_TIME = 240 ; /// life time for tunnel requests in the cache.
static const time_t SEARCH_REQUESTS_LIFE_TIME = 240 ; /// life time for search requests in the cache
static const time_t REGULAR_TUNNEL_DIGGING_TIME = 300 ; /// maximum interval between two tunnel digging campaigns.
static const time_t MAXIMUM_TUNNEL_IDLE_TIME = 60 ; /// maximum life time of an unused tunnel.
static const time_t EMPTY_TUNNELS_DIGGING_TIME = 50 ; /// look into tunnels regularly every 50 sec.
static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE = 5 ; /// estimate tunnel speed every 5 seconds
static const time_t TUNNEL_CLEANING_LAPS_TIME = 10 ; /// clean tunnels every 10 secs
static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 2 ; /// Tunnel management calls every 2 secs.
static const uint32_t MAX_TUNNEL_REQS_PER_SECOND = 1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before
static const uint32_t MAX_ALLOWED_SR_IN_CACHE = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS =5000 ; /// maximum number of search results forwarded back to the source.
static const time_t TUNNEL_REQUESTS_LIFE_TIME = 240 ; /// life time for tunnel requests in the cache.
static const time_t SEARCH_REQUESTS_LIFE_TIME = 240 ; /// life time for search requests in the cache
static const time_t REGULAR_TUNNEL_DIGGING_TIME = 300 ; /// maximum interval between two tunnel digging campaigns.
static const time_t MAXIMUM_TUNNEL_IDLE_TIME = 60 ; /// maximum life time of an unused tunnel.
static const time_t EMPTY_TUNNELS_DIGGING_TIME = 50 ; /// look into tunnels regularly every 50 sec.
static const time_t TUNNEL_SPEED_ESTIMATE_LAPSE = 5 ; /// estimate tunnel speed every 5 seconds
static const time_t TUNNEL_CLEANING_LAPS_TIME = 10 ; /// clean tunnels every 10 secs
static const time_t TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS = 2 ; /// Tunnel management calls every 2 secs.
static const uint32_t MAX_TUNNEL_REQS_PER_SECOND = 1 ; /// maximum number of tunnel requests issued per second. Was 0.5 before
static const uint32_t MAX_ALLOWED_SR_IN_CACHE = 120 ; /// maximum number of search requests allowed in cache. That makes 2 per sec.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_FILES =5000 ; /// maximum number of search results forwarded back to the source.
static const uint32_t TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT= 100 ; /// default maximum number of search results forwarded back source.
static const float depth_peer_probability[7] = { 1.0f,0.99f,0.9f,0.7f,0.6f,0.5,0.4f } ;
@ -151,13 +156,15 @@ void p3turtle::getItemNames(std::map<uint8_t,std::string>& names) const
{
names.clear();
names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST ] = "Search request";
names[RS_TURTLE_SUBTYPE_SEARCH_RESULT ] = "Search result";
names[RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST ] = "Filename substring search request";
names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST ] = "Generic search request";
names[RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT ] = "File search result";
names[RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT ] = "Generic search result";
names[RS_TURTLE_SUBTYPE_OPEN_TUNNEL ] = "Tunnel request";
names[RS_TURTLE_SUBTYPE_TUNNEL_OK ] = "Tunnel response";
names[RS_TURTLE_SUBTYPE_FILE_REQUEST ] = "Data request";
names[RS_TURTLE_SUBTYPE_FILE_DATA ] = "Data chunk";
names[RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST ] = "RegExp search";
names[RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST ] = "Filename RegExp search request";
names[RS_TURTLE_SUBTYPE_GENERIC_DATA ] = "Generic data";
names[RS_TURTLE_SUBTYPE_FILE_MAP ] = "Chunk map";
names[RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST ] = "Chunk map request";
@ -165,7 +172,7 @@ void p3turtle::getItemNames(std::map<uint8_t,std::string>& names) const
names[RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST ] = "Chunk CRC request";
}
void p3turtle::setEnabled(bool b)
void p3turtle::setEnabled(bool b)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_turtle_routing_enabled = b;
@ -184,7 +191,7 @@ bool p3turtle::enabled() const
}
void p3turtle::setSessionEnabled(bool b)
void p3turtle::setSessionEnabled(bool b)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_turtle_routing_session_enabled = b;
@ -242,7 +249,7 @@ int p3turtle::tick()
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
_last_tunnel_management_time = now ;
// Update traffic statistics. The constants are important: they allow a smooth variation of the
// Update traffic statistics. The constants are important: they allow a smooth variation of the
// traffic speed, which is used to moderate tunnel requests statistics.
//
_traffic_info = _traffic_info*0.9 + _traffic_info_buffer* (0.1 / (float)TIME_BETWEEN_TUNNEL_MANAGEMENT_CALLS) ;
@ -370,8 +377,8 @@ void p3turtle::manageTunnels()
// - the hash hasn't been tunneled for more than REGULAR_TUNNEL_DIGGING_TIME seconds, even if downloading.
//
// Candidate hashes are sorted, by olderness. The older gets tunneled first. At most MAX_TUNNEL_REQS_PER_SECOND are
// treated at once, as this method is called every second.
// Note: Because REGULAR_TUNNEL_DIGGING_TIME is larger than EMPTY_TUNNELS_DIGGING_TIME, files being downloaded get
// treated at once, as this method is called every second.
// Note: Because REGULAR_TUNNEL_DIGGING_TIME is larger than EMPTY_TUNNELS_DIGGING_TIME, files being downloaded get
// re-tunneled in priority. As this happens less, they don't obliterate tunneling for files that have no tunnels yet.
std::vector<std::pair<TurtleFileHash,time_t> > hashes_to_digg ;
@ -552,7 +559,7 @@ void p3turtle::autoWash()
// Now remove all the virtual peers ids at the client services. Off mutex!
//
for(uint32_t i=0;i<services_vpids_to_remove.size();++i)
{
#ifdef P3TURTLE_DEBUG
@ -614,7 +621,7 @@ void p3turtle::locked_closeTunnel(TurtleTunnelId tid,std::vector<std::pair<RsTur
// Let's be cautious. Normally we should never be here without consistent information,
// but still, this happens, rarely.
//
if(_virtual_peers.find(vpid) != _virtual_peers.end())
if(_virtual_peers.find(vpid) != _virtual_peers.end())
_virtual_peers.erase(_virtual_peers.find(vpid)) ;
std::map<TurtleFileHash,TurtleHashInfo>::iterator it(_incoming_file_hashes.find(hash)) ;
@ -659,7 +666,7 @@ void p3turtle::locked_closeTunnel(TurtleTunnelId tid,std::vector<std::pair<RsTur
// Also remove the associated virtual peer
//
if(_virtual_peers.find(vpid) != _virtual_peers.end())
if(_virtual_peers.find(vpid) != _virtual_peers.end())
_virtual_peers.erase(_virtual_peers.find(vpid)) ;
}
}
@ -724,7 +731,7 @@ bool p3turtle::loadList(std::list<RsItem*>& load)
RsConfigKeyValueSet *vitem = dynamic_cast<RsConfigKeyValueSet*>(*it) ;
if(vitem != NULL)
for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit)
for(std::list<RsTlvKeyValue>::const_iterator kit = vitem->tlvkvs.pairs.begin(); kit != vitem->tlvkvs.pairs.end(); ++kit)
{
if(kit->key == "TURTLE_CONFIG_MAX_TR_RATE")
{
@ -824,7 +831,7 @@ int p3turtle::handleIncoming()
RsTurtleGenericTunnelItem *gti = dynamic_cast<RsTurtleGenericTunnelItem *>(item) ;
if(gti != NULL)
routeGenericTunnelItem(gti) ; /// Generic packets, that travel through established tunnels.
routeGenericTunnelItem(gti) ; /// Generic packets, that travel through established tunnels.
else /// These packets should be destroyed by the client.
{
/// Special packets that require specific treatment, because tunnels do not exist for these packets.
@ -833,10 +840,12 @@ int p3turtle::handleIncoming()
switch(item->PacketSubType())
{
case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST:
case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST:
case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST: handleSearchRequest(dynamic_cast<RsTurtleSearchRequestItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_SEARCH_RESULT : handleSearchResult(dynamic_cast<RsTurtleSearchResultItem *>(item)) ;
case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT :
case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : handleSearchResult(dynamic_cast<RsTurtleSearchResultItem *>(item)) ;
break ;
case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : handleTunnelRequest(dynamic_cast<RsTurtleOpenTunnelItem *>(item)) ;
@ -862,8 +871,6 @@ int p3turtle::handleIncoming()
//
void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// take a look at the item and test against inconsistent values
// - If the item destimation is
@ -871,7 +878,7 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
std::cerr << "Received search request from peer " << item->PeerId() << ": " << std::endl ;
item->print(std::cerr,0) ;
#endif
uint32_t item_size = RsTurtleSerialiser().size(item);
if(item_size > TURTLE_MAX_SEARCH_REQ_ACCEPTED_SERIAL_SIZE)
@ -882,27 +889,57 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
std::cerr << " Caught a turtle search item with arbitrary large size from " << item->PeerId() << " of size " << item_size << " and depth " << item->depth << ". This is not allowed => dropping." << std::endl;
return ;
}
if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
if(_search_requests_origins.size() > MAX_ALLOWED_SR_IN_CACHE)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Dropping, because the search request cache is full." << std::endl ;
std::cerr << " Dropping, because the search request cache is full." << std::endl ;
#endif
std::cerr << " More than " << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. A peer is probably trying to flood your network See the depth charts to find him." << std::endl;
return ;
std::cerr << " More than " << MAX_ALLOWED_SR_IN_CACHE << " search request in cache. A peer is probably trying to flood your network See the depth charts to find him." << std::endl;
return ;
}
// If the item contains an already handled search request, give up. This
// happens when the same search request gets relayed by different peers
//
if(_search_requests_origins.find(item->request_id) != _search_requests_origins.end())
{
#ifdef P3TURTLE_DEBUG
std::cerr << " This is a bouncing request. Ignoring and deleting it." << std::endl ;
#endif
return ;
}
}
// If the item contains an already handled search request, give up. This
// happens when the same search request gets relayed by different peers
//
if(_search_requests_origins.find(item->request_id) != _search_requests_origins.end())
// Perform local search off-mutex,because this might call some services that are above turtle in the mutex chain.
uint32_t search_result_count = 0;
uint32_t max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_DEFAULT;
if(item->PeerId() != _own_id) // is the request not coming from us?
{
#ifdef P3TURTLE_DEBUG
std::cerr << " This is a bouncing request. Ignoring and deleting it." << std::endl ;
std::cerr << " Request not from us. Performing local search" << std::endl ;
#endif
return ;
std::list<RsTurtleSearchResultItem*> search_results ;
performLocalSearch(item,search_result_count,search_results,max_allowed_hits) ;
for(auto it(search_results.begin());it!=search_results.end();++it)
{
(*it)->request_id = item->request_id ;
(*it)->depth = 0 ;
(*it)->PeerId(item->PeerId()) ;
sendItem(*it) ;
}
}
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// This is a new request. Let's add it to the request map, and forward it to
// open peers.
@ -910,67 +947,17 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
req.origin = item->PeerId() ;
req.time_stamp = time(NULL) ;
req.depth = item->depth ;
req.result_count = 0;
req.result_count = search_result_count;
req.keywords = item->GetKeywords() ;
// If it's not for us, perform a local search. If something found, forward the search result back.
if(item->PeerId() != _own_id)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Request not from us. Performing local search" << std::endl ;
#endif
std::list<TurtleFileInfo> result ;
item->performLocalSearch(result) ;
RsTurtleSearchResultItem *res_item = NULL ;
uint32_t item_size = 0 ;
#ifdef P3TURTLE_DEBUG
if(!result.empty())
std::cerr << " " << result.size() << " matches found. Sending back to origin (" << item->PeerId() << ")." << std::endl ;
#endif
while(!result.empty() && req.result_count < TURTLE_SEARCH_RESULT_MAX_HITS)
{
// Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity.
//
static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ;
if(res_item == NULL)
{
res_item = new RsTurtleSearchResultItem ;
item_size = 0 ;
res_item->depth = 0 ;
res_item->request_id = item->request_id ;
res_item->PeerId(item->PeerId()) ; // send back to the same guy
}
res_item->result.push_back(result.front()) ;
++req.result_count ; // increase hit number for this particular search request.
item_size += 8 /* size */ + result.front().hash.serial_size() + result.front().name.size() ;
result.pop_front() ;
if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || result.empty() || req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ;
#endif
sendItem(res_item) ;
res_item = NULL ;
}
}
}
req.service_id = item->serviceId() ;
req.max_allowed_hits = max_allowed_hits;
// if enough has been sent back already, do not sarch further
#ifdef P3TURTLE_DEBUG
std::cerr << " result count = " << req.result_count << std::endl;
#endif
if(req.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
if(req.result_count >= max_allowed_hits)
return ;
// If search depth not too large, also forward this search request to all other peers.
@ -1007,14 +994,14 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
// Copy current item and modify it.
RsTurtleSearchRequestItem *fwd_item = item->clone() ;
// increase search depth, except in some rare cases, to prevent correlation between
// increase search depth, except in some rare cases, to prevent correlation between
// TR sniffing and friend names. The strategy is to not increase depth if the depth
// is 1:
// If B receives a TR of depth 1 from A, B cannot deduice that A is downloading the
// file, since A might have shifted the depth.
//
if(!random_dshift)
++(fwd_item->depth) ;
++(fwd_item->depth) ;
fwd_item->PeerId(*it) ;
@ -1028,67 +1015,194 @@ void p3turtle::handleSearchRequest(RsTurtleSearchRequestItem *item)
#endif
}
// This function should be removed in the future, when file search will also use generic search items.
void p3turtle::performLocalSearch(RsTurtleSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& search_results,uint32_t& max_allowed_hits)
{
RsTurtleFileSearchRequestItem *ftsearch = dynamic_cast<RsTurtleFileSearchRequestItem*>(item) ;
if(ftsearch != NULL)
{
performLocalSearch_files(ftsearch,req_result_count,search_results,max_allowed_hits) ;
return ;
}
RsTurtleGenericSearchRequestItem *gnsearch = dynamic_cast<RsTurtleGenericSearchRequestItem*>(item) ;
if(gnsearch != NULL)
{
performLocalSearch_generic(gnsearch,req_result_count,search_results,max_allowed_hits) ;
return ;
}
}
void p3turtle::performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
{
unsigned char *search_result_data = NULL ;
uint32_t search_result_data_len = 0 ;
RsTurtleClientService *client = NULL ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
auto it = _registered_services.find(item->service_id) ;
if(it == _registered_services.end())
return ;
client = it->second ;
}
if(client->receiveSearchRequest(item->search_data,item->search_data_len,search_result_data,search_result_data_len,max_allowed_hits))
{
RsTurtleGenericSearchResultItem *result_item = new RsTurtleGenericSearchResultItem ;
result_item->result_data = search_result_data ;
result_item->result_data_len = search_result_data_len ;
result.push_back(result_item) ;
}
}
void p3turtle::performLocalSearch_files(RsTurtleFileSearchRequestItem *item,uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits)
{
#ifdef P3TURTLE_DEBUG
std::cerr << "Performing rsFiles->search()" << std::endl ;
#endif
// now, search!
std::list<TurtleFileInfo> initialResults ;
item->search(initialResults) ;
#ifdef P3TURTLE_DEBUG
std::cerr << initialResults.size() << " matches found." << std::endl ;
#endif
result.clear() ;
RsTurtleFTSearchResultItem *res_item = NULL ;
uint32_t item_size = 0 ;
static const uint32_t RSTURTLE_MAX_SEARCH_RESPONSE_SIZE = 10000 ;
max_allowed_hits = TURTLE_SEARCH_RESULT_MAX_HITS_FILES;
for(auto it(initialResults.begin());it!=initialResults.end();++it)
{
if(res_item == NULL)
{
res_item = new RsTurtleFTSearchResultItem ;
item_size = 0 ;
result.push_back(res_item) ;
}
res_item->result.push_back(*it);
// Let's chop search results items into several chunks of finite size to avoid exceeding streamer's capacity.
//
++req_result_count ; // increase hit number for this particular search request.
item_size += 8 /* size */ + it->hash.serial_size() + it->name.size() ;
if(item_size > RSTURTLE_MAX_SEARCH_RESPONSE_SIZE || req_result_count >= max_allowed_hits)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Sending back chunk of size " << item_size << ", for " << res_item->result.size() << " elements." << std::endl ;
#endif
res_item = NULL ; // forces creation of a new item.
}
}
}
void p3turtle::handleSearchResult(RsTurtleSearchResultItem *item)
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// Find who actually sent the corresponding request.
//
std::map<TurtleRequestId,TurtleSearchRequestInfo>::iterator it = _search_requests_origins.find(item->request_id) ;
#ifdef P3TURTLE_DEBUG
std::cerr << "Received search result:" << std::endl ;
item->print(std::cerr,0) ;
#endif
if(it == _search_requests_origins.end())
std::list<std::pair<RsTurtleSearchResultItem*,RsTurtleClientService*> > results_to_notify_off_mutex ;
{
// This is an error: how could we receive a search result corresponding to a search item we
// have forwarded but that it not in the list ??
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
// Find who actually sent the corresponding request.
//
std::map<TurtleRequestId,TurtleSearchRequestInfo>::iterator it = _search_requests_origins.find(item->request_id) ;
std::cerr << __PRETTY_FUNCTION__ << ": search result has no peer direction!" << std::endl ;
return ;
}
// Is this result's target actually ours ?
if(it->second.origin == _own_id)
{
it->second.result_count += item->result.size() ;
returnSearchResult(item) ; // Yes, so send upward.
}
else
{ // Nope, so forward it back.
#ifdef P3TURTLE_DEBUG
std::cerr << " Forwarding result back to " << it->second.origin << std::endl;
std::cerr << "Received search result:" << std::endl ;
item->print(std::cerr,0) ;
#endif
// We update the total count forwarded back, and chop it to TURTLE_SEARCH_RESULT_MAX_HITS.
uint32_t n = item->result.size(); // not so good!
if(it->second.result_count >= TURTLE_SEARCH_RESULT_MAX_HITS)
if(it == _search_requests_origins.end())
{
std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl;
// This is an error: how could we receive a search result corresponding to a search item we
// have forwarded but that it not in the list ??
std::cerr << __PRETTY_FUNCTION__ << ": search result for request " << std::hex << item->request_id << std::dec << " has no peer direction!" << std::endl ;
return ;
}
if(it->second.result_count + n > TURTLE_SEARCH_RESULT_MAX_HITS)
{
for(uint32_t i=it->second.result_count + n; i>TURTLE_SEARCH_RESULT_MAX_HITS;--i)
item->result.pop_back() ;
// Is this result's target actually ours ?
it->second.result_count = TURTLE_SEARCH_RESULT_MAX_HITS ;
if(it->second.origin == _own_id)
{
it->second.result_count += item->count() ;
auto it2 = _registered_services.find(it->second.service_id) ;
if(it2 != _registered_services.end())
results_to_notify_off_mutex.push_back(std::make_pair(item,it2->second)) ;
else
std::cerr << "(EE) cannot find client service for ID " << std::hex << it->second.service_id << std::dec << ": search result item will be dropped." << std::endl;
}
else
it->second.result_count += n ;
{ // Nope, so forward it back.
#ifdef P3TURTLE_DEBUG
std::cerr << " Forwarding result back to " << it->second.origin << std::endl;
#endif
// We update the total count forwarded back, and chop it to TURTLE_SEARCH_RESULT_MAX_HITS.
RsTurtleSearchResultItem *fwd_item = new RsTurtleSearchResultItem(*item) ; // copy the item
uint32_t n = item->count(); // not so good!
// Normally here, we should setup the forward adress, so that the owner's
// of the files found can be further reached by a tunnel.
if(it->second.result_count >= it->second.max_allowed_hits)
{
std::cerr << "(WW) exceeded turtle search result to forward. Req=" << std::hex << item->request_id << std::dec << ": dropping item with " << n << " elements." << std::endl;
return ;
}
fwd_item->PeerId(it->second.origin) ;
fwd_item->depth = 0 ; // obfuscate the depth for non immediate friends. Result will always be 0. This effectively removes the information.
if(it->second.result_count + n > it->second.max_allowed_hits)
{
for(uint32_t i=it->second.result_count + n; i>it->second.max_allowed_hits;--i)
item->pop() ;
sendItem(fwd_item) ;
it->second.result_count = it->second.max_allowed_hits ;
}
else
it->second.result_count += n ;
RsTurtleSearchResultItem *fwd_item = item->duplicate();
// Normally here, we should setup the forward adress, so that the owner's
// of the files found can be further reached by a tunnel.
fwd_item->PeerId(it->second.origin) ;
fwd_item->depth = 0 ; // obfuscate the depth for non immediate friends. Result will always be 0. This effectively removes the information.
sendItem(fwd_item) ;
}
}
// now we notify clients off-mutex.
for(auto it(results_to_notify_off_mutex.begin());it!=results_to_notify_off_mutex.end();++it)
{
// Hack to use the old search result handling in ftServer. Normally ftServer should use the new method with serialized result.
#warning make sure memory is correctly deleted here
RsTurtleFTSearchResultItem *ftsr = dynamic_cast<RsTurtleFTSearchResultItem*>(it->first) ;
if(ftsr!=NULL)
{
RsServer::notify()->notifyTurtleSearchResult(ftsr->request_id,ftsr->result) ;
continue ;
}
RsTurtleGenericSearchResultItem *gnsr = dynamic_cast<RsTurtleGenericSearchResultItem*>(it->first) ;
if(gnsr!=NULL)
(*it).second->receiveSearchResult(gnsr->request_id,gnsr->result_data,gnsr->result_data_len) ;
}
}
// -----------------------------------------------------------------------------------//
@ -1150,8 +1264,8 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
// Let's figure out whether this packet is for us or not.
if(item->PeerId() == tunnel.local_dst && tunnel.local_src != _own_id) //direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT &&
{
if(item->PeerId() == tunnel.local_dst && tunnel.local_src != _own_id) //direction == RsTurtleGenericTunnelItem::DIRECTION_CLIENT &&
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Forwarding generic item to peer " << tunnel.local_src << std::endl ;
#endif
@ -1189,7 +1303,7 @@ void p3turtle::routeGenericTunnelItem(RsTurtleGenericTunnelItem *item)
// The packet was not forwarded, so it is for us. Let's treat it.
// This is done off-mutex, to avoid various deadlocks
//
handleRecvGenericTunnelItem(item) ;
delete item ;
@ -1312,13 +1426,13 @@ void p3turtle::sendTurtleData(const RsPeerId& virtual_peer_id,RsTurtleGenericTun
if(tunnel.local_src == _own_id)
{
item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_SERVER) ;
item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_SERVER) ;
item->PeerId(tunnel.local_dst) ;
_traffic_info_buffer.data_dn_Bps += ss ;
}
else if(tunnel.local_dst == _own_id)
{
item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ;
item->setTravelingDirection(RsTurtleGenericTunnelItem::DIRECTION_CLIENT) ;
item->PeerId(tunnel.local_src) ;
_traffic_info_buffer.data_up_Bps += ss ;
}
@ -1456,7 +1570,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
std::map<TurtleTunnelRequestId,TurtleTunnelRequestInfo>::iterator it = _tunnel_requests_origins.find(item->request_id) ;
if(it != _tunnel_requests_origins.end())
{
#ifdef P3TURTLE_DEBUG
@ -1559,7 +1673,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
// - the tunnel id will now be unique for a given route
// - allows a better balance of bandwidth for a given transfer
// - avoid the waste of items that get lost when re-routing a tunnel
#ifdef P3TURTLE_DEBUG
std::cerr << "Perturbating partial tunnel id. Original=" << std::hex << item->partial_tunnel_id ;
#endif
@ -1590,7 +1704,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
forward_probability = 1.0f / nb_online_ids ;
// Setting forward_probability to 1/nb_online_ids forces at most one TR up per TR dn. But if we are overflooded by
// TR dn, we still need to control them to avoid flooding the pqiHandler outqueue. So we additionally moderate the
// TR dn, we still need to control them to avoid flooding the pqiHandler outqueue. So we additionally moderate the
// forward probability so as to reduct the output rate accordingly.
//
if(_traffic_info.tr_dn_Bps / (float)TUNNEL_REQUEST_PACKET_SIZE > _max_tr_up_rate)
@ -1616,7 +1730,7 @@ void p3turtle::handleTunnelRequest(RsTurtleOpenTunnelItem *item)
// Copy current item and modify it.
RsTurtleOpenTunnelItem *fwd_item = new RsTurtleOpenTunnelItem(*item) ;
// increase search depth, except in some rare cases, to prevent correlation between
// increase search depth, except in some rare cases, to prevent correlation between
// TR sniffing and friend names. The strategy is to not increase depth if the depth
// is 1:
// If B receives a TR of depth 1 from A, B cannot deduice that A is downloading the
@ -1773,7 +1887,9 @@ void p3turtle::handleTunnelResult(RsTurtleTunnelOkItem *item)
// ------------------------------ IO with libretroshare ----------------------------//
// -----------------------------------------------------------------------------------//
//
void RsTurtleStringSearchRequestItem::performLocalSearch(std::list<TurtleFileInfo>& result) const
void RsTurtleStringSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
{
/* call to core */
std::list<DirDetails> initialResults;
@ -1796,7 +1912,7 @@ void RsTurtleStringSearchRequestItem::performLocalSearch(std::list<TurtleFileInf
for(std::list<DirDetails>::const_iterator it(initialResults.begin());it!=initialResults.end();++it)
{
// retain only file type
if (it->type == DIR_TYPE_DIR)
if (it->type == DIR_TYPE_DIR)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Skipping directory " << it->name << std::endl ;
@ -1812,7 +1928,7 @@ void RsTurtleStringSearchRequestItem::performLocalSearch(std::list<TurtleFileInf
result.push_back(i) ;
}
}
void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list<TurtleFileInfo>& result) const
void RsTurtleRegExpSearchRequestItem::search(std::list<TurtleFileInfo>& result) const
{
/* call to core */
std::list<DirDetails> initialResults;
@ -1835,7 +1951,7 @@ void RsTurtleRegExpSearchRequestItem::performLocalSearch(std::list<TurtleFileInf
for(std::list<DirDetails>::const_iterator it(initialResults.begin());it!=initialResults.end();++it)
{
// retain only file type
if (it->type == DIR_TYPE_DIR)
if (it->type == DIR_TYPE_DIR)
{
#ifdef P3TURTLE_DEBUG
std::cerr << " Skipping directory " << it->name << std::endl ;
@ -1907,6 +2023,34 @@ TurtleRequestId p3turtle::turtleSearch(const RsRegularExpression::LinearizedExpr
return id ;
}
TurtleRequestId p3turtle::turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service)
{
// generate a new search id.
TurtleRequestId id = generateRandomRequestId() ;
// Form a request packet that simulates a request from us.
//
RsTurtleGenericSearchRequestItem item ;
#ifdef P3TURTLE_DEBUG
std::cerr << "performing search. OwnId = " << _own_id << std::endl ;
#endif
item.PeerId(_own_id) ;
item.service_id = client_service->serviceId();
item.search_data = search_bin_data ;
item.search_data_len = search_bin_data_len ;
item.request_id = id ;
item.depth = 0 ;
// send it
handleSearchRequest(&item) ;
return id ;
}
void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *client_service,bool allow_multi_tunnels)
{
{
@ -1947,29 +2091,49 @@ void p3turtle::monitorTunnels(const RsFileHash& hash,RsTurtleClientService *clie
IndicateConfigChanged() ; // initiates saving of handled hashes.
}
void p3turtle::returnSearchResult(RsTurtleSearchResultItem *item)
{
// just cout for now, but it should be notified to the gui
#ifdef P3TURTLE_DEBUG
std::cerr << " Returning result for search request " << HEX_PRINT(item->request_id) << " upwards." << std::endl ;
#endif
RsServer::notify()->notifyTurtleSearchResult(item->request_id,item->result) ;
}
// RsTurtleGxsSearchResultGroupSummaryItem *gxs_sr_gs = dynamic_cast<RsTurtleGxsSearchResultGroupSummaryItem*>(item) ;
//
// if(gxs_sr_gs != NULL)
// {
// RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gs->request_id,gxs_sr_gs->result) ;
// return ;
// }
// RsTurtleGxsSearchResultGroupDataItem *gxs_sr_gd = dynamic_cast<RsTurtleGxsSearchResultGroupDataItem*>(item) ;
//
// if(gxs_sr_gd != NULL)
// {
//#warning MISSING CODE HERE TO HANDLE ENCRYPTED INCOMING GROUP DATA.
// //RsServer::notify()->notifyTurtleSearchResult(gxs_sr_gd->request_id,gxs_sr_gd->encrypted_nxs_group) ;
// return ;
// }
/// Warning: this function should never be called while the turtle mutex is locked.
/// Otherwize this is a possible source of cross-lock with the File mutex.
//
bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& peer_id,RsTurtleClientService *& service)
{
if(_registered_services.empty())
std::cerr << "Turtle router has no services registered. Tunnel requests cannot be handled." << std::endl;
std::map<uint16_t,RsTurtleClientService*> client_map ;
{
RsStackMutex stack(mTurtleMtx); /********** STACK LOCKED MTX ******/
for(std::list<RsTurtleClientService*>::const_iterator it(_registered_services.begin());it!=_registered_services.end();++it)
if( (*it)->handleTunnelRequest(hash,peer_id))
if(_registered_services.empty())
{
std::cerr << "Turtle router has no services registered. Tunnel requests cannot be handled." << std::endl;
return false ;
}
client_map = _registered_services ;
}
for(auto it(client_map.begin());it!=client_map.end();++it)
if( (*it).second->handleTunnelRequest(hash,peer_id))
{
service = *it ;
service = it->second ;
return true ;
}
@ -1979,14 +2143,9 @@ bool p3turtle::performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId&
void p3turtle::registerTunnelService(RsTurtleClientService *service)
{
#ifdef P3TURTLE_DEBUG
for(std::list<RsTurtleClientService*>::const_iterator it(_registered_services.begin());it!=_registered_services.end();++it)
if(service == *it)
throw std::runtime_error("p3turtle::registerTunnelService(): Cannot register the same service twice. Please fix the code!") ;
#endif
std::cerr << "p3turtle: registered new tunnel service " << (void*)service << std::endl;
std::cerr << "p3turtle: registered new tunnel service with ID=" << std::hex << service->serviceId() << std::dec << " and pointer " << (void*)service << std::endl;
_registered_services.push_back(service) ;
_registered_services[service->serviceId()] = service ;
_serialiser->registerClientService(service) ;
}
@ -2068,7 +2227,7 @@ std::string p3turtle::getPeerNameForVirtualPeerId(const RsPeerId& virtual_peer_i
std::map<TurtleVirtualPeerId,TurtleTunnelId>::const_iterator it(_virtual_peers.find(virtual_peer_id)) ;
if(it != _virtual_peers.end())
{
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2( _local_tunnels.find(it->second) ) ;
std::map<TurtleTunnelId,TurtleTunnel>::iterator it2( _local_tunnels.find(it->second) ) ;
if(it2 != _local_tunnels.end())
{
if(it2->second.local_src == _own_id)
@ -2080,6 +2239,28 @@ std::string p3turtle::getPeerNameForVirtualPeerId(const RsPeerId& virtual_peer_i
return name;
}
bool p3turtle::encryptData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item)
{
unsigned char *encrypted_data = NULL ;
uint32_t encrypted_data_len = 0 ;
if(!librs::crypto::encryptAuthenticateData(clear_data,clear_data_size,encryption_master_key,encrypted_data,encrypted_data_len))
{
delete encrypted_item ;
return false ;
}
encrypted_item = new RsTurtleGenericDataItem ;
encrypted_item->data_bytes = encrypted_data ;
encrypted_item->data_size = encrypted_data_len ;
return true;
}
bool p3turtle::decryptItem(const RsTurtleGenericDataItem* encrypted_item, uint8_t *encryption_master_key, unsigned char *& decrypted_data, uint32_t& decrypted_data_size)
{
return librs::crypto::decryptAuthenticateData((unsigned char*)encrypted_item->data_bytes,encrypted_item->data_size,encryption_master_key,decrypted_data,decrypted_data_size);
}
void p3turtle::getInfo( std::vector<std::vector<std::string> >& hashes_info,
std::vector<std::vector<std::string> >& tunnels_info,
std::vector<TurtleSearchRequestDisplayInfo >& search_reqs_info,
@ -2114,12 +2295,12 @@ void p3turtle::getInfo( std::vector<std::vector<std::string> >& hashes_info,
tunnel.push_back(printNumber(it->first,true)) ;
std::string name;
if(mLinkMgr->getPeerName(it->second.local_src,name))
if(mLinkMgr->getPeerName(it->second.local_src,name))
tunnel.push_back(name) ;
else
tunnel.push_back(it->second.local_src.toStdString()) ;
if(mLinkMgr->getPeerName(it->second.local_dst,name))
if(mLinkMgr->getPeerName(it->second.local_dst,name))
tunnel.push_back(name) ;
else
tunnel.push_back(it->second.local_dst.toStdString());

42
libretroshare/src/turtle/p3turtle.h
View file

@ -169,10 +169,12 @@ class TurtleSearchRequestInfo
{
public:
TurtlePeerId origin ; // where the request came from.
uint32_t time_stamp ; // last time the tunnel was actually used. Used for cleaning old tunnels.
int depth ; // depth of the request. Used to optimize tunnel length.
uint32_t result_count; // responses to this request. Useful to avoid spamming tunnel responses.
std::string keywords;
uint32_t time_stamp ; // last time the tunnel was actually used. Used for cleaning old tunnels.
int depth ; // depth of the request. Used to optimize tunnel length.
uint32_t result_count; // responses to this request. Useful to avoid spamming tunnel responses.
std::string keywords;
uint16_t service_id; // ID of the client service who issues the request. This is null if the request does not have a local origin.
uint32_t max_allowed_hits;// Max number of hits allowed for this search. This actually depends on the type of search (files, GXS groups, GXS group data, etc)
};
class TurtleTunnelRequestInfo
{
@ -244,11 +246,16 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
// the request id, which will be further used by the gui to store results
// as they come back.
//
// Eventually, search requests should be handled by client services. We will therefore
// remove the specific file search packets from the turtle router.
//
virtual TurtleSearchRequestId turtleSearch(const std::string& string_to_match) ;
virtual TurtleSearchRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
// The first two methods are old style search requests for FT, while the 3rd one is using a generic search data type, that is only to
// be deserialized by the service. The memory ownership is kept by the calling function. Similarly, the search response will be a
// generic data type that is to be deserialized by the client service.
//
// Eventually, search requests will use the generic system
// even for FT. We need to keep the old method for a while for backward compatibility.
//
virtual TurtleRequestId turtleSearch(const RsRegularExpression::LinearizedExpression& expr) ;
virtual TurtleRequestId turtleSearch(const std::string& string_to_match) ;
virtual TurtleRequestId turtleSearch(unsigned char *search_bin_data,uint32_t search_bin_data_len,RsTurtleClientService *client_service) ;
// Initiates tunnel handling for the given file hash. tunnels. Launches
// an exception if an error occurs during the initialization process. The
@ -332,6 +339,12 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
/// Send a data request into the correct tunnel for the given file hash
void sendTurtleData(const RsPeerId& virtual_peer_id, RsTurtleGenericTunnelItem *item) ;
/// Encrypts/decrypts an item, using a autenticated construction + chacha20, based on the given 32 bytes master key.
/// Input values are not touched (memory is not released). Memory ownership of outputs is left to the client.
///
static bool encryptData(const unsigned char *clear_data,uint32_t clear_data_size,uint8_t *encryption_master_key,RsTurtleGenericDataItem *& encrypted_item);
static bool decryptItem(const RsTurtleGenericDataItem *item, uint8_t* encryption_master_key, unsigned char *& decrypted_data,uint32_t& decrypted_data_size);
private:
//--------------------------- Admin/Helper functions -------------------------//
@ -385,10 +398,9 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
//------ Functions connecting the turtle router to other components.----------//
/// Performs a search calling local cache and search structure.
void performLocalSearch(const std::string& match_string,std::list<TurtleFileInfo>& result) ;
/// Returns a search result upwards (possibly to the gui)
void returnSearchResult(RsTurtleSearchResultItem *item) ;
void performLocalSearch (RsTurtleSearchRequestItem *item, uint32_t& req_result_count,std::list<RsTurtleSearchResultItem*>& result,uint32_t& max_allowed_hits) ;
void performLocalSearch_files (RsTurtleFileSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result, uint32_t &max_allowed_hits) ;
void performLocalSearch_generic(RsTurtleGenericSearchRequestItem *item, uint32_t& req_result_count, std::list<RsTurtleSearchResultItem*>& result, uint32_t &max_allowed_hits) ;
/// Returns true if the file with given hash is hosted locally, and accessible in anonymous mode the supplied peer.
virtual bool performLocalHashSearch(const TurtleFileHash& hash,const RsPeerId& client_peer_id,RsTurtleClientService *& service);
@ -410,7 +422,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
std::map<TurtleTunnelRequestId,TurtleTunnelRequestInfo> _tunnel_requests_origins ;
/// stores adequate tunnels for each file hash locally managed
std::map<TurtleFileHash,TurtleHashInfo> _incoming_file_hashes ;
std::map<TurtleFileHash,TurtleHashInfo> _incoming_file_hashes ;
/// stores file info for each file we provide.
std::map<TurtleTunnelId,RsTurtleClientService *> _outgoing_tunnel_client_services ;
@ -425,7 +437,7 @@ class p3turtle: public p3Service, public RsTurtle, public p3Config
std::set<TurtleFileHash> _hashes_to_remove ;
/// List of client services that have regitered.
std::list<RsTurtleClientService*> _registered_services ;
std::map<uint16_t,RsTurtleClientService*> _registered_services ;
time_t _last_clean_time ;
time_t _last_tunnel_management_time ;

51
libretroshare/src/turtle/rsturtleitem.cc
View file

@ -28,10 +28,12 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
{
case RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST : return new RsTurtleStringSearchRequestItem();
case RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST : return new RsTurtleRegExpSearchRequestItem();
case RS_TURTLE_SUBTYPE_SEARCH_RESULT : return new RsTurtleSearchResultItem();
case RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT : return new RsTurtleFTSearchResultItem();
case RS_TURTLE_SUBTYPE_OPEN_TUNNEL : return new RsTurtleOpenTunnelItem();
case RS_TURTLE_SUBTYPE_TUNNEL_OK : return new RsTurtleTunnelOkItem();
case RS_TURTLE_SUBTYPE_GENERIC_DATA : return new RsTurtleGenericDataItem();
case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST : return new RsTurtleGenericSearchRequestItem();
case RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT : return new RsTurtleGenericSearchResultItem();
default:
break ;
@ -48,20 +50,44 @@ RsItem *RsTurtleSerialiser::create_item(uint16_t service,uint8_t item_subtype) c
return NULL ;
}
std::string RsTurtleGenericSearchRequestItem::GetKeywords()
{
return std::string("Generic search : " + RsUtil::BinToHex(search_data,search_data_len,10));
}
void RsTurtleStringSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process (j,ctx,TLV_TYPE_STR_VALUE,match_string,"match_string") ;
RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth ,"depth") ;
}
void RsTurtleRegExpSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
RsTypeSerializer::serial_process(j,ctx,expr,"expr") ;
}
void RsTurtleGenericSearchRequestItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth,"depth") ;
RsTypeSerializer::serial_process<uint16_t>(j,ctx,service_id,"service_id") ;
RsTypeSerializer::serial_process<uint8_t >(j,ctx,request_type,"request_type") ;
RsTypeSerializer::TlvMemBlock_proxy prox(search_data,search_data_len) ;
RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ;
}
RsTurtleSearchRequestItem *RsTurtleGenericSearchRequestItem::clone() const
{
RsTurtleGenericSearchRequestItem *sr = new RsTurtleGenericSearchRequestItem ;
memcpy(sr,this,sizeof(RsTurtleGenericSearchRequestItem)) ;
sr->search_data = (unsigned char*)rs_malloc(search_data_len) ;
memcpy(sr->search_data,search_data,search_data_len) ;
return sr ;
}
template<> uint32_t RsTypeSerializer::serial_size(const RsRegularExpression::LinearizedExpression& r)
{
uint32_t s = 0 ;
@ -143,12 +169,31 @@ template<> void RsTypeSerializer::print_data(const std::string& n, const RsRegul
RS_TYPE_SERIALIZER_TO_JSON_NOT_IMPLEMENTED_DEF(RsRegularExpression::LinearizedExpression)
RS_TYPE_SERIALIZER_FROM_JSON_NOT_IMPLEMENTED_DEF(RsRegularExpression::LinearizedExpression)
void RsTurtleSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
void RsTurtleFTSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth ,"depth") ;
RsTypeSerializer::serial_process (j,ctx,result ,"result") ;
}
void RsTurtleGenericSearchResultItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)
{
RsTypeSerializer::serial_process<uint32_t>(j,ctx,request_id,"request_id") ;
RsTypeSerializer::serial_process<uint16_t>(j,ctx,depth ,"depth") ;
RsTypeSerializer::TlvMemBlock_proxy prox(result_data,result_data_len) ;
RsTypeSerializer::serial_process(j,ctx,prox,"search_data") ;
}
RsTurtleSearchResultItem *RsTurtleGenericSearchResultItem::duplicate() const
{
RsTurtleGenericSearchResultItem *sr = new RsTurtleGenericSearchResultItem ;
sr->result_data = (unsigned char*)rs_malloc(result_data_len) ;
memcpy(sr->result_data,result_data,result_data_len) ;
sr->result_data_len = result_data_len ;
sr->request_id = request_id ;
sr->depth = depth ;
return sr ;
}
template<> uint32_t RsTypeSerializer::serial_size(const TurtleFileInfo& i)
{

180
libretroshare/src/turtle/rsturtleitem.h
View file

@ -16,20 +16,24 @@
#include "serialiser/rsserializer.h"
const uint8_t RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST = 0x01 ;
const uint8_t RS_TURTLE_SUBTYPE_SEARCH_RESULT = 0x02 ;
const uint8_t RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT = 0x02 ;
const uint8_t RS_TURTLE_SUBTYPE_OPEN_TUNNEL = 0x03 ;
const uint8_t RS_TURTLE_SUBTYPE_TUNNEL_OK = 0x04 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_REQUEST = 0x07 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_DATA = 0x08 ;
const uint8_t RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST = 0x09 ;
const uint8_t RS_TURTLE_SUBTYPE_GENERIC_DATA = 0x0a ;
const uint8_t RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST = 0x0b ;
const uint8_t RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT = 0x0c ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP = 0x10 ;
const uint8_t RS_TURTLE_SUBTYPE_FILE_MAP_REQUEST = 0x11 ;
// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ; // unused
// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ;
const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC = 0x14 ;
const uint8_t RS_TURTLE_SUBTYPE_CHUNK_CRC_REQUEST = 0x15 ;
// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC = 0x12 ; // unused
// const uint8_t RS_TURTLE_SUBTYPE_FILE_CRC_REQUEST = 0x13 ;
class TurtleSearchRequestInfo ;
/***********************************************************************************/
/* Basic Turtle Item Class */
@ -46,49 +50,66 @@ class RsTurtleItem: public RsItem
/* Specific packets */
/***********************************************************************************/
class RsTurtleSearchResultItem: public RsTurtleItem
{
public:
RsTurtleSearchResultItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_SEARCH_RESULT), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT) ;}
// Class hierarchy is
//
// RsTurtleItem
// |
// +---- RsTurtleSearchRequestItem
// | |
// | +---- RsTurtleFileSearchRequestItem
// | | |
// | | +---- RsTurtleStringSearchRequestItem
// | | |
// | | +---- RsTurtleReqExpSearchRequestItem
// | |
// | +---- RsTurtleGenericSearchRequestItem
// |
// +---- RsTurtleSearchResultItem
// |
// +---- RsTurtleFTSearchResultItem
// |
// +---- RsTurtleGenericSearchResultItem
//
TurtleSearchRequestId request_id ; // Randomly generated request id.
uint16_t depth ; // The depth of a search result is obfuscated in this way:
// If the actual depth is 1, this field will be 1.
// If the actual depth is > 1, this field is a larger arbitrary integer.
std::list<TurtleFileInfo> result ;
void clear() { result.clear() ; }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
class RsTurtleSearchResultItem ;
class RsTurtleSearchRequestItem: public RsTurtleItem
{
public:
RsTurtleSearchRequestItem(uint32_t subtype) : RsTurtleItem(subtype), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_REQUEST) ;}
virtual ~RsTurtleSearchRequestItem() {}
virtual RsTurtleSearchRequestItem *clone() const = 0 ; // used for cloning in routing methods
virtual void performLocalSearch(std::list<TurtleFileInfo>&) const = 0 ; // abstracts the search method
virtual std::string GetKeywords() = 0;
virtual uint16_t serviceId() const= 0 ;
uint32_t request_id ; // randomly generated request id.
uint16_t depth ; // Used for limiting search depth.
};
class RsTurtleStringSearchRequestItem: public RsTurtleSearchRequestItem
class RsTurtleFileSearchRequestItem: public RsTurtleSearchRequestItem
{
public:
RsTurtleStringSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST) {}
RsTurtleFileSearchRequestItem(uint32_t subtype) : RsTurtleSearchRequestItem(subtype) {}
virtual ~RsTurtleFileSearchRequestItem() {}
virtual uint16_t serviceId() const { return RS_SERVICE_TYPE_FILE_TRANSFER ; }
virtual void search(std::list<TurtleFileInfo> &) const =0;
};
class RsTurtleStringSearchRequestItem: public RsTurtleFileSearchRequestItem
{
public:
RsTurtleStringSearchRequestItem() : RsTurtleFileSearchRequestItem(RS_TURTLE_SUBTYPE_STRING_SEARCH_REQUEST) {}
virtual ~RsTurtleStringSearchRequestItem() {}
virtual void search(std::list<TurtleFileInfo> &) const ;
std::string match_string ; // string to match
std::string GetKeywords() { return match_string; }
virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleStringSearchRequestItem(*this) ; }
virtual void performLocalSearch(std::list<TurtleFileInfo>&) const ;
void clear() { match_string.clear() ; }
@ -96,10 +117,11 @@ class RsTurtleStringSearchRequestItem: public RsTurtleSearchRequestItem
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem
class RsTurtleRegExpSearchRequestItem: public RsTurtleFileSearchRequestItem
{
public:
RsTurtleRegExpSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {}
RsTurtleRegExpSearchRequestItem() : RsTurtleFileSearchRequestItem(RS_TURTLE_SUBTYPE_REGEXP_SEARCH_REQUEST) {}
virtual ~RsTurtleRegExpSearchRequestItem() {}
RsRegularExpression::LinearizedExpression expr ; // Reg Exp in linearised mode
@ -110,15 +132,91 @@ class RsTurtleRegExpSearchRequestItem: public RsTurtleSearchRequestItem
delete ex;
return exs;
}
virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; }
virtual void performLocalSearch(std::list<TurtleFileInfo>&) const ;
virtual void search(std::list<TurtleFileInfo> &) const ;
virtual RsTurtleSearchRequestItem *clone() const { return new RsTurtleRegExpSearchRequestItem(*this) ; }
void clear() { expr = RsRegularExpression::LinearizedExpression(); }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
class RsTurtleGenericSearchRequestItem: public RsTurtleSearchRequestItem
{
public:
RsTurtleGenericSearchRequestItem() : RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST) {}
virtual ~RsTurtleGenericSearchRequestItem() { clear(); }
uint16_t service_id ; // service to search
uint32_t search_data_len ;
uint8_t request_type ; // type of request. This is used to limit the number of responses.
unsigned char *search_data ;
std::string GetKeywords() ;
virtual uint16_t serviceId() const { return service_id ; }
virtual RsTurtleSearchRequestItem *clone() const ;
virtual uint32_t requestType() const { return request_type; }
void clear() { free(search_data); search_data=NULL; search_data_len=0; }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
private:
RsTurtleGenericSearchRequestItem(const RsTurtleGenericSearchRequestItem&): RsTurtleSearchRequestItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_REQUEST) {} // make the object non copi-able.
RsTurtleGenericSearchRequestItem& operator=(const RsTurtleGenericSearchRequestItem&) { return *this;}
};
class RsTurtleSearchResultItem: public RsTurtleItem
{
public:
RsTurtleSearchResultItem(uint8_t subtype) : RsTurtleItem(subtype), request_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_SEARCH_RESULT) ;}
TurtleSearchRequestId request_id ; // Randomly generated request id.
uint16_t depth ; // The depth of a search result is obfuscated in this way:
// If the actual depth is 1, this field will be 1.
// If the actual depth is > 1, this field is a larger arbitrary integer.
virtual uint32_t count() const =0;
virtual void pop() =0;
virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx)=0;
virtual RsTurtleSearchResultItem *duplicate() const =0;
};
class RsTurtleFTSearchResultItem: public RsTurtleSearchResultItem
{
public:
RsTurtleFTSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_FT_SEARCH_RESULT){}
std::list<TurtleFileInfo> result ;
void clear() { result.clear() ; }
uint32_t count() const { return result.size() ; }
virtual void pop() { result.pop_back() ;}
virtual RsTurtleSearchResultItem *duplicate() const { return new RsTurtleFTSearchResultItem(*this) ; }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
class RsTurtleGenericSearchResultItem: public RsTurtleSearchResultItem
{
public:
RsTurtleGenericSearchResultItem() : RsTurtleSearchResultItem(RS_TURTLE_SUBTYPE_GENERIC_SEARCH_RESULT){}
virtual ~RsTurtleGenericSearchResultItem() {}
uint32_t count() const { return result_data_len/50 ; } // This is a blind size estimate. We should probably use the actual size to limit search results.
virtual void pop() {}
unsigned char *result_data ;
uint32_t result_data_len ;
virtual RsTurtleSearchResultItem *duplicate() const ;
void clear() { free(result_data); result_data=NULL; result_data_len=0; }
protected:
void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx);
};
/***********************************************************************************/
/* Turtle Tunnel Item classes */
/***********************************************************************************/
@ -128,10 +226,10 @@ class RsTurtleOpenTunnelItem: public RsTurtleItem
public:
RsTurtleOpenTunnelItem() : RsTurtleItem(RS_TURTLE_SUBTYPE_OPEN_TUNNEL), request_id(0), partial_tunnel_id(0), depth(0) { setPriorityLevel(QOS_PRIORITY_RS_TURTLE_OPEN_TUNNEL) ;}
TurtleFileHash file_hash ; // hash to match
uint32_t request_id ; // randomly generated request id.
TurtleFileHash file_hash ; // hash to match
uint32_t request_id ; // randomly generated request id.
uint32_t partial_tunnel_id ; // uncomplete tunnel id. Will be completed at destination.
uint16_t depth ; // Used for limiting search depth.
uint16_t depth ; // Used for limiting search depth.
void clear() { file_hash.clear() ;}
protected:
@ -167,24 +265,24 @@ class RsTurtleGenericTunnelItem: public RsTurtleItem
/// Does this packet stamps tunnels when it passes through ?
/// This is used for keeping trace weither tunnels are active or not.
virtual bool shouldStampTunnel() const = 0 ;
/// All tunnels derived from RsTurtleGenericTunnelItem should have a tunnel id to
/// All tunnels derived from RsTurtleGenericTunnelItem should have a tunnel id to
/// indicate which tunnel they are travelling through.
virtual TurtleTunnelId tunnelId() const { return tunnel_id ; }
/// Indicate weither the packet is a client packet (goign back to the
/// client) or a server packet (going to the server. Typically file
/// requests are server packets, whereas file data are client packets.
virtual Direction travelingDirection() const { return direction ; }
virtual void setTravelingDirection(Direction d) { direction = d; }
Direction direction ; // This does not need to be serialised. It's only used by the client services, optionnally,
// and is set by the turtle router according to which direction the item travels.
uint32_t tunnel_id ; // Id of the tunnel to travel through
};

140
libretroshare/src/turtle/turtleclientservice.h
View file

@ -42,51 +42,125 @@ class p3turtle ;
class RsTurtleClientService
{
public:
// Handling of tunnel request for the given hash. Most of the time, it's a search in a predefined list.
// The output info_string is used by the turtle router to display info about tunnels it manages. It is
// not passed to the tunnel.
/*!
* \brief serviceId
* Returns the ID of the client service. This is used to pass the ID to search requests, from the client services
* \return
* The service ID.
*/
virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) { return false ; }
virtual uint16_t serviceId() const
{
std::cerr << "!!!!!! Received request for service ID in turtle router client, but the client service is not handling it !!!!!!!" << std::endl ;
return 0 ;
}
/*!
* \brief handleTunnelRequest
Handling of tunnel request for the given hash. To be derived by the service in order to tell the turtle router
whether the service handles this hash or not. Most of the time, it's a search in a predefined list.
* \return true if the service
*/
virtual bool handleTunnelRequest(const RsFileHash& /*hash*/,const RsPeerId& /*peer_id*/) { return false ; }
// This method is called by the turtle router to send data that comes out of a turtle tunnel.
// The turtle router stays responsible for the memory management of data. Most of the time the
// data chunk is a serialized item to be de-serialized by the client service.
//
// Parameters:
// virtual_peer_id : name of the tunnel that sent the data
// data : memory chunk for the data
// size : size of data
// item->direction : direction of travel:
// RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a client
// RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a server
//
// Most of the time this parameter is not used by services, except when some info (such as chunk maps, chat items, etc) go
// both ways, and their nature cannot suffice to determine where they should be handled.
//
// By default (if not overloaded), the method will just free the data, as any subclass should do as well.
// Note: p3turtle stays owner of the item, so the client should not delete it!
//
virtual void receiveTurtleData(RsTurtleGenericTunnelItem */*item*/,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/)
/*!
* \brief receiveTurtleData
* This method is called by the turtle router to send data that comes out of a turtle tunnel, and should
* be overloaded by the client service.
* The turtle router stays responsible for the memory management of data. Most of the time the
* data chunk is a serialized item to be de-serialized by the client service.
*
* Parameters:
* virtual_peer_id : name of the tunnel that sent the data
* data : memory chunk for the data
* size : size of data
* item->direction : direction of travel:
* RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a client
* RsTurtleGenericTunnelItem::DIRECTION_CLIENT: the service is acting as a server
*
* Most of the time this parameter is not used by services, except when some info (such as chunk maps, chat items, etc) go
* both ways, and their nature cannot suffice to determine where they should be handled.
*
* By default (if not overloaded), the method will just free the data, as any subclass should do as well.
* Note: p3turtle stays owner of the item, so the client should not delete it!
*/
virtual void receiveTurtleData(const RsTurtleGenericTunnelItem * /* item */,const RsFileHash& /*hash*/,const RsPeerId& /*virtual_peer_id*/,RsTurtleGenericTunnelItem::Direction /*direction*/)
{
std::cerr << "!!!!!! Received Data from turtle router, but the client service is not handling it !!!!!!!!!!" << std::endl ;
}
// Method for creating specific items of the client service. The
// method has a default behavior of not doing anything, since most client
// services might only use the generic item already provided by the turtle
// router: RsTurtleGenericDataItem
/*!
* \brief receiveSearchRequest
* This method is called by the turtle router to notify the client of a search request in the form generic data. The returned
* result contains the serialised generic result returned by the client.
*
* The turtle router keeps the memory ownership over search_request_data
*
* \param search_request_data generic serialized search data
* \param search_request_data_len length of the serialized search data
* \param search_result_data generic serialized search result data
* \param search_result_data_len length of the serialized search result data
* \param max_allowed_hits max number of hits allowed to be sent back and forwarded
*
* \return true if the search is successful.
*/
virtual bool receiveSearchRequest(unsigned char */*search_request_data*/,
uint32_t /*search_request_data_len*/,
unsigned char *& /*search_result_data*/,
uint32_t& /*search_result_data_len*/,
uint32_t& /* max_allows_hits */)
{
std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
return false;
}
/*!
* \brief receiveSearchResult
* This method is called by the turtle router to notify the client of a search result. The result is serialized for the current class to read.
*
* \param search_result_data result data. Memory ownership is owned by the turtle router. So do not delete!
* \param search_result_data length of result data
*/
virtual void receiveSearchResult(TurtleSearchRequestId /* request_id */,unsigned char * /*search_result_data*/,uint32_t /*search_result_data_len*/)
{
std::cerr << "!!!!!! Received search result from turtle router, but the client service who requested it is not handling it !!!!!!!!!!" << std::endl ;
}
/*!
* \brief serializer
* Method for creating specific items of the client service. The
* method has a default behavior of not doing anything, since most client
* services might only use the generic item already provided by the turtle
* router: RsTurtleGenericDataItem
*
* \return the client's serializer is returned
*/
virtual RsServiceSerializer *serializer() { return NULL ; }
// These methods are called by the turtle router to add/remove virtual peers when tunnels are created/deleted
//
/*!
* \brief addVirtualPeer
* These methods are called by the turtle router to notify the client in order to add/remove virtual peers when tunnels are created/deleted
* These methods must be overloaded, because a service which does not care about tunel being openned or closed is not supposed to need tunnels.
*
* \param hash hash that the tunnel responds to
* \param virtual_peer_id virtual peer id provided by turtle to allow the client to send data into this tunnel. This peer is related to the tunnel itself
* rather than to its destination. As such, multiple peer ids may actually send data to the same computer because multiple tunnels
* arrive at the same location.
* \param dir dir indicates which side the cient will be talking to: CLIENT means that the client is the server. SERVER means that the client acts
* as a client (and therefore actually requested the tunnel).
*/
virtual void addVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id,RsTurtleGenericTunnelItem::Direction dir) = 0 ;
virtual void removeVirtualPeer(const TurtleFileHash& hash,const TurtleVirtualPeerId& virtual_peer_id) = 0 ;
// This function is mandatory. It should do two things:
// 1 - keep a pointer to the turtle router, so as to be able to send data (e.g. copy pt into a local variable)
// 2 - call pt->registerTunnelService(this), so that the TR knows that service and can send back information to it.
//
/*!
* \brief connectToTurtleRouter
* This function must be overloaded by the client. It should do two things:
* 1 - keep a pointer to the turtle router, so as to be able to send data (e.g. store pt into a local variable)
* 2 - call pt->registerTunnelService(this), so that the TR knows that service and can send back information to it.
*
* \param pt A pointer to the turtle router.
*/
virtual void connectToTurtleRouter(p3turtle *pt) = 0 ;
};

10
libretroshare/src/use_libretroshare.pri
View file

@ -16,7 +16,15 @@ bitdht {
# when rapidjson is mainstream on all distribs, we will not need the sources
# anymore in the meantime, they are part of the RS directory so that it is
# always possible to find them
INCLUDEPATH *= $$system_path($$clean_path($${PWD}/../../rapidjson-1.1.0))
RAPIDJSON_AVAILABLE = $$system(pkg-config --atleast-version 1.1 RapidJSON && echo yes)
isEmpty(RAPIDJSON_AVAILABLE) {
message("using built-in rapidjson")
INCLUDEPATH *= $$system_path($$clean_path($${PWD}/../../rapidjson-1.1.0))
} else {
message("using systems rapidjson")
DEFINES *= HAS_RAPIDJSON
}
sLibs =
mLibs = $$RS_SQL_LIB ssl crypto $$RS_THREAD_LIB $$RS_UPNP_LIB