diff --git a/libretroshare/src/grouter/grouteritems.cc b/libretroshare/src/grouter/grouteritems.cc index a3df21c82..302465414 100644 --- a/libretroshare/src/grouter/grouteritems.cc +++ b/libretroshare/src/grouter/grouteritems.cc @@ -108,7 +108,7 @@ void RsGRouterGenericDataItem::serial_process(RsGenericSerializer::SerializeJob RsTypeSerializer::serial_process(j,ctx,signature,"signature") ; RsTypeSerializer::serial_process(j,ctx,duplication_factor,"duplication_factor") ; - RsTypeSerializer::serial_process(j,ctx,flags,"flags") ; + RS_SERIAL_PROCESS(flags); if(j == RsGenericSerializer::DESERIALIZE) // make sure the duplication factor is not altered by friends. In the worst case, the item will duplicate a bit more. { @@ -128,7 +128,7 @@ void RsGRouterGenericDataItem::serial_process(RsGenericSerializer::SerializeJob void RsGRouterSignedReceiptItem::serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx) { RsTypeSerializer::serial_process (j,ctx,routing_id,"routing_id") ; - RsTypeSerializer::serial_process (j,ctx,flags,"flags") ; + RS_SERIAL_PROCESS(flags); RsTypeSerializer::serial_process (j,ctx,destination_key,"destination_key") ; RsTypeSerializer::serial_process (j,ctx,service_id,"service_id") ; RsTypeSerializer::serial_process (j,ctx,data_hash,"data_hash") ; @@ -269,3 +269,4 @@ RsGRouterSignedReceiptItem *RsGRouterSignedReceiptItem::duplicate() const return item ; } +RsGRouterAbstractMsgItem::~RsGRouterAbstractMsgItem() = default; diff --git a/libretroshare/src/grouter/grouteritems.h b/libretroshare/src/grouter/grouteritems.h index baf7c0576..14957ab87 100644 --- a/libretroshare/src/grouter/grouteritems.h +++ b/libretroshare/src/grouter/grouteritems.h @@ -27,7 +27,7 @@ #include "serialiser/rstlvkeys.h" #include "rsitems/rsserviceids.h" #include "retroshare/rstypes.h" - +#include "retroshare/rsflags.h" #include "retroshare/rsgrouter.h" #include "groutermatrix.h" @@ -83,40 +83,59 @@ class RsGRouterNonCopyableObject // and routing ID. Sub-items are responsible for providing the serialised data to be signed for // both signing and checking. -class RsGRouterAbstractMsgItem: public RsGRouterItem +enum class RsGRouterItemFlags : uint32_t { -public: - explicit RsGRouterAbstractMsgItem(uint8_t pkt_subtype) : RsGRouterItem(pkt_subtype), flags(0) {} - virtual ~RsGRouterAbstractMsgItem() {} + NONE = 0x0, + ENCRYPTED = 0x1, + SERVICE_UNKNOWN = 0x2 +}; +RS_REGISTER_ENUM_FLAGS_TYPE(RsGRouterItemFlags) + +struct RsGRouterAbstractMsgItem: RsGRouterItem +{ + explicit RsGRouterAbstractMsgItem(uint8_t pkt_subtype): + RsGRouterItem(pkt_subtype), flags(RsGRouterItemFlags::NONE) {} GRouterMsgPropagationId routing_id ; GRouterKeyId destination_key ; GRouterServiceId service_id ; RsTlvKeySignature signature ; // signs mid+destination_key+state - uint32_t flags ; // packet was delivered, not delivered, bounced, etc + + /// packet was delivered, not delivered, bounced, etc + RsGRouterItemFlags flags; + + ~RsGRouterAbstractMsgItem(); }; -class RsGRouterGenericDataItem: public RsGRouterAbstractMsgItem, public RsGRouterNonCopyableObject +class RsGRouterGenericDataItem: + public RsGRouterAbstractMsgItem, public RsGRouterNonCopyableObject { - public: - RsGRouterGenericDataItem() : RsGRouterAbstractMsgItem(RS_PKT_SUBTYPE_GROUTER_DATA), data_size(0), data_bytes(NULL), duplication_factor(0) { setPriorityLevel(QOS_PRIORITY_RS_GROUTER) ; } - virtual ~RsGRouterGenericDataItem() { clear() ; } +public: + RsGRouterGenericDataItem(): + RsGRouterAbstractMsgItem(RS_PKT_SUBTYPE_GROUTER_DATA), + data_size(0), data_bytes(nullptr), duplication_factor(0) + { setPriorityLevel(QOS_PRIORITY_RS_GROUTER); } - virtual void clear() - { - free(data_bytes); - data_bytes=NULL; - } + virtual ~RsGRouterGenericDataItem() { clear(); } + virtual void clear() + { + free(data_bytes); + data_bytes = nullptr; + } - virtual void serial_process(RsGenericSerializer::SerializeJob j,RsGenericSerializer::SerializeContext& ctx); + virtual void serial_process( + RsGenericSerializer::SerializeJob j, + RsGenericSerializer::SerializeContext& ctx ); - RsGRouterGenericDataItem *duplicate() const ; + RsGRouterGenericDataItem *duplicate() const; - // packet data - // - uint32_t data_size ; - uint8_t *data_bytes; - uint32_t duplication_factor ; // number of duplicates allowed. Should be capped at each de-serialise operation! + /// packet data + uint32_t data_size; + uint8_t* data_bytes; + + /** number of duplicates allowed. Should be capped at each de-serialise + * operation! */ + uint32_t duplication_factor; }; class RsGRouterSignedReceiptItem: public RsGRouterAbstractMsgItem diff --git a/libretroshare/src/grouter/p3grouter.cc b/libretroshare/src/grouter/p3grouter.cc index 6dbc444aa..44a09eab7 100644 --- a/libretroshare/src/grouter/p3grouter.cc +++ b/libretroshare/src/grouter/p3grouter.cc @@ -178,7 +178,7 @@ //////////////////////////////////////////////////////////////////////////////////////////////////////////// #include -#include +#include #include "util/rsrandom.h" #include "util/rsprint.h" @@ -188,7 +188,7 @@ #include "turtle/p3turtle.h" #include "gxs/rsgixs.h" #include "retroshare/rspeers.h" - +#include "util/cxx17retrocompat.h" #include "p3grouter.h" #include "grouteritems.h" #include "groutertypes.h" @@ -197,28 +197,22 @@ /**********************/ //#define GROUTER_DEBUG /**********************/ +#define GROUTER_DEBUG const std::string p3GRouter::SERVICE_INFO_APP_NAME = "Global Router" ; -p3GRouter::p3GRouter(p3ServiceControl *sc, RsGixs *is) - : p3Service(), p3Config(), mServiceControl(sc), mTurtle(NULL), mGixs(is), grMtx("GRouter") -{ - addSerialType(new RsGRouterSerialiser()) ; - - _last_autowash_time = 0 ; - _last_debug_output_time = 0 ; - _last_config_changed = 0 ; - _last_matrix_update_time = 0 ; - _debug_enabled = true ; - - _random_salt = RSRandom::random_u64() ; - - _changed = false ; -} +p3GRouter::p3GRouter(p3ServiceControl *sc, RsGixs *is) : + p3Service(), p3Config(), mServiceControl(sc), mTurtle(nullptr), mGixs(is), + grMtx("GRouter"), _changed(false), _debug_enabled(true), + _last_autowash_time(0), _last_matrix_update_time(0), + _last_debug_output_time(0), _last_config_changed(0), + _random_salt(RsRandom::random_u64()), + mMissingKeyQueueMtx("GRouterMissingKeyQueue") +{ addSerialType(new RsGRouterSerialiser()); } int p3GRouter::tick() { - rstime_t now = time(NULL) ; + rstime_t now = time(nullptr); // Sort incoming service data // @@ -242,6 +236,41 @@ int p3GRouter::tick() // handleTunnels() ; + /* Handle items in mMissingKeyQueue */ + if(now > mMissingKeyQueueCheckLastCheck + mMissingKeyQueueCheckEvery) + { + mMissingKeyQueueCheckLastCheck = now; + + RS_STACK_MUTEX(mMissingKeyQueueMtx); + for(auto it = mMissingKeyQueue.begin(); it != mMissingKeyQueue.end();) + { + const RsGxsId& senderId = it->first->signature.keyId; + if(rsIdentity->isKnownId(senderId)) + { + Dbg2() << __PRETTY_FUNCTION__ << " got key: " << senderId + << " for item pending validation, calling item handler" + << std::endl; + + handleIncomingItem(it->first.get()); + it = mMissingKeyQueue.erase(it); + } + else + { + Dbg3() << __PRETTY_FUNCTION__ << " requesting missing key: " + << senderId << " to validate pending item" << std::endl; + + /* At this point the network status may have varied a lot since + * we received the item, so we don't even know if the peer who + * forwarded the item is still online, moreover the fact that + * after specific request we haven't got the key yet suggests it + * is not a good route toward the key, so request it to all + * available peers */ + rsIdentity->requestIdentity(senderId); + ++it; + } + } + } + // Update routing matrix // if(now > _last_matrix_update_time + RS_GROUTER_MATRIX_UPDATE_PERIOD) @@ -325,14 +354,12 @@ bool p3GRouter::unregisterKey(const RsGxsId& key_id,const GRouterServiceId& sid) Sha1CheckSum hash = makeTunnelHash(key_id,sid) ; - std::map::iterator it = _owned_key_ids.find(hash) ; - + const auto it = _owned_key_ids.find(hash); if(it == _owned_key_ids.end()) { -#ifdef GROUTER_DEBUG - std::cerr << "p3GRouter::unregisterKey(): key " << key_id << " not found." << std::endl; -#endif - return false ; + RsErr() << __PRETTY_FUNCTION__ << " key " << key_id << " not found." + << std::endl; + return false; } #ifdef GROUTER_DEBUG @@ -479,7 +506,7 @@ void p3GRouter::handleLowLevelTransactionAckItem(RsGRouterTransactionAcknItem *t #endif } -void p3GRouter::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash &/*hash*/, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction /*direction*/) +void p3GRouter::receiveTurtleData(const RsTurtleGenericTunnelItem *gitem, const RsFileHash & hash, const RsPeerId &virtual_peer_id, RsTurtleGenericTunnelItem::Direction direction) { #ifdef GROUTER_DEBUG std::cerr << "p3GRouter::receiveTurtleData() " << std::endl; @@ -1304,7 +1331,7 @@ bool p3GRouter::locked_sendTransactionData(const RsPeerId& pid,const RsGRouterTr void p3GRouter::autoWash() { bool items_deleted = false ; - rstime_t now = time(NULL) ; + rstime_t now = time(nullptr); std::map > failed_msgs ; @@ -1395,9 +1422,22 @@ void p3GRouter::autoWash() it->second.clear() ; _incoming_data_pipes.erase(it) ; it = ittmp ; - } - else - ++it ; + } + else + ++it ; + + /* Cleanup timed out items in mMissingKeyQueue */ + mMissingKeyQueueMtx.lock(); + while( mMissingKeyQueue.begin() != mMissingKeyQueue.end() && + mMissingKeyQueue.front().second <= now ) + { + RsWarn() << __PRETTY_FUNCTION__ << " Deleting timed out item from " + << "unknown RsGxsId: " + << mMissingKeyQueue.front().first->signature.keyId + << std::endl; + mMissingKeyQueue.pop_front(); + } + mMissingKeyQueueMtx.unlock(); } // Look into pending items. @@ -1492,27 +1532,31 @@ void p3GRouter::handleIncoming() RsGRouterAbstractMsgItem *item = _incoming_items.front() ; _incoming_items.pop_front() ; - RsGRouterGenericDataItem *generic_data_item ; - RsGRouterSignedReceiptItem *receipt_item ; - - if(NULL != (generic_data_item = dynamic_cast(item))) - handleIncomingDataItem(generic_data_item) ; - else if(NULL != (receipt_item = dynamic_cast(item))) - handleIncomingReceiptItem(receipt_item) ; - else - std::cerr << "Item has unknown type (not data nor signed receipt). Dropping!" << std::endl; - + handleIncomingItem(item); delete item ; } } -void p3GRouter::handleIncomingReceiptItem(RsGRouterSignedReceiptItem *receipt_item) +void p3GRouter::handleIncomingItem(const RsGRouterAbstractMsgItem *item) +{ + const RsGRouterGenericDataItem *generic_data_item ; + const RsGRouterSignedReceiptItem *receipt_item ; + + if(NULL != (generic_data_item = dynamic_cast(item))) + handleIncomingDataItem(generic_data_item) ; + else if(NULL != (receipt_item = dynamic_cast(item))) + handleIncomingReceiptItem(receipt_item) ; + else + std::cerr << "Item has unknown type (not data nor signed receipt). Dropping!" << std::endl; +} + +void p3GRouter::handleIncomingReceiptItem(const RsGRouterSignedReceiptItem *receipt_item) { bool changed = false ; #ifdef GROUTER_DEBUG std::cerr << "Handling incoming signed receipt item." << std::endl; std::cerr << "Item content:" << std::endl; - receipt_item->print(std::cerr,2) ; + const_cast(receipt_item)->print(std::cerr,2) ; #endif RsGxsId signer_id ; @@ -1613,17 +1657,17 @@ void p3GRouter::handleIncomingReceiptItem(RsGRouterSignedReceiptItem *receipt_it IndicateConfigChanged() ; } -Sha1CheckSum p3GRouter::computeDataItemHash(RsGRouterGenericDataItem *data_item) +Sha1CheckSum p3GRouter::computeDataItemHash(const RsGRouterGenericDataItem *data_item) { RsGRouterSerialiser signature_serializer(RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE | RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER); - uint32_t signed_data_size = signature_serializer.size(data_item); + uint32_t signed_data_size = signature_serializer.size(const_cast(data_item)); uint32_t total_size = signed_data_size + data_item->signature.TlvSize() ; RsTemporaryMemory mem(total_size) ; uint32_t offset = 0 ; uint32_t tmp_size = total_size ; - signature_serializer.serialise(data_item,mem,&tmp_size) ; + signature_serializer.serialise(const_cast(data_item),mem,&tmp_size) ; if(tmp_size != signed_data_size) std::cerr << "(EE) Some error occured in p3GRouter::computeDataItemHash(). Mismatched offset/data size" << std::endl; @@ -1637,12 +1681,12 @@ Sha1CheckSum p3GRouter::computeDataItemHash(RsGRouterGenericDataItem *data_item) return RsDirUtil::sha1sum(mem,total_size) ; } -void p3GRouter::handleIncomingDataItem(RsGRouterGenericDataItem *data_item) +void p3GRouter::handleIncomingDataItem(const RsGRouterGenericDataItem *data_item) { #ifdef GROUTER_DEBUG std::cerr << "Handling incoming data item. " << std::endl; std::cerr << "Item content:" << std::endl; - data_item->print(std::cerr,2) ; + const_cast(data_item)->print(std::cerr,2) ; #endif // we find 3 things: @@ -1654,15 +1698,15 @@ void p3GRouter::handleIncomingDataItem(RsGRouterGenericDataItem *data_item) // Send a receipt? if A && B // Notify client? if A && !C // - GRouterClientService *client = NULL ; + GRouterClientService *clientService = NULL ; GRouterServiceId service_id = data_item->service_id ; RsGRouterSignedReceiptItem *receipt_item = NULL ; Sha1CheckSum item_hash = computeDataItemHash(data_item) ; bool item_is_already_known = false ; - bool item_is_for_us = false ; bool cache_has_changed = false ; + bool item_is_for_us = _owned_key_ids.find( makeTunnelHash(data_item->destination_key,service_id) ) != _owned_key_ids.end() ; // A - Find client and service ID from destination key. #ifdef GROUTER_DEBUG @@ -1671,23 +1715,12 @@ void p3GRouter::handleIncomingDataItem(RsGRouterGenericDataItem *data_item) { RS_STACK_MUTEX(grMtx) ; - std::map::const_iterator its = _registered_services.find(service_id) ; - - if(its == _registered_services.end()) - { - std::cerr << " ERROR: client id " << service_id << " not registered. Consistency error." << std::endl; - return ; - } - client = its->second ; - // also check wether this item is for us or not - item_is_for_us = _owned_key_ids.find( makeTunnelHash(data_item->destination_key,service_id) ) != _owned_key_ids.end() ; - #ifdef GROUTER_DEBUG std::cerr << " item is " << (item_is_for_us?"":"not") << " for us." << std::endl; #endif - std::map::iterator it = _pending_messages.find(data_item->routing_id) ; + auto it = _pending_messages.find(data_item->routing_id) ; if(it != _pending_messages.end()) { @@ -1709,26 +1742,56 @@ void p3GRouter::handleIncomingDataItem(RsGRouterGenericDataItem *data_item) std::cerr << " item is new." << std::endl; #endif } + + if(!item_is_already_known) + { + uint32_t error_status ; + + if(!verifySignedDataItem(data_item,RsIdentityUsage::GLOBAL_ROUTER_SIGNATURE_CHECK,error_status)) // we should get proper flags out of this + { + switch(error_status) + { + case RsGixs::RS_GIXS_ERROR_KEY_NOT_AVAILABLE: + { + RS_STACK_MUTEX(mMissingKeyQueueMtx); + + rstime_t timeout = time(nullptr) + mMissingKeyQueueEntryTimeout; + RsGxsId authorId = data_item->signature.keyId; + mMissingKeyQueue.push_back( std::make_pair(std::unique_ptr(data_item->duplicate()), timeout) ); + + /* Do not request the missing key here to the peer which forwarded the item as verifySignedDataItem(...) does it already */ + + RsInfo() << __PRETTY_FUNCTION__ << " Received a message from unknown RsGxsId: " << authorId <<". Cannot verify signature yet, storing in mMissingKeyQueue for later processing. Timeout: " << timeout << std::endl; + return; + } + default: + RsWarn() << __PRETTY_FUNCTION__ << " item signature verification FAILED with: " << error_status << ", Dropping!" << std::endl; + return; + } + } +#ifdef GROUTER_DEBUG + else + std::cerr << " verifying item signature: CHECKED!" ; +#endif + } + // At this point, if item is already known, it is guarrantied to be identical to the stored item. // If the item is for us, and not already known, check the signature and hash, and generate a signed receipt if(item_is_for_us && !item_is_already_known) { + // Check that we actually have a registered service ready to accept this item. If not, drop it. + + { + RS_STACK_MUTEX(grMtx) ; + auto its = _registered_services.find(service_id) ; + + if(its != _registered_services.end()) + clientService = its->second ; + } + #ifdef GROUTER_DEBUG std::cerr << " step B: item is for us and is new, so make sure it's authentic and create a receipt" << std::endl; -#endif - uint32_t error_status ; - - if(!verifySignedDataItem(data_item,RsIdentityUsage::GLOBAL_ROUTER_SIGNATURE_CHECK,error_status)) // we should get proper flags out of this - { - std::cerr << " verifying item signature: FAILED! Droping that item" ; - std::cerr << " You probably received a message from a person you don't have key." << std::endl; - std::cerr << " Signature key ID: " << data_item->signature.keyId << std::endl; - return ; - } -#ifdef GROUTER_DEBUG - else - std::cerr << " verifying item signature: CHECKED!" ; #endif // No we need to send a signed receipt to the sender. @@ -1737,7 +1800,13 @@ void p3GRouter::handleIncomingDataItem(RsGRouterGenericDataItem *data_item) receipt_item->service_id = data_item->service_id ; receipt_item->routing_id = data_item->routing_id ; receipt_item->destination_key = data_item->signature.keyId ; - receipt_item->flags = 0 ; + receipt_item->flags = RsGRouterItemFlags::NONE ; + + if(!clientService) + { + receipt_item->flags = RsGRouterItemFlags::SERVICE_UNKNOWN; + RsWarn() << __PRETTY_FUNCTION__ << " got a message from: " << data_item->signature.keyId << " for an unkown service: " << data_item->service_id << " is your RetroShare version updated?" << std::endl; + } #ifdef GROUTER_DEBUG std::cerr << " preparing signed receipt." << std::endl; @@ -1813,7 +1882,7 @@ void p3GRouter::handleIncomingDataItem(RsGRouterGenericDataItem *data_item) // if the item is for us and is not already known, notify the client. - if(item_is_for_us && !item_is_already_known) + if(clientService && !item_is_already_known) { // compute the hash before decryption. @@ -1835,9 +1904,9 @@ void p3GRouter::handleIncomingDataItem(RsGRouterGenericDataItem *data_item) std::cerr << " notyfying client." << std::endl; #endif - if(client->acceptDataFromPeer(decrypted_item->signature.keyId)) + if(clientService->acceptDataFromPeer(decrypted_item->signature.keyId)) { - client->receiveGRouterData(decrypted_item->destination_key,decrypted_item->signature.keyId,service_id,decrypted_item->data_bytes,decrypted_item->data_size); + clientService->receiveGRouterData(decrypted_item->destination_key,decrypted_item->signature.keyId,service_id,decrypted_item->data_bytes,decrypted_item->data_size); decrypted_item->data_bytes = NULL ; decrypted_item->data_size = 0 ; @@ -1871,7 +1940,7 @@ bool p3GRouter::locked_getLocallyRegisteredClientFromServiceId(const GRouterServ return true ; } -void p3GRouter::addRoutingClue(const GRouterKeyId& id,const RsPeerId& peer_id) +void p3GRouter::addRoutingClue(const RsGxsId& id, const RsPeerId& peer_id) { RS_STACK_MUTEX(grMtx) ; #ifdef GROUTER_DEBUG @@ -1890,7 +1959,7 @@ bool p3GRouter::registerClientService(const GRouterServiceId& id,GRouterClientSe bool p3GRouter::encryptDataItem(RsGRouterGenericDataItem *item,const RsGxsId& destination_key) { - assert(!(item->flags & RS_GROUTER_DATA_FLAGS_ENCRYPTED)) ; + assert(!(item->flags & RsGRouterItemFlags::ENCRYPTED)); #ifdef GROUTER_DEBUG std::cerr << " Encrypting data for key " << destination_key << std::endl; @@ -1915,7 +1984,7 @@ bool p3GRouter::encryptDataItem(RsGRouterGenericDataItem *item,const RsGxsId& de free(item->data_bytes) ; item->data_bytes = encrypted_data ; item->data_size = encrypted_size ; - item->flags |= RS_GROUTER_DATA_FLAGS_ENCRYPTED ; + item->flags |= RsGRouterItemFlags::ENCRYPTED; #ifdef GROUTER_DEBUG std::cerr << " Encrypted size = " << encrypted_size << std::endl; @@ -1926,7 +1995,7 @@ return true ; } bool p3GRouter::decryptDataItem(RsGRouterGenericDataItem *item) { - assert(item->flags & RS_GROUTER_DATA_FLAGS_ENCRYPTED) ; + assert(!!(item->flags & RsGRouterItemFlags::ENCRYPTED)); #ifdef GROUTER_DEBUG std::cerr << " decrypting data for key " << item->destination_key << std::endl; @@ -1952,7 +2021,7 @@ bool p3GRouter::decryptDataItem(RsGRouterGenericDataItem *item) free(item->data_bytes) ; item->data_bytes = decrypted_data ; item->data_size = decrypted_size ; - item->flags &= ~RS_GROUTER_DATA_FLAGS_ENCRYPTED ; + item->flags &= ~RsGRouterItemFlags::ENCRYPTED; return true ; } @@ -2010,56 +2079,62 @@ bool p3GRouter::signDataItem(RsGRouterAbstractMsgItem *item,const RsGxsId& signi return false ; } } -bool p3GRouter::verifySignedDataItem(RsGRouterAbstractMsgItem *item,const RsIdentityUsage::UsageCode& info,uint32_t& error_status) +bool p3GRouter::verifySignedDataItem(const RsGRouterAbstractMsgItem *item,const RsIdentityUsage::UsageCode& info,uint32_t& error_status) { try { if( rsReputations->overallReputationLevel(item->signature.keyId) == RsReputationLevel::LOCALLY_NEGATIVE ) - { - std::cerr << "(WW) received global router message from banned identity " << item->signature.keyId << ". Rejecting the message." << std::endl; - return false ; - } - RsGRouterSerialiser signature_serializer(RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE | RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER); + { + RsWarn() << __PRETTY_FUNCTION__ << " received global router " + << "message from banned identity " << item->signature.keyId + << ". Rejecting the message." << std::endl; + return false; + } - uint32_t data_size = signature_serializer.size(item) ; - RsTemporaryMemory data(data_size) ; + RsGRouterSerialiser signature_serializer( + RsGenericSerializer::SERIALIZATION_FLAG_SIGNATURE | + RsGenericSerializer::SERIALIZATION_FLAG_SKIP_HEADER ); + + uint32_t data_size = signature_serializer.size(const_cast(item)); // the const cast shouldn't be necessary if size() took a const. + RsTemporaryMemory data(data_size); if(data == NULL) throw std::runtime_error("Cannot allocate data.") ; - if(!signature_serializer.serialise(item,data,&data_size)) - throw std::runtime_error("Cannot serialise signed data.") ; + if(!signature_serializer.serialise(const_cast(item),data,&data_size)) + throw std::runtime_error("Cannot serialise signed data."); - RsIdentityUsage use(RS_SERVICE_TYPE_GROUTER,info) ; + RsIdentityUsage use(RS_SERVICE_TYPE_GROUTER,info); - if(!mGixs->validateData(data,data_size,item->signature,true,use, error_status)) - { - switch(error_status) - { - case RsGixs::RS_GIXS_ERROR_KEY_NOT_AVAILABLE: - { - std::list peer_ids ; - peer_ids.push_back(item->PeerId()) ; - - std::cerr << "(EE) Key for GXS Id " << item->signature.keyId << " is not available. Cannot verify. Asking key to peer " << item->PeerId() << std::endl; - - mGixs->requestKey(item->signature.keyId,peer_ids,use) ; // request the key around - } - break ; - case RsGixs::RS_GIXS_ERROR_SIGNATURE_MISMATCH: std::cerr << "(EE) Signature mismatch. Spoofing/Corrupted/MITM?." << std::endl; - break ; - default: std::cerr << "(EE) Signature verification failed on GRouter message. Unknown error status: " << error_status << std::endl; - break ; - } - return false; - } + if(!mGixs->validateData( data, data_size, item->signature, true, use, error_status )) + { + switch(error_status) + { + case RsGixs::RS_GIXS_ERROR_KEY_NOT_AVAILABLE: + { + std::list peer_ids; + peer_ids.push_back(item->PeerId()); - return true ; - } - catch(std::exception& e) - { - std::cerr << " signature verification failed. Error: " << e.what() << std::endl; + RsWarn() << __PRETTY_FUNCTION__ << " Key for GXS Id " << item->signature.keyId << " is not available. Cannot verify. Asking key to peer " << item->PeerId() << std::endl; + mGixs->requestKey(item->signature.keyId,peer_ids,use); + } + break; + case RsGixs::RS_GIXS_ERROR_SIGNATURE_MISMATCH: + RsWarn() << __PRETTY_FUNCTION__ << " Signature mismatch. " << "Spoofing/Corrupted/MITM?." << std::endl; + break; + default: + RsErr() << __PRETTY_FUNCTION__ << " Signature verification failed on GRouter message. Unknown error status: " << error_status << std::endl; + break; + } + return false; + } + + return true; + } + catch(std::exception& e) + { + RsErr() << __PRETTY_FUNCTION__ << " Failed. Error: " << e.what() << std::endl; return false ; } } @@ -2093,7 +2168,7 @@ bool p3GRouter::cancel(GRouterMsgPropagationId mid) return true ; } -bool p3GRouter::sendData(const RsGxsId& destination,const GRouterServiceId& client_id,const uint8_t *data, uint32_t data_size,const RsGxsId& signing_id, GRouterMsgPropagationId &propagation_id) +bool p3GRouter::sendData( const RsGxsId& destination, const GRouterServiceId& client_id, const uint8_t* data, uint32_t data_size, const RsGxsId& signing_id, GRouterMsgPropagationId&propagation_id ) { // std::cerr << "GRouter currently disabled." << std::endl; // return false; @@ -2127,7 +2202,7 @@ bool p3GRouter::sendData(const RsGxsId& destination,const GRouterServiceId& clie data_item->duplication_factor = GROUTER_MAX_DUPLICATION_FACTOR ; data_item->service_id = client_id ; data_item->destination_key = destination ; - data_item->flags = 0 ; // this is unused for now. + data_item->flags = RsGRouterItemFlags::NONE ; // this is unused for now. // First, encrypt. @@ -2204,8 +2279,12 @@ return true ; Sha1CheckSum p3GRouter::makeTunnelHash(const RsGxsId& destination,const GRouterServiceId& client) { - assert( destination.SIZE_IN_BYTES == 16) ; - assert(Sha1CheckSum::SIZE_IN_BYTES == 20) ; + static_assert( RsGxsId::SIZE_IN_BYTES == 16, + "This function breaks if RsGxsId size changes" ); + static_assert( Sha1CheckSum::SIZE_IN_BYTES == 20, + "This function breaks if Sha1CheckSum size changes" ); + static_assert( sizeof(client) == 4, + "This function breaks if client service id size changes" ); uint8_t bytes[20] ; memcpy(bytes,destination.toByteArray(),16) ; diff --git a/libretroshare/src/grouter/p3grouter.h b/libretroshare/src/grouter/p3grouter.h index fca320843..236c8b7a5 100644 --- a/libretroshare/src/grouter/p3grouter.h +++ b/libretroshare/src/grouter/p3grouter.h @@ -22,8 +22,8 @@ #pragma once #include -#include #include +#include #include "retroshare/rsgrouter.h" #include "retroshare/rstypes.h" @@ -33,15 +33,13 @@ #include "turtle/turtleclientservice.h" #include "services/p3service.h" #include "pqi/p3cfgmgr.h" - +#include "util/rsdebug.h" #include "groutertypes.h" #include "groutermatrix.h" #include "grouteritems.h" // To be put in pqi/p3cfgmgr.h -// static const uint32_t CONFIG_TYPE_GROUTER = 0x0016 ; -static const uint32_t RS_GROUTER_DATA_FLAGS_ENCRYPTED = 0x0001 ; class p3LinkMgr ; class p3turtle ; @@ -126,7 +124,8 @@ public: // Routing clue collection methods // //===================================================// - virtual void addRoutingClue(const GRouterKeyId& id,const RsPeerId& peer_id) ; + virtual void addRoutingClue( + const RsGxsId& id, const RsPeerId& peer_id) override; //===================================================// // Client/server request services // @@ -220,7 +219,7 @@ private: void handleLowLevelTransactionChunkItem(RsGRouterTransactionChunkItem *chunk_item); void handleLowLevelTransactionAckItem(RsGRouterTransactionAcknItem*) ; - static Sha1CheckSum computeDataItemHash(RsGRouterGenericDataItem *data_item); + static Sha1CheckSum computeDataItemHash(const RsGRouterGenericDataItem *data_item); std::ostream& grouter_debug() const { @@ -238,8 +237,9 @@ private: void handleIncoming() ; - void handleIncomingReceiptItem(RsGRouterSignedReceiptItem *receipt_item) ; - void handleIncomingDataItem(RsGRouterGenericDataItem *data_item) ; + void handleIncomingItem(const RsGRouterAbstractMsgItem *item); + void handleIncomingReceiptItem(const RsGRouterSignedReceiptItem *receipt_item) ; + void handleIncomingDataItem(const RsGRouterGenericDataItem *data_item) ; bool locked_getLocallyRegisteredClientFromServiceId(const GRouterServiceId& service_id,GRouterClientService *& client); @@ -252,7 +252,7 @@ private: // signs an item with the given key. bool signDataItem(RsGRouterAbstractMsgItem *item,const RsGxsId& id) ; - bool verifySignedDataItem(RsGRouterAbstractMsgItem *item, const RsIdentityUsage::UsageCode &info, uint32_t &error_status) ; + bool verifySignedDataItem(const RsGRouterAbstractMsgItem *item, const RsIdentityUsage::UsageCode &info, uint32_t &error_status) ; bool encryptDataItem(RsGRouterGenericDataItem *item,const RsGxsId& destination_key) ; bool decryptDataItem(RsGRouterGenericDataItem *item) ; @@ -352,4 +352,23 @@ private: rstime_t _last_config_changed ; uint64_t _random_salt ; + + /** Temporarly store items that could not have been verified yet due to + * missing author key, attempt to handle them once in a while. + * The items are discarded if after mMissingKeyQueueEntryTimeout the key + * hasn't been received yet, and are not saved on RetroShare stopping. */ + std::list< std::pair< + std::unique_ptr, rstime_t > > mMissingKeyQueue; + RsMutex mMissingKeyQueueMtx; /// protect mMissingKeyQueue + + /// @see mMissingKeyQueue + static constexpr rstime_t mMissingKeyQueueEntryTimeout = 600; + + /// @see mMissingKeyQueue + static constexpr rstime_t mMissingKeyQueueCheckEvery = 30; + + /// @see mMissingKeyQueue + rstime_t mMissingKeyQueueCheckLastCheck = 0; + + RS_SET_CONTEXT_DEBUG_LEVEL(2) }; diff --git a/libretroshare/src/retroshare/rsgrouter.h b/libretroshare/src/retroshare/rsgrouter.h index e29c5df50..84349f75e 100644 --- a/libretroshare/src/retroshare/rsgrouter.h +++ b/libretroshare/src/retroshare/rsgrouter.h @@ -19,12 +19,13 @@ * along with this program. If not, see . * * * *******************************************************************************/ - #pragma once #include "util/rsdir.h" +#include "util/rsdeprecate.h" #include "retroshare/rsids.h" #include "retroshare/rsgxsifacetypes.h" +#include "rsitems/rsserviceids.h" typedef RsGxsId GRouterKeyId ; // we use SSLIds, so that it's easier in the GUI to mix up peer ids with grouter ids. typedef uint32_t GRouterServiceId ; diff --git a/libretroshare/src/retroshare/rsidentity.h b/libretroshare/src/retroshare/rsidentity.h index ddda6e65a..67844cde0 100644 --- a/libretroshare/src/retroshare/rsidentity.h +++ b/libretroshare/src/retroshare/rsidentity.h @@ -511,9 +511,13 @@ struct RsIdentity : RsGxsIfaceHelper * @brief request details of a not yet known identity to the network * @jsonapi{development} * @param[in] id id of the identity to request + * @param[in] peers optional list of the peers to ask for the key, if empty + * all online peers are asked. * @return false on error, true otherwise */ - virtual bool requestIdentity(const RsGxsId& id) = 0; + virtual bool requestIdentity( + const RsGxsId& id, + const std::vector& peers = std::vector() ) = 0; /// default base URL used for indentity links @see exportIdentityLink static const std::string DEFAULT_IDENTITY_BASE_URL; diff --git a/libretroshare/src/services/p3idservice.cc b/libretroshare/src/services/p3idservice.cc index 0821f3286..85de0abcb 100644 --- a/libretroshare/src/services/p3idservice.cc +++ b/libretroshare/src/services/p3idservice.cc @@ -1163,92 +1163,89 @@ bool p3IdService::havePrivateKey(const RsGxsId &id) if(! isOwnId(id)) return false ; - RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ - return mKeyCache.is_cached(id) ; + RS_STACK_MUTEX(mIdMtx); + return mKeyCache.is_cached(id); } static void mergeIds(std::map >& idmap,const RsGxsId& id,const std::list& peers) { - // merge the two lists (I use a std::set to make it more efficient) -#ifdef DEBUG_IDS - std::cerr << "p3IdService::requestKey(): merging list with existing pending request." << std::endl; -#endif + /* merge the two lists, use std::set to avoid duplicates efficiently */ - std::list& old_peers(idmap[id]) ; // create if necessary - std::set new_peers ; + std::set new_peers(std::begin(peers), std::end(peers)); - for(std::list::const_iterator it(peers.begin());it!=peers.end();++it) - new_peers.insert(*it) ; - - for(std::list::iterator it(old_peers.begin());it!=old_peers.end();++it) - new_peers.insert(*it) ; - - old_peers.clear(); - - for(std::set::iterator it(new_peers.begin());it!=new_peers.end();++it) - old_peers.push_back(*it) ; + std::list& stored_peers(idmap[id]); + std::copy( std::begin(stored_peers), std::end(stored_peers), + std::inserter(new_peers, std::begin(new_peers)) ); + stored_peers.clear(); + std::copy( std::begin(new_peers), std::end(new_peers), + std::inserter(stored_peers, std::begin(stored_peers)) ); } -bool p3IdService::requestIdentity(const RsGxsId& id) +bool p3IdService::requestIdentity( + const RsGxsId& id, const std::vector& peers ) { + std::list askPeersList(peers.begin(), peers.end()); + + // Empty list passed? Ask to all online peers. + if(askPeersList.empty()) rsPeers->getOnlineList(askPeersList); + + if(askPeersList.empty()) // Still empty? Fail! + { + RsErr() << __PRETTY_FUNCTION__ << " failure retrieving peers list" + << std::endl; + return false; + } + RsIdentityUsage usageInfo( RsServiceType::GXSID, RsIdentityUsage::IDENTITY_DATA_UPDATE ); - std::list onlinePeers; - return rsPeers && rsPeers->getOnlineList(onlinePeers) - && requestKey(id, onlinePeers, usageInfo); + return requestKey(id, askPeersList, usageInfo); } bool p3IdService::requestKey(const RsGxsId &id, const std::list& peers,const RsIdentityUsage& use_info) { - if(id.isNull()) - { - std::cerr << "(EE) nul ID requested to p3IdService. This should not happen. Callstack:" << std::endl; - print_stacktrace(); - return false ; - } + Dbg3() << __PRETTY_FUNCTION__ << " id: " << id << std::endl; - if (haveKey(id)) - return true; - else - { - // Normally we should call getIdDetails(), but since the key is not known, we need to digg a possibly old information - // from the reputation system, which keeps its own list of banned keys. Of course, the owner ID is not known at this point. + if(id.isNull()) + { + RsErr() << __PRETTY_FUNCTION__ << " cannot request null id" + << std::endl; + return false; + } -#ifdef DEBUG_IDS - std::cerr << "p3IdService::requesting key " << id <getReputationInfo(id,RsPgpId(),info) ; + if(isKnownId(id)) return true; - if( info.mOverallReputationLevel == RsReputationLevel::LOCALLY_NEGATIVE ) - { - std::cerr << "(II) not requesting Key " << id << " because it has been banned." << std::endl; + /* Normally we should call getIdDetails(), but since the key is not known, + * we need to dig a possibly old information from the reputation system, + * which keeps its own list of banned keys. + * Of course, the owner ID is not known at this point.c*/ - { - RS_STACK_MUTEX(mIdMtx); /********** STACK LOCKED MTX ******/ - mIdsNotPresent.erase(id) ; - } - return true; - } + RsReputationInfo info; + rsReputations->getReputationInfo(id, RsPgpId(), info); - RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + if( info.mOverallReputationLevel == RsReputationLevel::LOCALLY_NEGATIVE ) + { + RsInfo() << __PRETTY_FUNCTION__ << " not requesting Key " << id + << " because it has been banned." << std::endl; - std::map >::iterator rit = mIdsNotPresent.find(id) ; + RS_STACK_MUTEX(mIdMtx); + mIdsNotPresent.erase(id); - if(rit != mIdsNotPresent.end()) - { - if(!peers.empty()) - mergeIds(mIdsNotPresent,id,peers) ; + return false; + } - return true ; - } - } - { - RS_STACK_MUTEX(mIdMtx); /********** STACK LOCKED MTX ******/ - mKeysTS[id].usage_map[use_info] = time(NULL) ; - } + { + RS_STACK_MUTEX(mIdMtx); + mergeIds(mIdsNotPresent, id, peers); + mKeysTS[id].usage_map[use_info] = time(nullptr); + } return cache_request_load(id, peers); } @@ -2772,34 +2769,30 @@ bool p3IdService::cache_store(const RsGxsIdGroupItem *item) bool p3IdService::cache_request_load(const RsGxsId &id, const std::list &peers) { -#ifdef DEBUG_IDS - std::cerr << "p3IdService::cache_request_load(" << id << ")" << std::endl; -#endif // DEBUG_IDS + Dbg4() << __PRETTY_FUNCTION__ << " id: " << id << std::endl; - { - RsStackMutex stack(mIdMtx); /********** STACK LOCKED MTX ******/ + { + RS_STACK_MUTEX(mIdMtx); + // merge, even if peers is empty + mergeIds(mCacheLoad_ToCache, id, peers); + } - mergeIds(mCacheLoad_ToCache,id,peers) ; // merge, even if peers is empty - } + if(RsTickEvent::event_count(GXSID_EVENT_CACHELOAD) > 0) + { + Dbg3() << __PRETTY_FUNCTION__ << " cache reload already scheduled " + << "skipping" << std::endl; + return true; + } - if (RsTickEvent::event_count(GXSID_EVENT_CACHELOAD) > 0) - { - /* its already scheduled */ - return true; - } + int32_t age = 0; + if( RsTickEvent::prev_event_ago(GXSID_EVENT_CACHELOAD, age) && age < MIN_CYCLE_GAP ) + { + RsTickEvent::schedule_in(GXSID_EVENT_CACHELOAD, MIN_CYCLE_GAP - age); + return true; + } - int32_t age = 0; - if (RsTickEvent::prev_event_ago(GXSID_EVENT_CACHELOAD, age)) - { - if (age < MIN_CYCLE_GAP) - { - RsTickEvent::schedule_in(GXSID_EVENT_CACHELOAD, MIN_CYCLE_GAP - age); - return true; - } - } - - RsTickEvent::schedule_now(GXSID_EVENT_CACHELOAD); - return true; + RsTickEvent::schedule_now(GXSID_EVENT_CACHELOAD); + return true; } @@ -2923,71 +2916,88 @@ bool p3IdService::cache_load_for_token(uint32_t token) void p3IdService::requestIdsFromNet() { - RsStackMutex stack(mIdMtx); + RS_STACK_MUTEX(mIdMtx); - if(!mNes) - { - std::cerr << "(WW) cannot request missing GXS IDs because network service is not present." << std::endl; - return ; - } + if(!mNes) + { + RsErr() << __PRETTY_FUNCTION__ << " Cannot request missing GXS IDs " + << "because network service is not present." << std::endl; + return; + } std::map >::iterator cit; std::map > requests; - // Transform to appropriate structure ( > map) to make request to nes per peer ID - // Only delete entries in mIdsNotPresent that can actually be performed. + /* Transform to appropriate structure ( > map) + * to make request to nes per peer ID + * Only delete entries in mIdsNotPresent that can actually be performed, or + * that have empty peer list */ - for(cit = mIdsNotPresent.begin(); cit != mIdsNotPresent.end();) - { -#ifdef DEBUG_IDS - std::cerr << "p3IdService::requestIdsFromNet() Id not found, deferring for net request: " << cit->first << std::endl; -#endif + for(cit = mIdsNotPresent.begin(); cit != mIdsNotPresent.end();) + { + Dbg2() << __PRETTY_FUNCTION__ << " Processing missing key RsGxsId: " + << cit->first << std::endl; - const std::list& peers = cit->second; - std::list::const_iterator cit2; + const RsGxsId& gxsId = cit->first; + const std::list& peers = cit->second; + std::list::const_iterator cit2; bool request_can_proceed = false ; - for(cit2 = peers.begin(); cit2 != peers.end(); ++cit2) - if(rsPeers->isOnline(*cit2) || mNes->isDistantPeer(*cit2)) // make sure that the peer in online, so that we know that the request has some chance to succeed. - { - requests[*cit2].push_back(cit->first); - request_can_proceed = true ; -#ifdef DEBUG_IDS - std::cerr << " will ask ID " << cit->first << " to peer ID " << *cit2 << std::endl; -#endif - } + for(cit2 = peers.begin(); cit2 != peers.end(); ++cit2) + { + const RsPeerId& peer = *cit2; - if(request_can_proceed || peers.empty()) - { - std::map >::iterator tmp(cit); - ++tmp ; - mIdsNotPresent.erase(cit) ; - cit = tmp ; - } - else - { -#ifdef DEBUG_IDS - std::cerr << "(EE) no online peers among supply list in ID request for groupId " << cit->first << ". Keeping it until peers show up."<< std::endl; -#endif - ++cit ; - } - } + if(rsPeers->isOnline(peer) || mNes->isDistantPeer(peer)) + { + /* make sure that the peer in online, so that we know that the + * request has some chance to succeed.*/ + requests[peer].push_back(cit->first); + request_can_proceed = true ; - for(std::map >::const_iterator cit2(requests.begin()); cit2 != requests.end(); ++cit2) - { - std::list::const_iterator gxs_id_it = cit2->second.begin(); - std::list grpIds; - for(; gxs_id_it != cit2->second.end(); ++gxs_id_it) - { -#ifdef DEBUG_IDS - std::cerr << " asking ID " << *gxs_id_it << " to peer ID " << cit2->first << std::endl; -#endif - grpIds.push_back(RsGxsGroupId(*gxs_id_it)); - } + Dbg2() << __PRETTY_FUNCTION__ << " Moving missing key RsGxsId:" + << gxsId << " to peer: " << peer << " requests queue" + << std::endl; + } + } - mNes->requestGrp(grpIds, cit2->first); - } + const bool noPeersFound = peers.empty(); + if(noPeersFound) + RsWarn() << __PRETTY_FUNCTION__ << " No peers supplied to request " + << "RsGxsId: " << gxsId << " dropping." << std::endl; + + if(request_can_proceed || noPeersFound) + { + std::map >::iterator tmp(cit); + ++tmp; + mIdsNotPresent.erase(cit); + cit = tmp; + } + else + { + RsInfo() << __PRETTY_FUNCTION__ << " no online peers among supplied" + << " list in request for RsGxsId: " << gxsId + << ". Keeping it until peers show up."<< std::endl; + ++cit; + } + } + + for( std::map >::const_iterator cit2( + requests.begin() ); cit2 != requests.end(); ++cit2 ) + { + const RsPeerId& peer = cit2->first; + std::list grpIds; + for( std::list::const_iterator gxs_id_it = cit2->second.begin(); + gxs_id_it != cit2->second.end(); ++gxs_id_it ) + { + Dbg2() << __PRETTY_FUNCTION__ << " passing RsGxsId: " << *gxs_id_it + << " request for peer: " << peer + << " to RsNetworkExchangeService " << std::endl; + grpIds.push_back(RsGxsGroupId(*gxs_id_it)); + } + + mNes->requestGrp(grpIds, peer); + } } bool p3IdService::cache_update_if_cached(const RsGxsId &id, std::string serviceString) @@ -4698,14 +4708,12 @@ void p3IdService::handle_event(uint32_t event_type, const std::string &/*elabel* break; case GXSID_EVENT_REQUEST_IDS: requestIdsFromNet(); - break; - - - default: - /* error */ - std::cerr << "p3IdService::handle_event() Unknown Event Type: " << event_type; - std::cerr << std::endl; - break; + break; + default: + RsErr() << __PRETTY_FUNCTION__ << " Unknown Event Type: " + << event_type << std::endl; + print_stacktrace(); + break; } } diff --git a/libretroshare/src/services/p3idservice.h b/libretroshare/src/services/p3idservice.h index 26121cd8a..3c6b6677c 100644 --- a/libretroshare/src/services/p3idservice.h +++ b/libretroshare/src/services/p3idservice.h @@ -381,7 +381,10 @@ public: RsGxsId* id = nullptr); /// @see RsIdentity - bool requestIdentity(const RsGxsId& id) override; + bool requestIdentity( + const RsGxsId& id, + const std::vector& peers = std::vector() + ) override; /**************** RsGixsReputation Implementation ****************/ @@ -494,7 +497,7 @@ private: /* MUTEX PROTECTED DATA (mIdMtx - maybe should use a 2nd?) */ - std::map mPgpFingerprintMap; + std::map mPgpFingerprintMap; std::list mGroupsToProcess; /************************************************************************ @@ -633,5 +636,5 @@ private: bool mAutoAddFriendsIdentitiesAsContacts; uint32_t mMaxKeepKeysBanned; - RS_SET_CONTEXT_DEBUG_LEVEL(1) + RS_SET_CONTEXT_DEBUG_LEVEL(2) };