mirror of
https://github.com/RetroShare/RetroShare.git
synced 2025-07-30 09:48:59 -04:00
Expanded DHT history analysis, and ran some more tests on DHT behaviour.
- Tweaked query parameters (quicker discard, longer requery period). The analysis suggests that in general the DHT is running okay with some minor issues. - BitDHT peers respond to ~50% of FIND_NODE queries. This means we drop them, and so some ONLINE friends, become UNREACHABLE friends. This appears to be caused by too many FIND_NODE queries reaching BitDHT, and the peer starts dropping them. This appears to come from specific peers, which are not RS nodes. Proposed solution is to Track and Filter Over-Querying Peers. (Only one query per peer every 30 seconds). - Should be more lenient on the exact query matches. Allow some dropped messages. - Queries appear to run longer than necessary. A better query completion criteria needs to be established. git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@6614 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
drbob
parent
1554e17f48
commit
0608f67d71
6 changed files with 488 additions and 98 deletions
|
|
@ -3,21 +3,35 @@
|
|||
#include "bitdht/bdhistory.h"
|
||||
#include "bitdht/bdstddht.h"
|
||||
#include "bitdht/bdmsgs.h"
|
||||
#include <set>
|
||||
|
||||
#define MIN_RESEND_PERIOD 60
|
||||
|
||||
void bdMsgHistoryList::addMsg(time_t ts, uint32_t msgType, bool incoming)
|
||||
bdMsgHistoryList::bdMsgHistoryList()
|
||||
:mPeerVersion("Unknown")
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
|
||||
void bdMsgHistoryList::addMsg(time_t ts, uint32_t msgType, bool incoming, const bdNodeId *aboutId)
|
||||
{
|
||||
// std::cerr << "bdMsgHistoryList::addMsg()";
|
||||
// std::cerr << std::endl;
|
||||
|
||||
uint32_t msg = msgType | (incoming ? MSG_DIRECTION_INCOMING : MSG_DIRECTION_OUTGOING);
|
||||
bdMsgHistoryItem msg(msgType, incoming, aboutId);
|
||||
msgHistory.insert(std::make_pair(ts, msg));
|
||||
}
|
||||
|
||||
void bdMsgHistoryList::setPeerType(time_t /* ts */, std::string version)
|
||||
{
|
||||
mPeerVersion = version;
|
||||
}
|
||||
|
||||
int bdMsgHistoryList::msgCount(time_t start_ts, time_t end_ts)
|
||||
{
|
||||
std::multimap<time_t, uint32_t>::iterator sit, eit, it;
|
||||
std::multimap<time_t, bdMsgHistoryItem>::iterator sit, eit, it;
|
||||
sit = msgHistory.lower_bound(start_ts);
|
||||
eit = msgHistory.upper_bound(end_ts);
|
||||
int count = 0;
|
||||
|
|
@ -26,7 +40,7 @@ int bdMsgHistoryList::msgCount(time_t start_ts, time_t end_ts)
|
|||
return count;
|
||||
}
|
||||
|
||||
bool bdMsgHistoryList::msgClear(time_t before)
|
||||
bool bdMsgHistoryList::msgClear(time_t before)
|
||||
{
|
||||
if (before == 0)
|
||||
{
|
||||
|
|
@ -48,11 +62,21 @@ bool bdMsgHistoryList::msgClear(time_t before)
|
|||
return false;
|
||||
}
|
||||
|
||||
void bdMsgHistoryList::msgClear()
|
||||
{
|
||||
msgHistory.clear();
|
||||
}
|
||||
|
||||
|
||||
void bdMsgHistoryList::clearHistory()
|
||||
{
|
||||
msgClear();
|
||||
}
|
||||
|
||||
void bdMsgHistoryList::printHistory(std::ostream &out, int mode, time_t start_ts, time_t end_ts)
|
||||
{
|
||||
//out << "AGE: MSGS => incoming, <= outgoing" << std::endl;
|
||||
std::multimap<time_t, uint32_t>::iterator sit, eit, it;
|
||||
std::multimap<time_t, bdMsgHistoryItem>::iterator sit, eit, it;
|
||||
sit = msgHistory.lower_bound(start_ts);
|
||||
eit = msgHistory.upper_bound(end_ts);
|
||||
time_t curr_ts = 0;
|
||||
|
|
@ -86,29 +110,32 @@ void bdMsgHistoryList::printHistory(std::ostream &out, int mode, time_t start_ts
|
|||
out << "\tTS: " << time(NULL) - curr_ts << " ";
|
||||
}
|
||||
|
||||
if (MSG_DIRECTION_INCOMING & it->second)
|
||||
std::string name;
|
||||
bitdht_msgtype(it->second.msgType, name);
|
||||
|
||||
if (it->second.incoming)
|
||||
{
|
||||
uint32_t type = it->second-MSG_DIRECTION_INCOMING;
|
||||
out << "( => ";
|
||||
std::string name;
|
||||
if (bitdht_msgtype(type, name))
|
||||
{
|
||||
out << name;
|
||||
}
|
||||
out << " )";
|
||||
out << "( =I> ";
|
||||
}
|
||||
else
|
||||
{
|
||||
out << "( ";
|
||||
uint32_t type = it->second - MSG_DIRECTION_OUTGOING;
|
||||
std::string name;
|
||||
if (bitdht_msgtype(type, name))
|
||||
{
|
||||
out << name;
|
||||
}
|
||||
out << " <= )";
|
||||
out << "( <O= ";
|
||||
}
|
||||
|
||||
out << name << " ";
|
||||
if ((it->second.aboutId.data[0] == 0)
|
||||
&& (it->second.aboutId.data[3] == 0)
|
||||
&& (it->second.aboutId.data[3] == 0)
|
||||
&& (it->second.aboutId.data[3] == 0))
|
||||
{
|
||||
/* don't print anything */
|
||||
}
|
||||
else
|
||||
{
|
||||
bdStdPrintNodeId(out, &(it->second.aboutId));
|
||||
}
|
||||
out << " )";
|
||||
|
||||
}
|
||||
break;
|
||||
} // end of switch.
|
||||
|
|
@ -126,7 +153,7 @@ bool bdMsgHistoryList::canSend()
|
|||
{
|
||||
std::cerr << "bdMsgHistoryList::canSend()";
|
||||
|
||||
std::multimap<time_t, uint32_t>::reverse_iterator rit;
|
||||
std::multimap<time_t, bdMsgHistoryItem>::reverse_iterator rit;
|
||||
|
||||
rit = msgHistory.rbegin();
|
||||
if (rit != msgHistory.rend())
|
||||
|
|
@ -159,11 +186,11 @@ bool bdMsgHistoryList::validPeer()
|
|||
{
|
||||
std::cerr << "bdMsgHistoryList::validPeer()";
|
||||
|
||||
std::multimap<time_t, uint32_t>::iterator it;
|
||||
std::multimap<time_t, bdMsgHistoryItem>::iterator it;
|
||||
|
||||
for(it = msgHistory.begin(); it != msgHistory.end(); it++)
|
||||
{
|
||||
if (MSG_DIRECTION_INCOMING & it->second)
|
||||
if (it->second.incoming)
|
||||
{
|
||||
std::cerr << " Incoming Msg... so validPeer";
|
||||
std::cerr << std::endl;
|
||||
|
|
@ -179,11 +206,123 @@ bool bdMsgHistoryList::validPeer()
|
|||
}
|
||||
|
||||
|
||||
#define MAX_PING_PER_MINUTE 2
|
||||
#define MAX_QUERY_PER_MINUTE 2
|
||||
|
||||
bool bdMsgHistoryList::analysePeer()
|
||||
|
||||
{
|
||||
/* analyse and print out details of the peers messages */
|
||||
bool flagged = false;
|
||||
|
||||
//out << "AGE: MSGS => incoming, <= outgoing" << std::endl;
|
||||
std::multimap<time_t, bdMsgHistoryItem>::iterator sit, eit, it;
|
||||
sit = msgHistory.begin();
|
||||
eit = msgHistory.end();
|
||||
if (sit == eit)
|
||||
{
|
||||
// nothing here.
|
||||
return false;
|
||||
}
|
||||
|
||||
time_t start_ts = sit->first;
|
||||
time_t end_ts = msgHistory.rbegin()->first; // must exist.
|
||||
|
||||
|
||||
// don't divide by zero.
|
||||
if (end_ts - start_ts < 60)
|
||||
{
|
||||
end_ts = start_ts + 60;
|
||||
}
|
||||
|
||||
|
||||
/* what do we want to analyse? */
|
||||
|
||||
/* if we have sent / recved too many queries or pings */
|
||||
|
||||
int in_ping = 0;
|
||||
int out_ping = 0;
|
||||
int in_query = 0;
|
||||
int out_query = 0;
|
||||
int in_other = 0;
|
||||
int out_other = 0;
|
||||
|
||||
for(it = sit; it != eit; it++)
|
||||
{
|
||||
if (it->second.incoming)
|
||||
{
|
||||
switch(it->second.msgType)
|
||||
{
|
||||
case BITDHT_MSG_TYPE_PING:
|
||||
in_ping++;
|
||||
break;
|
||||
case BITDHT_MSG_TYPE_FIND_NODE:
|
||||
in_query++;
|
||||
break;
|
||||
default:
|
||||
in_other++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
switch(it->second.msgType)
|
||||
{
|
||||
case BITDHT_MSG_TYPE_PING:
|
||||
out_ping++;
|
||||
break;
|
||||
case BITDHT_MSG_TYPE_FIND_NODE:
|
||||
out_query++;
|
||||
break;
|
||||
default:
|
||||
out_other++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
float in_ping_per_min = in_ping * 60.0 / (end_ts - start_ts);
|
||||
float out_ping_per_min = out_ping * 60.0 / (end_ts - start_ts);
|
||||
float in_query_per_min = in_query * 60.0 / (end_ts - start_ts);
|
||||
float out_query_per_min = out_query * 60.0 / (end_ts - start_ts);
|
||||
|
||||
if ((in_ping_per_min > MAX_PING_PER_MINUTE) ||
|
||||
(out_ping_per_min > MAX_PING_PER_MINUTE) ||
|
||||
(in_query_per_min > MAX_PING_PER_MINUTE) ||
|
||||
(out_query_per_min > MAX_PING_PER_MINUTE))
|
||||
{
|
||||
flagged = true;
|
||||
}
|
||||
|
||||
if (flagged)
|
||||
{
|
||||
/* print header */
|
||||
std::ostream &out = std::cerr;
|
||||
out << "BdHistoryAnalysis has flagged peer: ";
|
||||
bdStdPrintId(out, &mId);
|
||||
out << std::endl;
|
||||
|
||||
out << "PeerType: " << mPeerVersion;
|
||||
out << std::endl;
|
||||
|
||||
out << "Ping In Per Min : " << in_ping_per_min << std::endl;
|
||||
out << "Ping Out Per Min : " << out_ping_per_min << std::endl;
|
||||
out << "Query In Per Min : " << in_query_per_min << std::endl;
|
||||
out << "Query Out Per Min: " << out_query_per_min << std::endl;
|
||||
|
||||
out << "Message History: ";
|
||||
out << std::endl;
|
||||
|
||||
printHistory(out, 0, 0, time(NULL));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
bdHistory::bdHistory(time_t store_period)
|
||||
:mStorePeriod(store_period) { return; }
|
||||
|
||||
|
||||
void bdHistory::addMsg(const bdId *id, bdToken * /*transId*/, uint32_t msgType, bool incoming)
|
||||
void bdHistory::addMsg(const bdId *id, bdToken * /*transId*/, uint32_t msgType, bool incoming, const bdNodeId *aboutId)
|
||||
{
|
||||
//std::cerr << "bdHistory::addMsg() ";
|
||||
//bdStdPrintId(std::cerr, id);
|
||||
|
|
@ -193,10 +332,18 @@ void bdHistory::addMsg(const bdId *id, bdToken * /*transId*/, uint32_t msgType,
|
|||
|
||||
std::map<bdId, bdMsgHistoryList>::iterator it;
|
||||
bdMsgHistoryList &histRef = mHistory[*id]; /* will instaniate empty */
|
||||
histRef.addMsg(now, msgType, incoming);
|
||||
histRef.mId = *id;
|
||||
histRef.addMsg(now, msgType, incoming, aboutId);
|
||||
|
||||
/* add to mMsgTimeline */
|
||||
mMsgTimeline.insert(std::make_pair(now, MsgRegister(id, msgType, incoming)));
|
||||
mMsgTimeline.insert(std::make_pair(now, MsgRegister(id, msgType, incoming, aboutId)));
|
||||
}
|
||||
|
||||
void bdHistory::setPeerType(const bdId *id, std::string version)
|
||||
{
|
||||
std::map<bdId, bdMsgHistoryList>::iterator it;
|
||||
bdMsgHistoryList &histRef = mHistory[*id]; /* will instaniate empty */
|
||||
histRef.setPeerType(time(NULL), version);
|
||||
}
|
||||
|
||||
void bdHistory::printMsgs()
|
||||
|
|
@ -216,13 +363,13 @@ void bdHistory::printMsgs()
|
|||
/* print header */
|
||||
out << "Msgs for ";
|
||||
bdStdPrintId(out, &(it->first));
|
||||
out << " v:" << it->second.mPeerVersion;
|
||||
out << std::endl;
|
||||
|
||||
it->second.printHistory(out, 0, 0, time(NULL));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
out << "Msg Timeline:";
|
||||
time_t now = time(NULL);
|
||||
std::multimap<time_t, MsgRegister>::iterator hit;
|
||||
|
|
@ -233,11 +380,11 @@ void bdHistory::printMsgs()
|
|||
|
||||
if (hit->second.incoming)
|
||||
{
|
||||
out << " => ";
|
||||
out << " =I> ";
|
||||
}
|
||||
else
|
||||
{
|
||||
out << " <= ";
|
||||
out << " <O= ";
|
||||
}
|
||||
|
||||
std::string name;
|
||||
|
|
@ -264,8 +411,8 @@ void bdHistory::cleanupOldMsgs()
|
|||
return; // no cleanup
|
||||
}
|
||||
|
||||
std::list<bdId> to_cleanup;
|
||||
std::list<bdId>::iterator cit;
|
||||
std::set<bdId> to_cleanup;
|
||||
std::set<bdId>::iterator cit;
|
||||
|
||||
time_t before = time(NULL) - mStorePeriod;
|
||||
|
||||
|
|
@ -273,7 +420,7 @@ void bdHistory::cleanupOldMsgs()
|
|||
while((mMsgTimeline.begin() != mMsgTimeline.end()) && (mMsgTimeline.begin()->first < before))
|
||||
{
|
||||
std::multimap<time_t, MsgRegister>::iterator it = mMsgTimeline.begin();
|
||||
to_cleanup.push_back(it->second.id);
|
||||
to_cleanup.insert(it->second.id);
|
||||
mMsgTimeline.erase(it);
|
||||
}
|
||||
|
||||
|
|
@ -286,20 +433,31 @@ void bdHistory::cleanupOldMsgs()
|
|||
{
|
||||
if (hit->second.msgClear(before))
|
||||
{
|
||||
mHistory.erase(hit);
|
||||
// don't erase actual entry (so we remember peer type).
|
||||
//mHistory.erase(hit);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
void bdHistory::clearHistory()
|
||||
{
|
||||
mHistory.clear();
|
||||
// Switched to a alternative clear, so we don't drop peers, and remember their type.
|
||||
//mHistory.clear();
|
||||
|
||||
|
||||
std::map<bdId, bdMsgHistoryList> ::iterator it;
|
||||
for(it = mHistory.begin(); it != mHistory.end(); it++)
|
||||
{
|
||||
it->second.clearHistory();
|
||||
}
|
||||
}
|
||||
|
||||
bool bdHistory::canSend(const bdId *id)
|
||||
{
|
||||
|
||||
std::map<bdId, bdMsgHistoryList> ::iterator it;
|
||||
it = mHistory.find(*id);
|
||||
if (it != mHistory.end())
|
||||
|
|
@ -325,4 +483,144 @@ bool bdHistory::validPeer(const bdId *id)
|
|||
return false;
|
||||
}
|
||||
|
||||
bool bdHistory::analysePeers()
|
||||
{
|
||||
std::map<bdId, bdMsgHistoryList> ::iterator it;
|
||||
for(it = mHistory.begin(); it != mHistory.end(); it++)
|
||||
{
|
||||
it->second.analysePeer();
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
/* Temp data class. */
|
||||
class TypeStats
|
||||
{
|
||||
public:
|
||||
|
||||
TypeStats() :nodes(0) { return; }
|
||||
|
||||
std::map<uint32_t, uint32_t> incoming, outgoing;
|
||||
int nodes;
|
||||
|
||||
|
||||
void printStats(std::ostream &out, const TypeStats *refStats)
|
||||
{
|
||||
std::map<uint32_t, uint32_t>::iterator it;
|
||||
std::map<uint32_t, uint32_t>::const_iterator rit;
|
||||
|
||||
out << " Nodes: " << nodes;
|
||||
if (refStats)
|
||||
{
|
||||
out << " (" << 100.0 * nodes / (float) refStats->nodes << " %)";
|
||||
}
|
||||
out << std::endl;
|
||||
|
||||
out << " Incoming Msgs";
|
||||
out << std::endl;
|
||||
for(it = incoming.begin(); it != incoming.end(); it++)
|
||||
{
|
||||
uint32_t count = 0;
|
||||
if (refStats)
|
||||
{
|
||||
rit = refStats->incoming.find(it->first);
|
||||
if (rit != refStats->incoming.end())
|
||||
{
|
||||
count = rit->second;
|
||||
}
|
||||
}
|
||||
printStatsLine(out, it->first, it->second, count);
|
||||
}
|
||||
|
||||
out << " Outgoing Msgs";
|
||||
out << std::endl;
|
||||
for(it = outgoing.begin(); it != outgoing.end(); it++)
|
||||
{
|
||||
uint32_t count = 0;
|
||||
if (refStats)
|
||||
{
|
||||
rit = refStats->outgoing.find(it->first);
|
||||
if (rit != refStats->outgoing.end())
|
||||
{
|
||||
count = rit->second;
|
||||
}
|
||||
}
|
||||
printStatsLine(out, it->first, it->second, count);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
void printStatsLine(std::ostream &out, uint32_t msgType, uint32_t count, uint32_t global)
|
||||
{
|
||||
std::string name;
|
||||
bitdht_msgtype(msgType, name);
|
||||
out << "\t" << name << " " << count;
|
||||
if (global != 0)
|
||||
{
|
||||
out << " (" << 100.0 * count / (float) global << " %)";
|
||||
}
|
||||
out << std::endl;
|
||||
}
|
||||
|
||||
}; /* end of TypeStats */
|
||||
|
||||
bool bdHistory::peerTypeAnalysis()
|
||||
{
|
||||
|
||||
std::map<std::string, TypeStats> mTypeStats;
|
||||
TypeStats globalStats;
|
||||
|
||||
std::map<bdId, bdMsgHistoryList> ::iterator it;
|
||||
for(it = mHistory.begin(); it != mHistory.end(); it++)
|
||||
{
|
||||
if (it->second.msgHistory.empty())
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
std::string version = it->second.mPeerVersion;
|
||||
// group be first two bytes.
|
||||
version = it->second.mPeerVersion.substr(0,2);
|
||||
TypeStats &stats = mTypeStats[version];
|
||||
|
||||
stats.nodes++;
|
||||
globalStats.nodes++;
|
||||
|
||||
std::multimap<time_t, bdMsgHistoryItem>::iterator lit;
|
||||
for (lit = it->second.msgHistory.begin(); lit != it->second.msgHistory.end(); lit++)
|
||||
{
|
||||
if (lit->second.incoming)
|
||||
{
|
||||
stats.incoming[lit->second.msgType]++;
|
||||
globalStats.incoming[lit->second.msgType]++;
|
||||
}
|
||||
else
|
||||
{
|
||||
stats.outgoing[lit->second.msgType]++;
|
||||
globalStats.outgoing[lit->second.msgType]++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
std::map<std::string, TypeStats>::iterator tit;
|
||||
for(tit = mTypeStats.begin(); tit != mTypeStats.end(); tit++)
|
||||
{
|
||||
std::cerr << "Stats for Peer Type: " << tit->first;
|
||||
std::cerr << std::endl;
|
||||
tit->second.printStats(std::cerr, &globalStats);
|
||||
}
|
||||
|
||||
std::cerr << "Global Stats: ";
|
||||
std::cerr << std::endl;
|
||||
|
||||
globalStats.printStats(std::cerr, NULL);
|
||||
return true;
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue