From 26af7c900e590775a9877755fed982a8593cdd89 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 17 Apr 2016 22:58:48 -0400 Subject: [PATCH 01/43] changed heart beat limit to a larger value. Apparently too tight a value causes disconnections due to packets being stuck sometimes (old bug) --- libretroshare/src/pqi/pqiperson.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/pqi/pqiperson.cc b/libretroshare/src/pqi/pqiperson.cc index e4efcf136..efd78afb9 100644 --- a/libretroshare/src/pqi/pqiperson.cc +++ b/libretroshare/src/pqi/pqiperson.cc @@ -153,7 +153,7 @@ int pqiperson::tick() #endif //if lastHeartbeatReceived is 0, it might be not activated so don't do a net reset. - if ( active && time(NULL) > lastHeartbeatReceived + HEARTBEAT_REPEAT_TIME * 5) + if ( active && time(NULL) > lastHeartbeatReceived + HEARTBEAT_REPEAT_TIME * 20) { int ageLastIncoming = time(NULL) - activepqi->getLastIncomingTS(); From 8be53dd79a77fd832986ea67e57e843ad6a608fa Mon Sep 17 00:00:00 2001 From: Chuck Rhode aka Manuel Davis Date: Tue, 19 Apr 2016 11:11:58 -0500 Subject: [PATCH 02/43] Improve inserting placemark in ChatLobby. *append* inserts text at end but with formatting in effect at current cursor position, such as in the middle of a hotlink, which would be strange. This OTOH inserts text with formatting in effect on the last line, which may be strange or not. --- retroshare-gui/src/gui/chat/ChatWidget.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/retroshare-gui/src/gui/chat/ChatWidget.cpp b/retroshare-gui/src/gui/chat/ChatWidget.cpp index d3613a3c5..cd9353b65 100644 --- a/retroshare-gui/src/gui/chat/ChatWidget.cpp +++ b/retroshare-gui/src/gui/chat/ChatWidget.cpp @@ -1725,7 +1725,11 @@ void ChatWidget::quote() void ChatWidget::dropPlacemark() { - ui->textBrowser->append("----------"); + ui->textBrowser->moveCursor(QTextCursor::End); // *append* inserts text at end but with formatting in effect at + ui->textBrowser->append("----------"); // current cursor position, such as in the middle of a hotlink, + // which would be strange. This OTOH inserts text with + // formatting in effect on the last line, which may be strange + // or not. } void ChatWidget::saveImage() From 5645e44f9cf83abbc16494afdd18f9170e23a2c8 Mon Sep 17 00:00:00 2001 From: Chuck Rhode aka Manuel Davis Date: Tue, 19 Apr 2016 11:32:56 -0500 Subject: [PATCH 03/43] Change regular expression(s) for identifying hotlinks in feral text. --- retroshare-gui/src/util/HandleRichText.cpp | 90 ++++++++++++++++------ 1 file changed, 68 insertions(+), 22 deletions(-) diff --git a/retroshare-gui/src/util/HandleRichText.cpp b/retroshare-gui/src/util/HandleRichText.cpp index 0c059f048..d686e12b2 100644 --- a/retroshare-gui/src/util/HandleRichText.cpp +++ b/retroshare-gui/src/util/HandleRichText.cpp @@ -17,6 +17,10 @@ * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, * Boston, MA 02110-1301, USA. + * + * ccr . 2016 Jan 30 . Change regular expression(s) for identifying + * . . hotlinks in feral text. + * ****************************************************************/ #include @@ -56,7 +60,7 @@ protected: public: const EmbeddedType myType; - QRegExp myRE; + QList myREs; }; /** @@ -67,10 +71,42 @@ class EmbedInHtmlAhref : public EmbedInHtml public: EmbedInHtmlAhref() : EmbedInHtml(Ahref) { - myRE.setPattern("(\\bretroshare://[^\\s]*)|(\\bhttps?://[^\\s]*)|(\\bfile://[^\\s]*)|(\\bwww\\.[^\\s]*)"); - } -}; + // myRE.setPattern("(\\bretroshare://[^\\s]*)|(\\bhttps?://[^\\s]*)|(\\bfile://[^\\s]*)|(\\bwww\\.[^\\s]*)"); + // The following regular expressions for finding URLs in + // plain text are borrowed from *gnome-terminal*: + + QString regPassCharset = "[-\\w,?;\\.:/!%$^*&~\\\"#']"; + QString regHost = "[-\\w]+(\\.[-\\w]+)*"; + QString regPort = "(?:\\:\\d{1,5})?"; + QString regPathCharset = "[-\\w_$\\.+!*,;@&=?/~#%]"; + QString regPathTermSet = "[^\\]'.}<>) \\t\\r\\n,\\\"]"; + QStringList regSchemes; +// regSchemes.append("news:"); +// regSchemes.append("telnet:"); +// regSchemes.append("nntp:"); +// regSchemes.append("file:/"); + regSchemes.append("https?:"); +// regSchemes.append("ftps?:"); +// regSchemes.append("sftp:"); +// regSchemes.append("webcal:"); + regSchemes.append("retroshare:"); + QString regScheme = "((?:" + regSchemes.join(")|(?:") + "))"; + QString regUserPass = "[-\\w]+(?:%s+)?" % regPassCharset; + QString regUrlPath = "(?:(/" + regPathCharset + "+(?:[(]" + regPathCharset +"*[)])*" + regPathCharset + "*)*" + regPathTermSet + ")?"; + QStringList regHotLinkFinders; + regHotLinkFinders.append(regScheme + "//(?:" + regUserPass + "@)?"+ regHost + regPort + regUrlPath); +// regHotLinkFinders.append("(?:(?:www)|(?:ftp))[-\\w]*\\." + regHost + regPort + regUrlPath); +// regHotLinkFinders.append("(?:(?:callto:)|(?:h323:)|(?:sip:))[-\\w][-\\w\\.]*(?:" + regPort + "/[a-z0-9]+)?@" + regHost); +// regHotLinkFinders.append("(?:mailto:)?[-\\w][-\\w\\.]*@[-\\w]+\\." + regHost); +// regHotLinkFinders.append("news:[\\w^_{|}~!\\\"#$%&'()*+,\\./;:=?`]+"); + while (!regHotLinkFinders.isEmpty()) { + myREs.append(QRegExp(regHotLinkFinders.takeFirst(), Qt::CaseInsensitive)); + }; + } +}; + + /** * This class is used to store information for embedding smileys into tags. * @@ -134,7 +170,7 @@ void RsHtml::initEmoticons(const QHash< QString, QString >& hash) */ } newRE.chop(1); // remove last | - defEmbedImg.myRE.setPattern(newRE); + defEmbedImg.myREs.append(QRegExp(newRE)); } bool RsHtml::canReplaceAnchor(QDomDocument &/*doc*/, QDomElement &/*element*/, const RetroShareLink &link) @@ -243,7 +279,7 @@ void RsHtml::replaceAnchorWithImg(QDomDocument &doc, QDomElement &element, QText * nodes. Any other kind of node is terminal. * * If the node is of type Text, its data is checked against the user-provided - * regular expression. If there is a match, the text is cut in three parts: the + * regular expression(s). If there is a match, the text is cut in three parts: the * preceding part that will be inserted before, the part to be replaced, and the * following part which will be itself checked against the regular expression. * @@ -252,13 +288,10 @@ void RsHtml::replaceAnchorWithImg(QDomDocument &doc, QDomElement &element, QText * * @param[in] doc The whole DOM tree, necessary to create new nodes * @param[in,out] currentElement The current node (which is of type Element) - * @param[in] embedInfos The regular expression and the type of embedding to use + * @param[in] embedInfos The regular expression(s) and the type of embedding to use */ void RsHtml::embedHtml(QTextDocument *textDocument, QDomDocument& doc, QDomElement& currentElement, EmbedInHtml& embedInfos, ulong flag) { - if(embedInfos.myRE.pattern().length() == 0) // we'll get stuck with an empty regexp - return; - QDomNodeList children = currentElement.childNodes(); for(uint index = 0; index < (uint)children.length(); index++) { QDomNode node = children.item(index); @@ -298,15 +331,20 @@ void RsHtml::embedHtml(QTextDocument *textDocument, QDomDocument& doc, QDomEleme } } else if(node.isText()) { - // child is a text, we parse it - QString tempText = node.toText().data(); - if(embedInfos.myRE.indexIn(tempText) == -1) + // child is a text, we parse it + QString tempText = node.toText().data(); + for (int patNdx = 0; patNdx < embedInfos.myREs.size(); ++patNdx) { + QRegExp myRE = embedInfos.myREs.at(patNdx); + if(myRE.pattern().length() == 0) // we'll get stuck with an empty regexp + return; + + if(myRE.indexIn(tempText) == -1) continue; // there is at least one link inside, we start replacing int currentPos = 0; int nextPos = 0; - while((nextPos = embedInfos.myRE.indexIn(tempText, currentPos)) != -1) { + while((nextPos = myRE.indexIn(tempText, currentPos)) != -1) { // if nextPos == 0 it means the text begins by a link if(nextPos > 0) { QDomText textPart = doc.createTextNode(tempText.mid(currentPos, nextPos - currentPos)); @@ -320,10 +358,10 @@ void RsHtml::embedHtml(QTextDocument *textDocument, QDomDocument& doc, QDomEleme case Ahref: { insertedTag = doc.createElement("a"); - insertedTag.setAttribute("href", embedInfos.myRE.cap(0)); - insertedTag.appendChild(doc.createTextNode(embedInfos.myRE.cap(0))); + insertedTag.setAttribute("href", myRE.cap(0)); + insertedTag.appendChild(doc.createTextNode(myRE.cap(0))); - RetroShareLink link(embedInfos.myRE.cap(0)); + RetroShareLink link(myRE.cap(0)); if (link.valid()) { QString title = link.title(); if (!title.isEmpty()) { @@ -340,11 +378,11 @@ void RsHtml::embedHtml(QTextDocument *textDocument, QDomDocument& doc, QDomEleme { insertedTag = doc.createElement("img"); const EmbedInHtmlImg& embedImg = static_cast(embedInfos); - // embedInfos.myRE.cap(0) may include spaces at the end/beginning -> trim! - insertedTag.setAttribute("src", embedImg.smileys[embedInfos.myRE.cap(0).trimmed()]); + // myRE.cap(0) may include spaces at the end/beginning -> trim! + insertedTag.setAttribute("src", embedImg.smileys[myRE.cap(0).trimmed()]); /* * NOTE - * Trailing spaces are matched, too. This leads to embedInfos.myRE.matchedLength() being incorrect. + * Trailing spaces are matched, too. This leads to myRE.matchedLength() being incorrect. * This hack reduces nextPos by one so that the new value of currentPos is calculated corretly. * This is needed to match multiple smileys since the leading whitespace in front of a smiley is required! * @@ -353,7 +391,7 @@ void RsHtml::embedHtml(QTextDocument *textDocument, QDomDocument& doc, QDomEleme * NOTE * Preceding spaces are also matched and removed. */ - if(embedInfos.myRE.cap(0).endsWith(' ')) + if(myRE.cap(0).endsWith(' ')) nextPos--; } break; @@ -362,7 +400,7 @@ void RsHtml::embedHtml(QTextDocument *textDocument, QDomDocument& doc, QDomEleme currentElement.insertBefore(insertedTag, node); index++; - currentPos = nextPos + embedInfos.myRE.matchedLength(); + currentPos = nextPos + myRE.matchedLength(); } // text after the last link, only if there's one, don't touch the index @@ -375,10 +413,18 @@ void RsHtml::embedHtml(QTextDocument *textDocument, QDomDocument& doc, QDomEleme index--; currentElement.removeChild(node); + break; + // We'd better not expect that + // subsequent hotlink patterns + // wouldn't also match replacements + // we've already made. They might, so + // skip 'em to be safe. + }; } } } + /** * Save space and tab out of bracket that XML loose. * From 39553a601c2c0afc4d0daaaf73964bd7b6f62a66 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 19 Apr 2016 21:58:30 -0400 Subject: [PATCH 04/43] removed debug info --- libretroshare/src/gxs/rsgenexchange.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/gxs/rsgenexchange.cc b/libretroshare/src/gxs/rsgenexchange.cc index cb50aceb6..468792b18 100644 --- a/libretroshare/src/gxs/rsgenexchange.cc +++ b/libretroshare/src/gxs/rsgenexchange.cc @@ -61,7 +61,7 @@ static const uint32_t INDEX_AUTHEN_ADMIN = 0x00000040; // admin key #define GXS_MASK "GXS_MASK_HACK" -#define GEN_EXCH_DEBUG 1 +//#define GEN_EXCH_DEBUG 1 #define MSG_CLEANUP_PERIOD 60*5 // 5 minutes #define INTEGRITY_CHECK_PERIOD 60*30 // 30 minutes From fc725b03a86f42fe8d92f8088fcc9d05101c7c8a Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 19 Apr 2016 22:04:30 -0400 Subject: [PATCH 05/43] added some debug info to pqistreamer --- libretroshare/src/pqi/pqistreamer.cc | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index f60e21d45..e8dc8053d 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -30,6 +30,7 @@ #include "util/rsdebug.h" #include "util/rsstring.h" #include "util/rsprint.h" +#include "util/rsscopetimer.h" #include "pqi/pqistreamer.h" #include "rsserver/p3face.h" @@ -440,18 +441,12 @@ int pqistreamer::handleoutgoing_locked() if ((!(mBio->cansend(0))) || (maxbytes < sentbytes)) { -#ifdef DEBUG_TRANSFERS +//#ifdef DEBUG_PQISTREAMER if (maxbytes < sentbytes) - { - std::cerr << "pqistreamer::handleoutgoing_locked() Stopped sending sentbytes > maxbytes. Sent " << sentbytes << " bytes "; - std::cerr << std::endl; - } + std::cerr << "pqistreamer::handleoutgoing_locked() Stopped sending: bio not ready. maxbytes=" << maxbytes << ", sentbytes=" << sentbytes << std::endl; else - { - std::cerr << "pqistreamer::handleoutgoing_locked() Stopped sending at cansend() is false"; - std::cerr << std::endl; - } -#endif + std::cerr << "pqistreamer::handleoutgoing_locked() Stopped sending: sentbytes=" << sentbytes << ", max=" << maxbytes << std::endl; +//#endif return 0; } @@ -493,6 +488,8 @@ int pqistreamer::handleoutgoing_locked() #endif if (mPkt_wpending) { + RsScopeTimer tmer("pqistreamer:"+PeerId().toStdString()) ; + // write packet. #ifdef DEBUG_PQISTREAMER std::cout << "Sending Out Pkt of size " << mPkt_wpending_size << " !" << std::endl; @@ -507,11 +504,14 @@ int pqistreamer::handleoutgoing_locked() // std::cerr << out << std::endl ; pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, out); #endif + std::cerr << PeerId() << ": sending failed. Only " << ss << " bytes sent over " << mPkt_wpending_size << std::endl; // pkt_wpending will kept til next time. // ensuring exactly the same data is written (openSSL requirement). return -1; } + else + std::cerr << PeerId() << ": sent " << ss << " bytes " << std::endl; ++nsent; outSentBytes_locked(mPkt_wpending_size); // this is the only time where we know exactly what was sent. From d01777165029f777cb1c50e8778b9aa4e2367b78 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Apr 2016 18:08:26 -0400 Subject: [PATCH 06/43] fixed missing sendTime in distant chat, causing inconsistent display in GUI --- libretroshare/src/chat/distantchat.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/libretroshare/src/chat/distantchat.cc b/libretroshare/src/chat/distantchat.cc index 03acb6ff8..3c951bb43 100644 --- a/libretroshare/src/chat/distantchat.cc +++ b/libretroshare/src/chat/distantchat.cc @@ -264,6 +264,7 @@ bool DistantChatService::initiateDistantChatConnexion(const RsGxsId& to_gxs_id, RsChatMsgItem *item = new RsChatMsgItem; item->message = "[Starting distant chat. Please wait for secure tunnel to be established]" ; item->chatFlags = RS_CHAT_FLAG_PRIVATE ; + item->sendTime = time(NULL) ; item->PeerId(RsPeerId(tunnel_id)) ; handleRecvChatMsgItem(item) ; From aba3d2fb35d17bd5025c3271d20717b16ab4008f Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Apr 2016 18:24:02 -0400 Subject: [PATCH 07/43] removed memory leak due to zeroing (on purpose!) a data chunk in GRouter before deleting it --- libretroshare/src/grouter/p3grouter.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/libretroshare/src/grouter/p3grouter.cc b/libretroshare/src/grouter/p3grouter.cc index 2770b52dd..306f72d74 100644 --- a/libretroshare/src/grouter/p3grouter.cc +++ b/libretroshare/src/grouter/p3grouter.cc @@ -613,7 +613,6 @@ RsGRouterAbstractMsgItem *GRouterDataInfo::addDataChunk(RsGRouterTransactionChun { RsItem *data_item = RsGRouterSerialiser().deserialise(incoming_data_buffer->chunk_data,&incoming_data_buffer->chunk_size) ; - incoming_data_buffer->chunk_data = NULL; delete incoming_data_buffer; incoming_data_buffer = NULL ; From d6ae71ebc80dbf27c78ae85d0cc3293ae1ba5141 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Apr 2016 20:21:29 -0400 Subject: [PATCH 08/43] removed pointer to QStringList in QHash, causing memory loss --- retroshare-gui/src/util/HandleRichText.cpp | 27 ++++++++++------------ 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/retroshare-gui/src/util/HandleRichText.cpp b/retroshare-gui/src/util/HandleRichText.cpp index d686e12b2..0e9a278dd 100644 --- a/retroshare-gui/src/util/HandleRichText.cpp +++ b/retroshare-gui/src/util/HandleRichText.cpp @@ -658,7 +658,7 @@ static void findBestColor(QString &val, qreal bglum, qreal desiredContrast) */ static void optimizeHtml(QDomDocument& doc , QDomElement& currentElement - , QHash &stylesList + , QHash &stylesList , QHash &knownStyle) { if (doc.documentElement().namedItem("style").toElement().attributeNode("RSOptimized").isAttr()) { @@ -677,10 +677,10 @@ static void optimizeHtml(QDomDocument& doc QString keyvalue = pair.at(1); keyvalue.replace(";",""); QStringList classUsingIt(pair.at(0).split(',')); - QStringList* exported = new QStringList(); + QStringList exported ; foreach (QString keyVal, classUsingIt) { if(!keyVal.trimmed().isEmpty()) { - exported->append(keyVal.trimmed().replace(".","")); + exported.append(keyVal.trimmed().replace(".","")); } } @@ -805,14 +805,10 @@ static void optimizeHtml(QDomDocument& doc foreach (QString pair, styles) { pair.replace(" ",""); if (!pair.isEmpty()) { - QStringList* stylesListItem = stylesList.value(pair); - if(!stylesListItem){ - // If value doesn't exist create it - stylesListItem = new QStringList(); - stylesList.insert(pair, stylesListItem); - } + QStringList& stylesListItem = stylesList[pair]; + //Add the new class to this value - stylesListItem->push_back(className); + stylesListItem.push_back(className); } } } @@ -846,7 +842,7 @@ static void optimizeHtml(QDomDocument& doc * @param desiredMinimumFontSize: Minimum font size. */ static void styleCreate(QDomDocument& doc - , QHash stylesList + , QHash& stylesList , unsigned int flag , qreal bglum , qreal desiredContrast @@ -886,12 +882,12 @@ static void styleCreate(QDomDocument& doc QString style = ""; - QHashIterator it(stylesList); + QHashIterator it(stylesList); while(it.hasNext()) { it.next(); - QStringList* classUsingIt = it.value(); + const QStringList& classUsingIt ( it.value()) ; bool first = true; - foreach(QString className, *classUsingIt) { + foreach(QString className, classUsingIt) { if (!className.trimmed().isEmpty()) { style += QString(first?".":",.") + className;// + " "; first = false; @@ -1007,7 +1003,8 @@ void RsHtml::optimizeHtml(QString &text, unsigned int flag /*= 0*/ } QDomElement body = doc.documentElement(); - QHash stylesList; + + QHash stylesList; QHash knownStyle; ::optimizeHtml(doc, body, stylesList, knownStyle); From 057249225e0c341dd883d50b1abce40a16a2a162 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Apr 2016 21:20:47 -0400 Subject: [PATCH 09/43] severely reduced packet grouping, which limited the effectiveness of QoS --- libretroshare/src/pqi/pqistreamer.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index f60e21d45..094d9d42a 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -456,7 +456,7 @@ int pqistreamer::handleoutgoing_locked() return 0; } #define GROUP_OUTGOING_PACKETS 1 -#define PACKET_GROUPING_SIZE_LIMIT 32768 +#define PACKET_GROUPING_SIZE_LIMIT 512 // send a out_pkt., else send out_data. unless // there is a pending packet. if (!mPkt_wpending) From e82c217cd25cdb8a427eb92c64e1558dce534005 Mon Sep 17 00:00:00 2001 From: csoler Date: Wed, 20 Apr 2016 22:42:09 -0400 Subject: [PATCH 10/43] half-implemented packet slicing (QoS part done) --- libretroshare/src/pqi/pqiqos.cc | 41 +++++++- libretroshare/src/pqi/pqiqos.h | 127 +++++++++++++++++++----- libretroshare/src/pqi/pqiqosstreamer.cc | 9 +- libretroshare/src/pqi/pqiqosstreamer.h | 2 +- libretroshare/src/pqi/pqistreamer.cc | 50 +++++++--- 5 files changed, 179 insertions(+), 50 deletions(-) diff --git a/libretroshare/src/pqi/pqiqos.cc b/libretroshare/src/pqi/pqiqos.cc index b0d5ec5f5..1453f559b 100644 --- a/libretroshare/src/pqi/pqiqos.cc +++ b/libretroshare/src/pqi/pqiqos.cc @@ -6,6 +6,8 @@ #include "pqiqos.h" +static const uint32_t MAX_COUNTER_VALUE = 1024u*1024u ; // 2^20 + pqiQoS::pqiQoS(uint32_t nb_levels,float alpha) : _item_queues(nb_levels),_alpha(alpha) { @@ -44,7 +46,7 @@ void pqiQoS::print() const std::cerr << std::endl; } -void pqiQoS::in_rsItem(void *ptr,int priority) +void pqiQoS::in_rsItem(void *ptr,int size,int priority) { if(uint32_t(priority) >= _item_queues.size()) { @@ -52,8 +54,11 @@ void pqiQoS::in_rsItem(void *ptr,int priority) priority = _item_queues.size()-1 ; } - _item_queues[priority].push(ptr) ; + _item_queues[priority].push(ptr,size,_id_counter++) ; ++_nb_items ; + + if(_id_counter >= MAX_COUNTER_VALUE) + _id_counter = 0 ; } // int pqiQoS::gatherStatistics(std::vector& per_service_count,std::vector& per_priority_count) const @@ -81,6 +86,19 @@ void pqiQoS::in_rsItem(void *ptr,int priority) void *pqiQoS::out_rsItem() +{ + bool starts,ends ; + uint32_t packet_id,offset,size ; + + void *res = out_rsItem(~0u,16,offset,size,starts,ends,packet_id) ; + + if(!starts || !ends) + std::cerr << "(EE) protocol error in pqiQoS. Will eventually kill connection!" << std::endl; + + return res ; +} + +void *pqiQoS::out_rsItem(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) { // Go through the queues. Increment counters. @@ -105,11 +123,26 @@ void *pqiQoS::out_rsItem() if(last >= 0) { assert(_nb_items > 0) ; - --_nb_items ; - return _item_queues[last].pop(); + + // now chop a slice of this item + + void *res = _item_queues[last].slice(max_slice_size,offset,size,starts,ends,packet_id) ; + + if(ends) + --_nb_items ; + + if( (offset % offset_unit) != 0) + std::cerr << "(EE) Severe error in pqiQoS::out_rsItem(). offset unit inconsistent with calculated offset." << std::endl; + + offset /= offset_unit ; + + return res ; } else return NULL ; } + + + diff --git a/libretroshare/src/pqi/pqiqos.h b/libretroshare/src/pqi/pqiqos.h index 0b7939f80..438f45622 100644 --- a/libretroshare/src/pqi/pqiqos.h +++ b/libretroshare/src/pqi/pqiqos.h @@ -36,55 +36,128 @@ #include #include +#include +#include #include #include +#include + class pqiQoS { public: pqiQoS(uint32_t max_levels,float alpha) ; + struct ItemRecord + { + void *data ; + uint32_t current_offset ; + uint32_t size ; + uint32_t id ; + }; + class ItemQueue { - public: - ItemQueue() - { - _item_count =0 ; - } - void *pop() + public: + ItemQueue() + { + _item_count =0 ; + } + void *pop() + { + if(_items.empty()) + return NULL ; + + void *item = _items.front().data ; + _items.pop_front() ; + --_item_count ; + + return item ; + } + + void *slice(uint32_t max_size,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) + { + if(_items.empty()) + return NULL ; + + ItemRecord& rec(_items.front()) ; + packet_id = rec.id ; + + // readily get rid of the item if it can be sent as a whole + + if(rec.current_offset == 0 && rec.size < max_size) + { + offset = 0 ; + starts = true ; + ends = true ; + size = rec.size ; + + return pop() ; + } + starts = (rec.current_offset == 0) ; + ends = (rec.current_offset + max_size > rec.size) ; + + if(rec.size < rec.current_offset) { - if(_items.empty()) - return NULL ; - - void *item = _items.front() ; - _items.pop_front() ; - --_item_count ; - - return item ; + std::cerr << "(EE) severe error in slicing in QoS." << std::endl; + pop() ; + return NULL ; } - - void push(void *item) + + size = std::min(max_size, uint32_t((int)rec.size - (int)rec.current_offset)) ; + void *mem = rs_malloc(size) ; + + if(!mem) { - _items.push_back(item) ; - ++_item_count ; - } + std::cerr << "(EE) memory allocation error in QoS." << std::endl; + pop() ; + return NULL ; + } + + memcpy(mem,&((unsigned char*)rec.data)[rec.current_offset],size) ; + + if(ends) // we're taking the whole stuff. So we can delete the entry. + { + free(rec.data) ; + _items.pop_front() ; + } + else + rec.current_offset += size ; + + return mem ; + } - uint32_t size() const { return _item_count ; } + void push(void *item,uint32_t size,uint32_t id) + { + ItemRecord rec ; + + rec.data = item ; + rec.current_offset = 0 ; + rec.size = size ; + rec.id = id ; + + _items.push_back(rec) ; + ++_item_count ; + } - float _threshold ; - float _counter ; - float _inc ; - uint32_t _item_count ; - std::list _items ; + uint32_t size() const { return _item_count ; } + + float _threshold ; + float _counter ; + float _inc ; + uint32_t _item_count ; + + std::list _items ; }; // This function pops items from the queue, y order of priority // void *out_rsItem() ; + void *out_rsItem(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) ; // This function is used to queue items. // - void in_rsItem(void *item,int priority) ; + void in_rsItem(void *item, int size, int priority) ; void print() const ; uint64_t qos_queue_size() const { return _nb_items ; } @@ -105,6 +178,8 @@ class pqiQoS std::vector _item_queues ; float _alpha ; uint64_t _nb_items ; + + uint32_t _id_counter ; }; diff --git a/libretroshare/src/pqi/pqiqosstreamer.cc b/libretroshare/src/pqi/pqiqosstreamer.cc index fc0dc2084..5470b55b8 100644 --- a/libretroshare/src/pqi/pqiqosstreamer.cc +++ b/libretroshare/src/pqi/pqiqosstreamer.cc @@ -52,10 +52,11 @@ int pqiQoSstreamer::getQueueSize(bool in) void pqiQoSstreamer::locked_storeInOutputQueue(void *ptr,int priority) { - _total_item_size += getRsItemSize(ptr) ; + uint32_t size = getRsItemSize(ptr) ; + _total_item_size += size ; ++_total_item_count ; - pqiQoS::in_rsItem(ptr,priority) ; + pqiQoS::in_rsItem(ptr,size,priority) ; } void pqiQoSstreamer::locked_clear_out_queue() @@ -65,9 +66,9 @@ void pqiQoSstreamer::locked_clear_out_queue() _total_item_count = 0 ; } -void *pqiQoSstreamer::locked_pop_out_data() +void *pqiQoSstreamer::locked_pop_out_data(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) { - void *out = pqiQoS::out_rsItem() ; + void *out = pqiQoS::out_rsItem(max_slice_size,offset_unit,offset,size,starts,ends,packet_id) ; if(out != NULL) { diff --git a/libretroshare/src/pqi/pqiqosstreamer.h b/libretroshare/src/pqi/pqiqosstreamer.h index 574ba94d6..da5cba923 100644 --- a/libretroshare/src/pqi/pqiqosstreamer.h +++ b/libretroshare/src/pqi/pqiqosstreamer.h @@ -40,7 +40,7 @@ class pqiQoSstreamer: public pqithreadstreamer, public pqiQoS virtual int locked_out_queue_size() const { return _total_item_count ; } virtual void locked_clear_out_queue() ; virtual int locked_compute_out_pkt_size() const { return _total_item_size ; } - virtual void *locked_pop_out_data() ; + virtual void *locked_pop_out_data(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id); //virtual int locked_gatherStatistics(std::vector& per_service_count,std::vector& per_priority_count) const; // extracting data. diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index e8dc8053d..37bdbe065 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -400,6 +400,37 @@ time_t pqistreamer::getLastIncomingTS() return mLastIncomingTs; } +// Packet slicing: +// +// Old : 02 0014 03 00000026 [data, 26 bytes] => [version 1B] [service 2B][subpacket 1B] [size 4B] +// New1: fv 0014 03 xxxxx sss [data, sss bytes] => [flags 0.5B version 0.5B] [service 2B][subpacket 1B] [packet counter 2.5B size 1.5B] +// New2: f xxxxxx ooo sssss [data, sss bytes] => [flags 0.5B] [2^24 packet count] [2^16 offset (in units of 16)] [size] +// +// Flags: 0x1 => incomplete packet continued after +// Flags: 0x2 => packet ending a previously incomplete packet +// +// - backward compatibility: +// * send one packet with service + subpacket = ffffff. Old peers will silently ignore such packets. +// * if received, mark the peer as able to decode the new packet type +// +// Mode 1: +// - Encode length on 1.5 Bytes (10 bits) => max slice size = 1024 +// - Encode packet ID on 2.5 Bytes (20 bits) => packet counter = [0...1056364] +// Mode 2: +// - Encode flags on 0.5 Bytes ( 4 bits) +// - Encode packet ID on 3.5 Bytes (28 bits) => packet counter = [0...16777216] +// - Encode offset on 2.0 Bytes (16 bits) => 65536 * 16 = // ax packet size = 1048576 +// - Encode size on 2.0 Bytes (16 bits) => 65536 // max slice size = 65536 +// +// - limit packet grouping to max size 1024. +// - new peers need to read flux, and properly extract partial sizes, and combine packets based on packet counter. +// - on sending, RS should grab slices of max size 1024 from pqiQoS. If smaller, possibly pack them together. +// pqiQoS keeps track of sliced packets and makes sure the output is consistent: +// * when a large packet needs to be send, only takes a slice and return it, and update the remaining part +// * always consider priority when taking new slices => a newly arrived fast packet will always get through. +// +// Max slice size should be customisable, depending on bandwidth. + int pqistreamer::handleoutgoing_locked() { #ifdef DEBUG_PQISTREAMER @@ -450,18 +481,16 @@ int pqistreamer::handleoutgoing_locked() return 0; } -#define GROUP_OUTGOING_PACKETS 1 -#define PACKET_GROUPING_SIZE_LIMIT 32768 +#define OPTIMAL_PACKET_SIZE 512 // send a out_pkt., else send out_data. unless // there is a pending packet. if (!mPkt_wpending) -#ifdef GROUP_OUTGOING_PACKETS { void *dta; mPkt_wpending_size = 0 ; int k=0; - while(mPkt_wpending_size < (uint32_t)maxbytes && mPkt_wpending_size < PACKET_GROUPING_SIZE_LIMIT && (dta = locked_pop_out_data())!=NULL ) + while(mPkt_wpending_size < (uint32_t)maxbytes && mPkt_wpending_size < OPTIMAL_PACKET_SIZE && (dta = locked_pop_out_data())!=NULL ) { uint32_t s = getRsItemSize(dta); mPkt_wpending = realloc(mPkt_wpending,s+mPkt_wpending_size) ; @@ -475,17 +504,7 @@ int pqistreamer::handleoutgoing_locked() std::cerr << "Packed " << k << " packets into " << mPkt_wpending_size << " bytes." << std::endl; #endif } -#else - { - void *dta = locked_pop_out_data() ; - - if(dta != NULL) - { - mPkt_wpending = dta ; - mPkt_wpending_size = getRsItemSize(dta); - } - } -#endif + if (mPkt_wpending) { RsScopeTimer tmer("pqistreamer:"+PeerId().toStdString()) ; @@ -804,6 +823,7 @@ continue_packet: std::cerr << "[" << (void*)pthread_self() << "] " << "deserializing. Size=" << pktlen << std::endl ; #endif + std::cerr << "Got packet with header " << RsUtil::BinToHex((char*)block,8) << std::endl; RsItem *pkt = mRsSerialiser->deserialise(block, &pktlen); if ((pkt != NULL) && (0 < handleincomingitem_locked(pkt,pktlen))) From 3fae108cb166ab83447dccd8187d549aa8d82cca Mon Sep 17 00:00:00 2001 From: Cyril Soler Date: Thu, 21 Apr 2016 12:58:48 -0400 Subject: [PATCH 11/43] removed div by zero in transfers dialog --- retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp index 0a7b7cb45..ce0f7f908 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp @@ -942,7 +942,8 @@ int TransfersDialog::addItem(int row, const FileInfo &fileInfo) qlonglong completed = fileInfo.transfered; qlonglong remaining = fileInfo.size - fileInfo.transfered; - qlonglong downloadtime = (fileInfo.size - fileInfo.transfered) / (fileInfo.tfRate * 1024.0); + + qlonglong downloadtime = (fileInfo.tfRate > 0)?( (fileInfo.size - fileInfo.transfered) / (fileInfo.tfRate * 1024.0) ) : 0 ; qint64 qi64LastDL = fileInfo.lastTS ; //std::numeric_limits::max(); if (qi64LastDL == 0) // file is complete, or any raison why the time has not been set properly @@ -1355,7 +1356,7 @@ void TransfersDialog::insertTransfers() qlonglong fileSize = info.size; qlonglong completed = pit->transfered; // double progress = (info.size > 0)?(pit->transfered * 100.0 / info.size):0.0; - qlonglong remaining = (info.size - pit->transfered) / (pit->tfRate * 1024.0); + qlonglong remaining = (pit->tfRate>0)?((info.size - pit->transfered) / (pit->tfRate * 1024.0)):0; // Estimate the completion. We need something more accurate, meaning that we need to // transmit the completion info. From 0c711a4c61359044010d45a6cd49d2596e7bf505 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 21 Apr 2016 19:41:21 -0400 Subject: [PATCH 12/43] added missing calls to EVP_CIPHER_CTX_cleanup() to release memory after decryption, causing memory leaks --- libretroshare/src/gxs/gxssecurity.cc | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index 248ceaeac..c7c533122 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -509,7 +509,7 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u // move to end out_offset += out_currOffset; - + // make sure offset has not gone passed valid memory bounds if(out_offset > max_outlen) { @@ -521,6 +521,8 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u // free encrypted key data free(ek); + EVP_CIPHER_CTX_cleanup(&ctx); + outlen = out_offset; return true; } @@ -538,6 +540,8 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u // out = NULL ; + EVP_CIPHER_CTX ctx; + EVP_CIPHER_CTX_init(&ctx); try { @@ -561,9 +565,7 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u } } - EVP_CIPHER_CTX ctx; unsigned char iv[EVP_MAX_IV_LENGTH]; - EVP_CIPHER_CTX_init(&ctx); std::vector ek(keys.size(),NULL) ; std::vector eklen(keys.size(),0) ; @@ -653,12 +655,16 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u if(ek[i]) free(ek[i]); outlen = out_offset; + + EVP_CIPHER_CTX_cleanup(&ctx); return true; } catch(std::exception& e) { std::cerr << "(EE) Exception caught while encrypting: " << e.what() << std::endl; + EVP_CIPHER_CTX_cleanup(&ctx); + if(out) free(out) ; out = NULL ; @@ -767,6 +773,7 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in, outlen += out_currOffset; free(ek); + EVP_CIPHER_CTX_cleanup(&ctx); return true; } @@ -783,9 +790,13 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in, #ifdef DISTRIB_DEBUG std::cerr << "GxsSecurity::decrypt() " << std::endl; #endif + EVP_CIPHER_CTX ctx; + EVP_CIPHER_CTX_init(&ctx); try { + out = NULL ; + // check that the input block has a valid format. uint32_t offset = 0 ; @@ -826,8 +837,6 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in, // decrypt - EVP_CIPHER_CTX ctx; - EVP_CIPHER_CTX_init(&ctx); bool succeed = false; for(uint32_t j=0;j Date: Thu, 21 Apr 2016 19:46:47 -0400 Subject: [PATCH 13/43] fixed possible uninitialised memory read in memory decryption return --- libretroshare/src/gxs/gxssecurity.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index c7c533122..399aef192 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -425,6 +425,8 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u // [--- Encrypted session key length ---|--- Encrypted session key ---|--- IV ---|---- Encrypted data ---] // + out = NULL ; + RSA *tmpkey = ::extractPublicKey(key) ; RSA *rsa_publish_pub = RSAPublicKey_dup(tmpkey) ; RSA_free(tmpkey) ; @@ -681,6 +683,7 @@ bool GxsSecurity::decrypt(uint8_t *& out, uint32_t & outlen, const uint8_t *in, // // This method can be used to decrypt multi-encrypted data, if passing he correct encrypted key block (corresponding to the given key) + out = NULL ; #ifdef GXS_SECURITY_DEBUG std::cerr << "GxsSecurity::decrypt() " << std::endl; #endif From fba3d37126fdfaf6bab71914c14b7a07b17c8aca Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 21 Apr 2016 19:47:34 -0400 Subject: [PATCH 14/43] fixed memory leak after receiving RsNxsItem deserialised from decrypted memory --- libretroshare/src/gxs/rsgxsnetservice.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 5aeb88bee..163b2cd85 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -3876,6 +3876,7 @@ bool RsGxsNetService::decryptSingleNxsItem(const RsNxsEncryptedDataItem *encrypt if(decrypted_mem!=NULL) { ditem = RsNxsSerialiser(mServType).deserialise(decrypted_mem,&decrypted_len) ; + free(decrypted_mem) ; if(ditem != NULL) { From eca83fdd62914a45673e6a62db497c732e5e7618 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 21 Apr 2016 19:54:50 -0400 Subject: [PATCH 15/43] removed some debug info --- libretroshare/src/gxs/rsgxsnetservice.cc | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 163b2cd85..8e8d52c1a 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -1626,8 +1626,10 @@ void RsGxsNetService::recvNxsItemQueue() GXSNETDEBUG_P_(item->PeerId()) << " decrypted item " << std::endl; #endif } +#ifdef NXS_NET_DEBUG_7 else - std::cerr << "(EE) Could not decrypt incoming encrypted NXS item. Probably a friend subscribed to a circle-restricted group." << std::endl; + GXSNETDEBUG_P_(item->PeerId()) << " (EE) Could not decrypt incoming encrypted NXS item. Probably a friend subscribed to a circle-restricted group." << std::endl; +#endif } switch(ni->PacketSubType()) @@ -1638,7 +1640,8 @@ void RsGxsNetService::recvNxsItemQueue() case RS_PKT_SUBTYPE_NXS_GRP_PUBLISH_KEY_ITEM:handleRecvPublishKeys (dynamic_cast(ni)) ; break ; default: - std::cerr << "Unhandled item subtype " << (uint32_t) ni->PacketSubType() << " in RsGxsNetService: " << std::endl; break; + if(ni->PacketSubType() != RS_PKT_SUBTYPE_NXS_ENCRYPTED_DATA_ITEM) + std::cerr << "Unhandled item subtype " << (uint32_t) ni->PacketSubType() << " in RsGxsNetService: " << std::endl; break; } delete item ; } From 7aea6e5bf85b059263eb903a7d308e87cc6f4ec6 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 21 Apr 2016 22:07:10 -0400 Subject: [PATCH 16/43] removed some debug info --- libretroshare/src/gxs/rsgxsnetservice.cc | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/gxs/rsgxsnetservice.cc b/libretroshare/src/gxs/rsgxsnetservice.cc index 8e8d52c1a..2910da321 100644 --- a/libretroshare/src/gxs/rsgxsnetservice.cc +++ b/libretroshare/src/gxs/rsgxsnetservice.cc @@ -3864,7 +3864,9 @@ bool RsGxsNetService::decryptSingleNxsItem(const RsNxsEncryptedDataItem *encrypt if(!GxsSecurity::decrypt(decrypted_mem,decrypted_len, (uint8_t*)encrypted_item->encrypted_data.bin_data,encrypted_item->encrypted_data.bin_len,private_keys)) { - std::cerr << "Failed! Cannot decrypt this item." << std::endl; +#ifdef NXS_NET_DEBUG_7 + GXSNETDEBUG_P_(encrypted_item->PeerId()) << " Failed! Cannot decrypt this item." << std::endl; +#endif decrypted_mem = NULL ; // for safety return false ; } From e9fa9eb317f871602b13f27753146bfb8005d85e Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 21 Apr 2016 22:07:50 -0400 Subject: [PATCH 17/43] removed tricky memory leak in chat lobbies due to handling of partial messages --- libretroshare/src/chat/distantchat.cc | 2 ++ libretroshare/src/chat/distantchat.h | 2 +- libretroshare/src/chat/distributedchat.cc | 5 --- libretroshare/src/chat/distributedchat.h | 1 - libretroshare/src/chat/p3chatservice.cc | 42 +++++++++++++---------- libretroshare/src/chat/p3chatservice.h | 5 +-- 6 files changed, 30 insertions(+), 27 deletions(-) diff --git a/libretroshare/src/chat/distantchat.cc b/libretroshare/src/chat/distantchat.cc index 3c951bb43..805728905 100644 --- a/libretroshare/src/chat/distantchat.cc +++ b/libretroshare/src/chat/distantchat.cc @@ -268,6 +268,8 @@ bool DistantChatService::initiateDistantChatConnexion(const RsGxsId& to_gxs_id, item->PeerId(RsPeerId(tunnel_id)) ; handleRecvChatMsgItem(item) ; + delete item ; // item is replaced by NULL if partial, but this is not the case here. + return true ; } diff --git a/libretroshare/src/chat/distantchat.h b/libretroshare/src/chat/distantchat.h index 5dc21e1d6..444817c06 100644 --- a/libretroshare/src/chat/distantchat.h +++ b/libretroshare/src/chat/distantchat.h @@ -63,7 +63,7 @@ public: // derived in p3ChatService, so as to pass down some info virtual void handleIncomingItem(RsItem *) = 0; - virtual bool handleRecvChatMsgItem(RsChatMsgItem *ci)=0 ; + virtual bool handleRecvChatMsgItem(RsChatMsgItem *& ci)=0 ; bool handleOutgoingItem(RsChatItem *) ; bool handleRecvItem(RsChatItem *) ; diff --git a/libretroshare/src/chat/distributedchat.cc b/libretroshare/src/chat/distributedchat.cc index f77fb3718..88460f1f7 100644 --- a/libretroshare/src/chat/distributedchat.cc +++ b/libretroshare/src/chat/distributedchat.cc @@ -424,11 +424,6 @@ void DistributedChatService::checkSizeAndSendLobbyMessage(RsChatItem *msg) sendChatItem(msg) ; } -bool DistributedChatService::locked_checkAndRebuildPartialLobbyMessage(RsChatLobbyMsgItem *ci) -{ - return true ; -} - bool DistributedChatService::handleRecvItem(RsChatItem *item) { switch(item->PacketSubType()) diff --git a/libretroshare/src/chat/distributedchat.h b/libretroshare/src/chat/distributedchat.h index c311727b1..89160a850 100644 --- a/libretroshare/src/chat/distributedchat.h +++ b/libretroshare/src/chat/distributedchat.h @@ -90,7 +90,6 @@ class DistributedChatService void addToSaveList(std::list& list) const ; bool processLoadListItem(const RsItem *item) ; - bool locked_checkAndRebuildPartialLobbyMessage(RsChatLobbyMsgItem *) ; void checkSizeAndSendLobbyMessage(RsChatItem *) ; bool sendLobbyChat(const ChatLobbyId &lobby_id, const std::string&) ; diff --git a/libretroshare/src/chat/p3chatservice.cc b/libretroshare/src/chat/p3chatservice.cc index 3f0c87f3d..bafdb575c 100644 --- a/libretroshare/src/chat/p3chatservice.cc +++ b/libretroshare/src/chat/p3chatservice.cc @@ -428,7 +428,14 @@ bool p3ChatService::sendChat(ChatId destination, std::string msg) return true; } -bool p3ChatService::locked_checkAndRebuildPartialMessage(RsChatMsgItem *ci) +// This method might take control over the memory, or modify it, possibly adding missing parts. +// This function looks weird because it cannot duplicate the message since it does not know +// what type of object it is and the duplicate method of lobby messages is reserved for +// ChatLobby bouncing objects. +// +// Returns false if the item shouldn't be used (and replaced to NULL) + +bool p3ChatService::locked_checkAndRebuildPartialMessage(RsChatMsgItem *& ci) { // Check is the item is ending an incomplete item. // @@ -445,13 +452,16 @@ bool p3ChatService::locked_checkAndRebuildPartialMessage(RsChatMsgItem *ci) ci->message = it->second->message + ci->message ; ci->chatFlags |= it->second->chatFlags ; - + + // always remove existing partial. The compound message is in ci now. + delete it->second ; - - if(!ci_is_incomplete) - _pendingPartialMessages.erase(it) ; + _pendingPartialMessages.erase(it) ; } + // now decide what to do: if ci is incomplete, store it and replace the pointer with NULL + // if complete, return it. + if(ci_is_incomplete) { #ifdef CHAT_DEBUG @@ -459,7 +469,8 @@ bool p3ChatService::locked_checkAndRebuildPartialMessage(RsChatMsgItem *ci) #endif // The item is a partial message. Push it, and wait for the rest. // - _pendingPartialMessages[ci->PeerId()] = ci ; + _pendingPartialMessages[ci->PeerId()] = ci ; // cannot use duplicate() here + ci = NULL ; // takes memory ownership over ci return false ; } else @@ -503,8 +514,10 @@ void p3ChatService::handleIncomingItem(RsItem *item) // RsChatMsgItem *ci = dynamic_cast(item) ; if(ci != NULL) - { - if(! handleRecvChatMsgItem(ci)) + { + handleRecvChatMsgItem(ci); + + if(ci) delete ci ; return ; // don't delete! It's handled by handleRecvChatMsgItem in some specific cases only. @@ -665,7 +678,7 @@ bool p3ChatService::checkForMessageSecurity(RsChatMsgItem *ci) return true ; } -bool p3ChatService::handleRecvChatMsgItem(RsChatMsgItem *ci) +bool p3ChatService::handleRecvChatMsgItem(RsChatMsgItem *& ci) { time_t now = time(NULL); std::string name; @@ -674,15 +687,8 @@ bool p3ChatService::handleRecvChatMsgItem(RsChatMsgItem *ci) { RsStackMutex stack(mChatMtx); /********** STACK LOCKED MTX ******/ - // This crap is because chat lobby messages use a different method for chunking messages using an additional - // subpacket ID, and a list of lobbies. We cannot just collapse the two because it would make the normal chat - // (and chat lobbies) not backward compatible. - - if(!DistributedChatService::locked_checkAndRebuildPartialLobbyMessage(dynamic_cast(ci))) - return true ; - - if(!locked_checkAndRebuildPartialMessage(ci)) - return true ; + if(!locked_checkAndRebuildPartialMessage(ci)) // we make sure this call does not take control over the memory + return true ; // message is a subpart of an existing message. So everything ok, but we need to return. } // Check for security. This avoids bombing messages, and so on. diff --git a/libretroshare/src/chat/p3chatservice.h b/libretroshare/src/chat/p3chatservice.h index 4fc9e1baf..6dbdb557c 100644 --- a/libretroshare/src/chat/p3chatservice.h +++ b/libretroshare/src/chat/p3chatservice.h @@ -205,7 +205,7 @@ private: void receiveStateString(const RsPeerId& id,const std::string& s) ; /// methods for handling various Chat items. - bool handleRecvChatMsgItem(RsChatMsgItem *item) ; // returns false if the item should be deleted. + bool handleRecvChatMsgItem(RsChatMsgItem *&item) ; // NULL-ifies the item if memory ownership is taken void handleRecvChatStatusItem(RsChatStatusItem *item) ; void handleRecvChatAvatarItem(RsChatAvatarItem *item) ; @@ -220,7 +220,8 @@ private: void checkSizeAndSendMessage(RsChatMsgItem *item) ; // keep for compatibility for a few weeks. /// Called when a RsChatMsgItem is received. The item may be collapsed with any waiting partial chat item from the same peer. - bool locked_checkAndRebuildPartialMessage(RsChatMsgItem *) ; + /// if so, the chat item will be turned to NULL + bool locked_checkAndRebuildPartialMessage(RsChatMsgItem *&) ; RsChatAvatarItem *makeOwnAvatarItem() ; RsChatStatusItem *makeOwnCustomStateStringItem() ; From 5261c3cd731a18e3a5cc62a07eab5c682a88e614 Mon Sep 17 00:00:00 2001 From: csoler Date: Thu, 21 Apr 2016 22:23:07 -0400 Subject: [PATCH 18/43] prevented turtle to not deleting a config item when it is not cast --- libretroshare/src/chat/p3chatservice.h | 3 ++- libretroshare/src/turtle/p3turtle.cc | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/chat/p3chatservice.h b/libretroshare/src/chat/p3chatservice.h index 6dbdb557c..3096309b0 100644 --- a/libretroshare/src/chat/p3chatservice.h +++ b/libretroshare/src/chat/p3chatservice.h @@ -205,7 +205,8 @@ private: void receiveStateString(const RsPeerId& id,const std::string& s) ; /// methods for handling various Chat items. - bool handleRecvChatMsgItem(RsChatMsgItem *&item) ; // NULL-ifies the item if memory ownership is taken + virtual bool handleRecvChatMsgItem(RsChatMsgItem *&item) ; // NULL-ifies the item if memory ownership is taken + void handleRecvChatStatusItem(RsChatStatusItem *item) ; void handleRecvChatAvatarItem(RsChatAvatarItem *item) ; diff --git a/libretroshare/src/turtle/p3turtle.cc b/libretroshare/src/turtle/p3turtle.cc index 6d0536b0e..cd0d94238 100644 --- a/libretroshare/src/turtle/p3turtle.cc +++ b/libretroshare/src/turtle/p3turtle.cc @@ -726,7 +726,7 @@ bool p3turtle::loadList(std::list& load) } } - delete vitem ; + delete *it ; } load.clear() ; return true ; From 10230dff6eb34c8947c581f5be8d118997599aac Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 22 Apr 2016 16:50:43 -0400 Subject: [PATCH 19/43] added missing SSL shutdown when replacing existing connection --- libretroshare/src/pqi/pqissl.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/libretroshare/src/pqi/pqissl.cc b/libretroshare/src/pqi/pqissl.cc index 2ec0f7ad8..15c088627 100644 --- a/libretroshare/src/pqi/pqissl.cc +++ b/libretroshare/src/pqi/pqissl.cc @@ -1074,6 +1074,12 @@ int pqissl::Initiate_SSL_Connection() "pqissl::Initiate_SSL_Connection() SSL Connection Okay"); #endif + if(ssl_connection != NULL) + { + SSL_shutdown(ssl_connection); + SSL_free(ssl_connection) ; + } + ssl_connection = ssl; net_internal_SSL_set_fd(ssl, sockfd); From 55e80871eb4d71a709c7e3499809d73425c40197 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 22 Apr 2016 17:30:55 -0400 Subject: [PATCH 20/43] added missing free for public keys used in envelop encryption --- libretroshare/src/gxs/gxssecurity.cc | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index 399aef192..e9dfc5567 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -544,11 +544,10 @@ bool GxsSecurity::encrypt(uint8_t *& out, uint32_t &outlen, const uint8_t *in, u out = NULL ; EVP_CIPHER_CTX ctx; EVP_CIPHER_CTX_init(&ctx); + std::vector public_keys(keys.size(),NULL); try { - std::vector public_keys(keys.size(),NULL); - for(uint32_t i=0;i Date: Fri, 22 Apr 2016 17:55:15 -0400 Subject: [PATCH 21/43] fixed memory leak in SubscribeToolButton menu --- .../src/gui/common/SubscribeToolButton.cpp | 15 ++++++++++----- .../src/gui/common/SubscribeToolButton.h | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/retroshare-gui/src/gui/common/SubscribeToolButton.cpp b/retroshare-gui/src/gui/common/SubscribeToolButton.cpp index 69de79e47..d263b6f14 100644 --- a/retroshare-gui/src/gui/common/SubscribeToolButton.cpp +++ b/retroshare-gui/src/gui/common/SubscribeToolButton.cpp @@ -10,6 +10,7 @@ SubscribeToolButton::SubscribeToolButton(QWidget *parent) : { mSubscribed = false; + mMenu = NULL ; setToolButtonStyle(Qt::ToolButtonTextBesideIcon); #ifdef USE_MENUBUTTONPOPUP @@ -46,14 +47,18 @@ void SubscribeToolButton::updateUi() setIcon(QIcon(":/images/accepted16.png")); setText(tr("Subscribed")); - QMenu *menu = new QMenu; - menu->addAction(QIcon(":/images/cancel.png"), tr("Unsubscribe"), this, SLOT(unsubscribePrivate())); + if(mMenu != NULL) // that's because setMenu does not give away memory ownership + delete mMenu ; + + mMenu = new QMenu; + mMenu->addAction(QIcon(":/images/cancel.png"), tr("Unsubscribe"), this, SLOT(unsubscribePrivate())); if (!mSubscribedActions.empty()) { - menu->addSeparator(); - menu->addActions(mSubscribedActions); + mMenu->addSeparator(); + mMenu->addActions(mSubscribedActions); } - setMenu(menu); + + setMenu(mMenu); #ifndef USE_MENUBUTTONPOPUP disconnect(this, SIGNAL(clicked()), this, SLOT(subscribePrivate())); diff --git a/retroshare-gui/src/gui/common/SubscribeToolButton.h b/retroshare-gui/src/gui/common/SubscribeToolButton.h index 38056923d..a2903893c 100644 --- a/retroshare-gui/src/gui/common/SubscribeToolButton.h +++ b/retroshare-gui/src/gui/common/SubscribeToolButton.h @@ -26,6 +26,7 @@ private: private: bool mSubscribed; QList mSubscribedActions; + QMenu *mMenu ; }; #endif // SUBSCRIBETOOLBUTTON_H From acd059bdf0abe9311b77eff03c45d242a3cdced6 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 22 Apr 2016 18:49:42 -0400 Subject: [PATCH 22/43] removed potential memory leak in TransfersDialog --- retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp | 4 ++-- retroshare-gui/src/gui/FileTransfer/TransfersDialog.h | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp index ce0f7f908..96ea76bce 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.cpp @@ -219,8 +219,8 @@ TransfersDialog::TransfersDialog(QWidget *parent) // workaround for Qt bug, should be solved in next Qt release 4.7.0 // http://bugreports.qt.nokia.com/browse/QTBUG-8270 - QShortcut *Shortcut = new QShortcut(QKeySequence (Qt::Key_Delete), ui.downloadList, 0, 0, Qt::WidgetShortcut); - connect(Shortcut, SIGNAL(activated()), this, SLOT( cancel ())); + mShortcut = new QShortcut(QKeySequence (Qt::Key_Delete), ui.downloadList, 0, 0, Qt::WidgetShortcut); + connect(mShortcut, SIGNAL(activated()), this, SLOT( cancel ())); //Selection Setup selection = ui.downloadList->selectionModel(); diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h index 617974fa4..95acfde53 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h @@ -31,6 +31,7 @@ #define IMAGE_TRANSFERS ":/icons/ktorrent_128.png" +class QShortcut; class DLListDelegate; class ULListDelegate; class QStandardItemModel; @@ -271,6 +272,8 @@ public slots: qlonglong getLastDL(int row, QStandardItemModel *model); qlonglong getPath(int row, QStandardItemModel *model); QString getSources(int row, QStandardItemModel *model); + + QShortcut *mShortcut ; }; #endif From d696f725336278907c0f2f0b210d95d87252bebf Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 22 Apr 2016 20:38:07 -0400 Subject: [PATCH 23/43] fixed compilation --- retroshare-gui/src/gui/FileTransfer/TransfersDialog.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h index 95acfde53..0e6e9ad4b 100644 --- a/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h +++ b/retroshare-gui/src/gui/FileTransfer/TransfersDialog.h @@ -244,6 +244,8 @@ private: QString downloads; QString uploads; + QShortcut *mShortcut ; + /** Qt Designer generated object */ Ui::TransfersDialog ui; @@ -272,8 +274,6 @@ public slots: qlonglong getLastDL(int row, QStandardItemModel *model); qlonglong getPath(int row, QStandardItemModel *model); QString getSources(int row, QStandardItemModel *model); - - QShortcut *mShortcut ; }; #endif From 5e94c772b3e6033d37cb29d307e8df6ce0c33d1d Mon Sep 17 00:00:00 2001 From: Nyfor Date: Mon, 4 Apr 2016 12:07:09 +0200 Subject: [PATCH 24/43] Fixed compilation for Clang. --- libretroshare/src/grouter/p3grouter.h | 13 ++----------- libretroshare/src/rsserver/rsinit.cc | 4 ---- libretroshare/src/util/argstream.h | 9 ++------- retroshare-gui/src/gui/elastic/graphwidget.cpp | 12 ++++++------ retroshare-gui/src/rshare.cpp | 14 +++++++------- retroshare-nogui/src/retroshare.cc | 7 ++----- 6 files changed, 19 insertions(+), 40 deletions(-) diff --git a/libretroshare/src/grouter/p3grouter.h b/libretroshare/src/grouter/p3grouter.h index 1626da7d7..8c6fbab5c 100644 --- a/libretroshare/src/grouter/p3grouter.h +++ b/libretroshare/src/grouter/p3grouter.h @@ -225,20 +225,13 @@ private: void handleLowLevelTransactionAckItem(RsGRouterTransactionAcknItem*) ; static Sha1CheckSum computeDataItemHash(RsGRouterGenericDataItem *data_item); -#ifdef __APPLE__ -public: -#endif - class nullstream: public std::ostream {}; std::ostream& grouter_debug() const { - static nullstream null ; - + static std::ostream null(0); return _debug_enabled?(std::cerr):null; } -#ifdef __APPLE__ -private: -#endif + void routePendingObjects() ; void handleTunnels() ; void autoWash() ; @@ -364,5 +357,3 @@ private: uint64_t _random_salt ; }; - -template p3GRouter::nullstream& operator<<(p3GRouter::nullstream& ns,const T&) { return ns ; } diff --git a/libretroshare/src/rsserver/rsinit.cc b/libretroshare/src/rsserver/rsinit.cc index d064dabf9..dce5a3c97 100644 --- a/libretroshare/src/rsserver/rsinit.cc +++ b/libretroshare/src/rsserver/rsinit.cc @@ -366,11 +366,7 @@ int RsInit::InitRetroShare(int argcIgnored, char **argvIgnored, bool strictCheck #ifdef LOCALNET_TESTING >> parameter('R',"restrict-port" ,portRestrictions ,"port1-port2","Apply port restriction" ,false) #endif -#ifdef __APPLE__ >> help('h',"help","Display this Help") ; -#else - >> help() ; -#endif as.defaultErrorHandling(true) ; diff --git a/libretroshare/src/util/argstream.h b/libretroshare/src/util/argstream.h index 83fb30f0a..325d76fd5 100644 --- a/libretroshare/src/util/argstream.h +++ b/libretroshare/src/util/argstream.h @@ -140,16 +140,11 @@ namespace protected: inline OptionHolder(char s, const char* l, - const char* desc); -#ifdef __APPLE__ + const char* desc); friend OptionHolder help(char s, const char* l, const char* desc); -#else - friend OptionHolder help(char s='h', - const char* l="help", - const char* desc="Display this help"); -#endif + private: std::string shortName_; std::string longName_; diff --git a/retroshare-gui/src/gui/elastic/graphwidget.cpp b/retroshare-gui/src/gui/elastic/graphwidget.cpp index 571cdc828..dd32051da 100644 --- a/retroshare-gui/src/gui/elastic/graphwidget.cpp +++ b/retroshare-gui/src/gui/elastic/graphwidget.cpp @@ -259,7 +259,7 @@ void GraphWidget::keyPressEvent(QKeyEvent *event) } } -static void convolveWithGaussian(double *forceMap,int S,int /*s*/) +static void convolveWithGaussian(double *forceMap,unsigned int S,int /*s*/) { static double *bf = NULL ; @@ -267,8 +267,8 @@ static void convolveWithGaussian(double *forceMap,int S,int /*s*/) { bf = new double[S*S*2] ; - for(int i=0;i"; //out << trow(tcol("-"ARG_HELP) + // tcol(tr("Displays this usage message and exits."))); - out << trow(tcol("-"ARG_RESET) + + out << trow(tcol("-" ARG_RESET) + tcol(tr("Resets ALL stored RetroShare settings."))); - out << trow(tcol("-"ARG_DATADIR" <dir>") + + out << trow(tcol("-" ARG_DATADIR" <dir>") + tcol(tr("Sets the directory RetroShare uses for data files."))); - out << trow(tcol("-"ARG_LOGFILE" <file>") + + out << trow(tcol("-" ARG_LOGFILE" <file>") + tcol(tr("Sets the name and location of RetroShare's logfile."))); - out << trow(tcol("-"ARG_LOGLEVEL" <level>") + + out << trow(tcol("-" ARG_LOGLEVEL" <level>") + tcol(tr("Sets the verbosity of RetroShare's logging.") + "
[" + Log::logLevels().join("|") +"]")); - out << trow(tcol("-"ARG_GUISTYLE" <style>") + + out << trow(tcol("-" ARG_GUISTYLE" <style>") + tcol(tr("Sets RetroShare's interface style.") + "
[" + QStyleFactory::keys().join("|") + "]")); - out << trow(tcol("-"ARG_GUISTYLESHEET" <stylesheet>") + + out << trow(tcol("-" ARG_GUISTYLESHEET" <stylesheet>") + tcol(tr("Sets RetroShare's interface stylesheets."))); - out << trow(tcol("-"ARG_LANGUAGE" <language>") + + out << trow(tcol("-" ARG_LANGUAGE" <language>") + tcol(tr("Sets RetroShare's language.") + "
[" + LanguageSupport::languageCodes().join("|") + "]")); out << ""; diff --git a/retroshare-nogui/src/retroshare.cc b/retroshare-nogui/src/retroshare.cc index 59a1bf1b4..7ad45b0eb 100644 --- a/retroshare-nogui/src/retroshare.cc +++ b/retroshare-nogui/src/retroshare.cc @@ -85,12 +85,9 @@ int main(int argc, char **argv) args >> parameter("docroot", docroot, "path", "Serve static files from this path.", false); // unfinished //args >> parameter("http-listen", listenAddress, "ipv6 address", "Listen only on the specified address.", false); - args >> option("http-allow-all", allowAllIps, "allow connections from all IP adresses (default= localhost only)"); -#ifdef __APPLE__ + args >> option("http-allow-all", allowAllIps, "allow connections from all IP adresses (default= localhost only)"); args >> help('h',"help","Display this Help"); -#else - args >> help(); -#endif + if (args.helpRequested()) { std::cerr << args.usage() << std::endl; From cad8c89746ae3e7c37f060986f54f9ca7763f429 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 23 Apr 2016 17:10:25 -0400 Subject: [PATCH 25/43] added packet slicing in pqistreamer --- libretroshare/src/pqi/pqiqosstreamer.cc | 3 +- libretroshare/src/pqi/pqiqosstreamer.h | 2 +- libretroshare/src/pqi/pqistreamer.cc | 166 +++++++++++++++++++----- libretroshare/src/pqi/pqistreamer.h | 10 +- 4 files changed, 144 insertions(+), 37 deletions(-) diff --git a/libretroshare/src/pqi/pqiqosstreamer.cc b/libretroshare/src/pqi/pqiqosstreamer.cc index 5470b55b8..c18388707 100644 --- a/libretroshare/src/pqi/pqiqosstreamer.cc +++ b/libretroshare/src/pqi/pqiqosstreamer.cc @@ -50,9 +50,8 @@ int pqiQoSstreamer::getQueueSize(bool in) // return pqiQoS::gatherStatistics(per_service_count,per_priority_count) ; //} -void pqiQoSstreamer::locked_storeInOutputQueue(void *ptr,int priority) +void pqiQoSstreamer::locked_storeInOutputQueue(void *ptr,int size,int priority) { - uint32_t size = getRsItemSize(ptr) ; _total_item_size += size ; ++_total_item_count ; diff --git a/libretroshare/src/pqi/pqiqosstreamer.h b/libretroshare/src/pqi/pqiqosstreamer.h index da5cba923..5ea3df65e 100644 --- a/libretroshare/src/pqi/pqiqosstreamer.h +++ b/libretroshare/src/pqi/pqiqosstreamer.h @@ -36,7 +36,7 @@ class pqiQoSstreamer: public pqithreadstreamer, public pqiQoS static const uint32_t PQI_QOS_STREAMER_MAX_LEVELS = 10 ; static const float PQI_QOS_STREAMER_ALPHA ; - virtual void locked_storeInOutputQueue(void *ptr,int priority) ; + virtual void locked_storeInOutputQueue(void *ptr, int size, int priority) ; virtual int locked_out_queue_size() const { return _total_item_count ; } virtual void locked_clear_out_queue() ; virtual int locked_compute_out_pkt_size() const { return _total_item_size ; } diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 37bdbe065..504cf14cb 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -39,9 +39,16 @@ const int pqistreamerzone = 8221; -static const int PQISTREAM_ABS_MAX = 100000000; /* 100 MB/sec (actually per loop) */ -static const int PQISTREAM_AVG_PERIOD = 5; // update speed estimate every 5 seconds -static const float PQISTREAM_AVG_FRAC = 0.8; // for bandpass filter over speed estimate. +static const int PQISTREAM_ABS_MAX = 100000000; /* 100 MB/sec (actually per loop) */ +static const int PQISTREAM_AVG_PERIOD = 5; // update speed estimate every 5 seconds +static const float PQISTREAM_AVG_FRAC = 0.8; // for bandpass filter over speed estimate. +static const int PQISTREAM_OPTIMAL_PACKET_SIZE = 512; // It is believed that this value should be lower than TCP slices and large enough as compare to encryption padding. + // most importantly, it should be constant, so as to allow correct QoS. +static const int PQISTREAM_OPTIMAL_SLICE_OFFSET_UNIT = 16 ; // slices offset in units of 16 bits. That allows bigger numbers encoded in 4 less bits. +static const int PQISTREAM_SLICE_FLAG_ENDS = 0x01; // these flags should be kept in the range 0x01-0x08 +static const int PQISTREAM_SLICE_FLAG_STARTS = 0x02; // +static const int PQISTREAM_SLICE_PROTOCOL_VERSION_ID = 0x10; // Protocol version ID. Should hold on the 4 lower bits. +static const int PQISTREAM_PARTIAL_PACKET_HEADER_SIZE = 8; // Same size than normal header, to make the code simpler. /* This removes the print statements (which hammer pqidebug) */ /*** @@ -279,7 +286,7 @@ int pqistreamer::status() return 0; } -void pqistreamer::locked_storeInOutputQueue(void *ptr,int) +void pqistreamer::locked_storeInOutputQueue(void *ptr,int,int) { mOutPkts.push_back(ptr); } @@ -317,7 +324,7 @@ int pqistreamer::queue_outpqi_locked(RsItem *pqi,uint32_t& pktsize) if (mRsSerialiser->serialise(pqi, ptr, &pktsize)) { - locked_storeInOutputQueue(ptr,pqi->priority_level()) ; + locked_storeInOutputQueue(ptr,pktsize,pqi->priority_level()) ; if (!(mBio_flags & BIN_FLAGS_NO_DELETE)) { @@ -404,7 +411,7 @@ time_t pqistreamer::getLastIncomingTS() // // Old : 02 0014 03 00000026 [data, 26 bytes] => [version 1B] [service 2B][subpacket 1B] [size 4B] // New1: fv 0014 03 xxxxx sss [data, sss bytes] => [flags 0.5B version 0.5B] [service 2B][subpacket 1B] [packet counter 2.5B size 1.5B] -// New2: f xxxxxx ooo sssss [data, sss bytes] => [flags 0.5B] [2^24 packet count] [2^16 offset (in units of 16)] [size] +// New2: fx xxxxxx oooo ssss [data, sss bytes] => [flags 0.5B] [2^28 packet count] [2^16 offset (in units of 16)] [size 2^16] // // Flags: 0x1 => incomplete packet continued after // Flags: 0x2 => packet ending a previously incomplete packet @@ -481,29 +488,83 @@ int pqistreamer::handleoutgoing_locked() return 0; } -#define OPTIMAL_PACKET_SIZE 512 - // send a out_pkt., else send out_data. unless - // there is a pending packet. - if (!mPkt_wpending) - { - void *dta; - mPkt_wpending_size = 0 ; - int k=0; + // send a out_pkt., else send out_data. unless there is a pending packet. The strategy is to + // - grab as many packets as possible while below the optimal packet size, so as to allow some packing and decrease encryption padding overhead (suposeddly) + // - limit packets size to OPTIMAL_PACKET_SIZE when sending big packets so as to keep as much QoS as possible. - while(mPkt_wpending_size < (uint32_t)maxbytes && mPkt_wpending_size < OPTIMAL_PACKET_SIZE && (dta = locked_pop_out_data())!=NULL ) - { - uint32_t s = getRsItemSize(dta); - mPkt_wpending = realloc(mPkt_wpending,s+mPkt_wpending_size) ; - memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size],dta,s) ; + if (!mPkt_wpending) + { + void *dta; + mPkt_wpending_size = 0 ; + int k=0; + + uint32_t slice_offset =0 ; + uint32_t slice_size=0; + bool slice_starts=true ; + bool slice_ends=true ; + uint32_t slice_packet_id=0 ; + + do + { + dta = locked_pop_out_data(PQISTREAM_OPTIMAL_PACKET_SIZE,PQISTREAM_OPTIMAL_SLICE_OFFSET_UNIT,slice_offset,slice_size,slice_starts,slice_ends,slice_packet_id) ; + + if(!dta) + break ; + + if(slice_size > 0xffff) + { + std::cerr << "(EE) protocol error in pqitreamer: slice size is too large and cannot be encoded." ; + free(mPkt_wpending) ; + mPkt_wpending_size = 0; + } + + if(slice_offset > 0xfffff || (slice_offset & 0xff)!=0) // 5 f, on purpose. Not a bug. + { + std::cerr << "(EE) protocol error in pqitreamer: slice size is too large and cannot be encoded." ; + free(mPkt_wpending) ; + mPkt_wpending_size = 0; + } + + if(slice_starts && slice_ends) // good old method. Send the packet as is, since it's a full packet. + { + mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size) ; + memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size],dta,slice_size) ; free(dta); - mPkt_wpending_size += s ; + mPkt_wpending_size += slice_size ; ++k ; - } + } + else // partial packet. We make a special header for it and insert it in the stream + { + mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE) ; + memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE],dta,slice_size) ; + free(dta); + + // New2: fp xxxxxx oooo ssss [data, sss bytes] => [flags 0.5B] [protocol version 0.5B] [2^24 packet count] [2^16 offset (in units of 16)] [size 2^16] + + uint8_t partial_flags = PQISTREAM_SLICE_PROTOCOL_VERSION_ID ; // includes version. Flags are in the first half-byte + if(slice_starts) partial_flags |= PQISTREAM_SLICE_FLAG_STARTS ; + if(slice_ends ) partial_flags |= PQISTREAM_SLICE_FLAG_ENDS ; + + ((char*)mPkt_wpending)[mPkt_wpending_size+0x00] = partial_flags ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x01] = uint8_t(slice_packet_id >> 16) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x02] = uint8_t(slice_packet_id >> 8) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x03] = uint8_t(slice_packet_id >> 0) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x04] = uint8_t(slice_offset >> 12) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x05] = uint8_t(slice_offset >> 4) & 0xff ; // not a bug. The last 4 bits are discarded because they are always 0 + ((char*)mPkt_wpending)[mPkt_wpending_size+0x06] = uint8_t(slice_size >> 8) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x07] = uint8_t(slice_size >> 0) & 0xff ; + + mPkt_wpending_size += slice_size + PQISTREAM_PARTIAL_PACKET_HEADER_SIZE; + ++k ; + } + } + while(mPkt_wpending_size < (uint32_t)maxbytes && mPkt_wpending_size < PQISTREAM_OPTIMAL_PACKET_SIZE ) ; + #ifdef DEBUG_PQISTREAMER - if(k > 1) - std::cerr << "Packed " << k << " packets into " << mPkt_wpending_size << " bytes." << std::endl; + if(k > 1) + std::cerr << "Packed " << k << " packets into " << mPkt_wpending_size << " bytes." << std::endl; #endif - } + } if (mPkt_wpending) { @@ -581,7 +642,7 @@ int pqistreamer::handleincoming_locked() void *block = mPkt_rpending; // initial read size: basic packet. - int blen = getRsPktBaseSize(); + int blen = getRsPktBaseSize(); // this is valid for both packet slices and normal un-sliced packets (same header size) int maxin = inAllowedBytes_locked(); @@ -657,7 +718,23 @@ start_packet_read: continue_packet: { // workout how much more to read. - int extralen = getRsItemSize(block) - blen; + + bool is_partial_packet = false ; + + int extralen =0; + int slice_offset = 0 ; + int slice_packet_id =0; + + if( ((char*)block)[0] == 0x10 || ((char*)block)[0] == 0x11 || ((char*)block)[0] == 0x12) + { + extralen = (int(((char*)block)[6]) << 8) + (int(((char*)block)[7])); + slice_offset = (int(((char*)block)[5]) << 4) + (int(((char*)block)[4]) << 12); + slice_packet_id = (int(((char*)block)[3]) << 0) + (int(((char*)block)[2]) << 8) + (int(((char*)block)[1]) << 16); + + is_partial_packet = true ; + } + else + extralen = getRsItemSize(block) - blen; #ifdef DEBUG_PQISTREAMER std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet getRsItemSize(block) = " << getRsItemSize(block) << std::endl ; @@ -822,9 +899,19 @@ continue_packet: #ifdef DEBUG_PQISTREAMER std::cerr << "[" << (void*)pthread_self() << "] " << "deserializing. Size=" << pktlen << std::endl ; #endif + RsItem *pkt ; + + if(is_partial_packet) + { + std::cerr << "Inputing partial packet " << RsUtil::BinToHex((char*)block,8) << std::endl; + + pkt = addPartialPacket(block,extralen,slice_offset,slice_packet_id) ; + } + else + pkt = mRsSerialiser->deserialise(block, &pktlen); + std::cerr << "Got packet with header " << RsUtil::BinToHex((char*)block,8) << std::endl; - RsItem *pkt = mRsSerialiser->deserialise(block, &pktlen); if ((pkt != NULL) && (0 < handleincomingitem_locked(pkt,pktlen))) { @@ -833,7 +920,7 @@ continue_packet: #endif inReadBytes_locked(pktlen); // only count deserialised packets, because that's what is actually been transfered. } - else + else if (!is_partial_packet) { #ifdef DEBUG_PQISTREAMER pqioutput(PQL_ALERT, pqistreamerzone, "Failed to handle Packet!"); @@ -864,6 +951,19 @@ continue_packet: return 0; } +RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t slice_offset,uint32_t slice_packet_id) +{ + std::map::iterator it = mPartialPackets.find(slice_packet_id) ; + + if(it == mPartialPackets.end()) + { + // make sure we really have starting packet. Otherwise this is an error. + } + else + { + // make sure this is a continuing packet, otherwise this is an error. + } +} /* BandWidth Management Assistance */ @@ -1094,8 +1194,14 @@ int pqistreamer::locked_gatherStatistics(std::list& out_lst,std:: return 1 ; } -void *pqistreamer::locked_pop_out_data() +void *pqistreamer::locked_pop_out_data(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) { + offset = 0 ; + size = 0 ; + starts = true ; + ends = true ; + packet_id = 0 ; + void *res = NULL ; if (!mOutPkts.empty()) @@ -1109,3 +1215,5 @@ void *pqistreamer::locked_pop_out_data() } return res ; } + + diff --git a/libretroshare/src/pqi/pqistreamer.h b/libretroshare/src/pqi/pqistreamer.h index ff1cf0a12..47c8540df 100644 --- a/libretroshare/src/pqi/pqistreamer.h +++ b/libretroshare/src/pqi/pqistreamer.h @@ -71,13 +71,12 @@ class pqistreamer: public PQInterface // These methods are redefined in pqiQoSstreamer // - virtual void locked_storeInOutputQueue(void *ptr,int priority) ; + virtual void locked_storeInOutputQueue(void *ptr, int size, int priority) ; virtual int locked_out_queue_size() const ; virtual void locked_clear_out_queue() ; virtual int locked_compute_out_pkt_size() const ; - virtual void *locked_pop_out_data() ; - //virtual int locked_gatherStatistics(std::vector& per_service_count,std::vector& per_priority_count) const; // extracting data. - virtual int locked_gatherStatistics(std::list& outqueue_stats,std::list& inqueue_stats); // extracting data. + virtual void *locked_pop_out_data(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id); + virtual int locked_gatherStatistics(std::list& outqueue_stats,std::list& inqueue_stats); // extracting data. void updateRates() ; @@ -156,7 +155,8 @@ class pqistreamer: public PQInterface std::list mCurrentStatsChunk_Out ; time_t mStatisticsTimeStamp ; - void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); + void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); + RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_offset, uint32_t slice_packet_id); }; #endif //MRK_PQI_STREAMER_HEADER From 0d84a5585f0e19721a28bf446616b81651c8cf1b Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 24 Apr 2016 11:29:21 -0400 Subject: [PATCH 26/43] updated ubuntu changelog --- build_scripts/Debian+Ubuntu/changelog | 35 +++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/build_scripts/Debian+Ubuntu/changelog b/build_scripts/Debian+Ubuntu/changelog index d92ea17ce..8669f307d 100644 --- a/build_scripts/Debian+Ubuntu/changelog +++ b/build_scripts/Debian+Ubuntu/changelog @@ -1,5 +1,40 @@ retroshare06 (0.6.0-1.XXXXXX~YYYYYY) YYYYYY; urgency=low + 0dc43c3 csoler Sat, 23 Apr 2016 08:29:53 -0400 Merge pull request #338 from Nyfor/master + 5e94c77 Nyfor Mon, 4 Apr 2016 12:07:09 +0200 Fixed compilation for Clang. + d696f72 csoler Fri, 22 Apr 2016 20:38:07 -0400 fixed compilation + acd059b csoler Fri, 22 Apr 2016 18:49:42 -0400 removed potential memory leak in TransfersDialog + 52cf66a csoler Fri, 22 Apr 2016 17:55:15 -0400 fixed memory leak in SubscribeToolButton menu + 55e8087 csoler Fri, 22 Apr 2016 17:30:55 -0400 added missing free for public keys used in envelop encryption + 10230df csoler Fri, 22 Apr 2016 16:50:43 -0400 added missing SSL shutdown when replacing existing connection + 5261c3c csoler Thu, 21 Apr 2016 22:23:07 -0400 prevented turtle to not deleting a config item when it is not cast + e9fa9eb csoler Thu, 21 Apr 2016 22:07:50 -0400 removed tricky memory leak in chat lobbies due to handling of partial messages + 7aea6e5 csoler Thu, 21 Apr 2016 22:07:10 -0400 removed some debug info + eca83fd csoler Thu, 21 Apr 2016 19:54:50 -0400 removed some debug info + fba3d37 csoler Thu, 21 Apr 2016 19:47:34 -0400 fixed memory leak after receiving RsNxsItem deserialised from decrypted memory + 0d3ff0e csoler Thu, 21 Apr 2016 19:46:47 -0400 fixed possible uninitialised memory read in memory decryption return + 0c711a4 csoler Thu, 21 Apr 2016 19:41:21 -0400 added missing calls to EVP_CIPHER_CTX_cleanup() to release memory after decryption, causing memory + 3fae108 csoler Thu, 21 Apr 2016 12:58:48 -0400 removed div by zero in transfers dialog + 0572492 csoler Wed, 20 Apr 2016 21:20:47 -0400 severely reduced packet grouping, which limited the effectiveness of QoS + d6ae71e csoler Wed, 20 Apr 2016 20:21:29 -0400 removed pointer to QStringList in QHash, causing memory loss + aba3d2f csoler Wed, 20 Apr 2016 18:24:02 -0400 removed memory leak due to zeroing (on purpose!) a data chunk in GRouter before deleting it + d017771 csoler Wed, 20 Apr 2016 18:08:26 -0400 fixed missing sendTime in distant chat, causing inconsistent display in GUI + 80a9fec csoler Tue, 19 Apr 2016 22:18:25 -0400 Merge pull request #353 from crhode/master + 39553a6 csoler Tue, 19 Apr 2016 21:58:30 -0400 removed debug info + 5645e44 Manuel Davis Tue, 19 Apr 2016 11:32:56 -0500 Change regular expression(s) for identifying hotlinks in feral text. + 8be53dd Manuel Davis Tue, 19 Apr 2016 11:11:58 -0500 Improve inserting placemark in ChatLobby. + 26af7c9 csoler Sun, 17 Apr 2016 22:58:48 -0400 changed heart beat limit to a larger value. Apparently too tight a value causes disconnections due + c14c240 csoler Sun, 17 Apr 2016 00:51:45 -0400 fixed update of circles GUI using a timer. This is not optimal, and should be replaced by a proper + 3000f94 csoler Sat, 16 Apr 2016 22:44:06 -0400 separated subscribe status from wether we are part of a circle or not. Still needs update of GUI + b861aa9 csoler Sat, 16 Apr 2016 17:10:36 -0400 Merge pull request #352 from AsamK/trailing_semicolon + e7ec204 AsamK Sat, 16 Apr 2016 20:40:24 +0200 Add trailing semi-colon to MimeType in .desktop files + fa8a585 csoler Fri, 15 Apr 2016 18:25:53 -0400 removed some debug info + d642934 csoler Fri, 15 Apr 2016 18:25:41 -0400 increased frequency of update for banned PGP nodes + + -- Cyril Soler Sun, 24 Apr 2016 12:00:00 -0500 + +retroshare06 (0.6.0-1.20160415.26574fd9~trusty) trusty; urgency=low + 2552b55 defnax Fri, 15 Apr 2016 20:32:00 +0200 Merge branch 'master' of https://github.com/RetroShare/RetroShare 85942f4 defnax Fri, 15 Apr 2016 20:30:32 +0200 improving the create circle layout cb6c2c9 csoler Thu, 14 Apr 2016 23:49:55 -0400 Merge pull request #350 from csoler/v0.6-Circles From a2c11f97c2f98a0ea20f8ab2e51c2d7a3e9de0a9 Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 24 Apr 2016 13:43:34 -0400 Subject: [PATCH 27/43] added packet reconstruction and deserialising --- libretroshare/src/pqi/pqistreamer.cc | 59 +++++++++++++++++++++++++--- libretroshare/src/pqi/pqistreamer.h | 10 ++++- 2 files changed, 62 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 504cf14cb..e1bac3245 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -719,7 +719,9 @@ continue_packet: { // workout how much more to read. - bool is_partial_packet = false ; + bool is_partial_packet = false ; + bool is_packet_starting = (((char*)block)[0] == 0x11) ; + bool is_packet_ending = (((char*)block)[0] == 0x12) ; int extralen =0; int slice_offset = 0 ; @@ -905,7 +907,7 @@ continue_packet: { std::cerr << "Inputing partial packet " << RsUtil::BinToHex((char*)block,8) << std::endl; - pkt = addPartialPacket(block,extralen,slice_offset,slice_packet_id) ; + pkt = addPartialPacket(block,extralen,slice_offset,slice_packet_id,is_packet_starting,is_packet_ending) ; } else pkt = mRsSerialiser->deserialise(block, &pktlen); @@ -951,17 +953,62 @@ continue_packet: return 0; } -RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t slice_offset,uint32_t slice_packet_id) +RsItem *pqistreamer::addPartialPacket(void *block,uint32_t len,uint32_t slice_offset,uint32_t slice_packet_id,bool is_packet_starting,bool is_packet_ending) { + if(is_packet_starting && is_packet_ending) + { + std::cerr << "(EE) unexpected situation. Got in addPartialPacket() a full packet both starting and ending" << std::endl; + free(block) ; + return NULL ; + } + std::map::iterator it = mPartialPackets.find(slice_packet_id) ; - + if(it == mPartialPackets.end()) { - // make sure we really have starting packet. Otherwise this is an error. + // make sure we really have a starting packet. Otherwise this is an error. + + if(!is_packet_starting) + { + std::cerr << "(EE) dropping non starting packet that has no record." << std::endl; + free(block) ; + return NULL ; + } + + mPartialPackets[slice_packet_id].mem = block ; + mPartialPackets[slice_packet_id].size = len ; + + return NULL ; // no need to check for ending } else { - // make sure this is a continuing packet, otherwise this is an error. + if(is_packet_starting) + { + std::cerr << "(WW) dropping unfinished existing packet that gets to be replaced by new starting packet." << std::endl; + free(it->second.mem) ; + it->second.mem = block ; + it->second.size = len ; + + return NULL ; + } + // make sure this is a continuing packet, otherwise this is an error. + + it->second.mem = realloc(it->second.mem, it->second.size + len) ; + memcpy( &((char*)it->second.mem)[it->second.size],block,len) ; + it->second.size += len ; + free(block) ; + + if(is_packet_ending) + { + RsItem *item = mRsSerialiser->deserialise(it->second.mem, &it->second.size); + + free(it->second.mem) ; + mPartialPackets.erase(it) ; + + return item ; + } + else + return NULL ; } } diff --git a/libretroshare/src/pqi/pqistreamer.h b/libretroshare/src/pqi/pqistreamer.h index 47c8540df..79c5fbefc 100644 --- a/libretroshare/src/pqi/pqistreamer.h +++ b/libretroshare/src/pqi/pqistreamer.h @@ -40,6 +40,12 @@ // The interface does not handle connection, just communication. // possible bioflags: BIN_FLAGS_NO_CLOSE | BIN_FLAGS_NO_DELETE +struct PartialPacketRecord +{ + void *mem ; + uint32_t size ; +}; + class pqistreamer: public PQInterface { public: @@ -156,7 +162,9 @@ class pqistreamer: public PQInterface time_t mStatisticsTimeStamp ; void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); - RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_offset, uint32_t slice_packet_id); + RsItem *addPartialPacket(void *block, uint32_t len, uint32_t slice_offset, uint32_t slice_packet_id,bool packet_starting,bool packet_ending); + + std::map mPartialPackets ; }; #endif //MRK_PQI_STREAMER_HEADER From f62957534edb9c20c63f326389473a622497cf3b Mon Sep 17 00:00:00 2001 From: csoler Date: Sun, 24 Apr 2016 21:18:44 -0400 Subject: [PATCH 28/43] fixed a few bugs in packet slicing in pqiQoS --- libretroshare/src/pqi/pqiqos.cc | 5 +- libretroshare/src/pqi/pqiqos.h | 240 +++++++++++++-------------- libretroshare/src/pqi/pqistreamer.cc | 79 +++++---- libretroshare/src/pqi/pqistreamer.h | 2 +- 4 files changed, 173 insertions(+), 153 deletions(-) diff --git a/libretroshare/src/pqi/pqiqos.cc b/libretroshare/src/pqi/pqiqos.cc index 1453f559b..a8678ca8c 100644 --- a/libretroshare/src/pqi/pqiqos.cc +++ b/libretroshare/src/pqi/pqiqos.cc @@ -6,7 +6,7 @@ #include "pqiqos.h" -static const uint32_t MAX_COUNTER_VALUE = 1024u*1024u ; // 2^20 +const uint32_t pqiQoS::MAX_PACKET_COUNTER_VALUE = (1 << 24) ; pqiQoS::pqiQoS(uint32_t nb_levels,float alpha) : _item_queues(nb_levels),_alpha(alpha) @@ -16,6 +16,7 @@ pqiQoS::pqiQoS(uint32_t nb_levels,float alpha) float c = 1.0f ; float inc = alpha ; _nb_items = 0 ; + _id_counter = 0 ; for(int i=((int)nb_levels)-1;i>=0;--i,c *= alpha) { @@ -57,7 +58,7 @@ void pqiQoS::in_rsItem(void *ptr,int size,int priority) _item_queues[priority].push(ptr,size,_id_counter++) ; ++_nb_items ; - if(_id_counter >= MAX_COUNTER_VALUE) + if(_id_counter >= MAX_PACKET_COUNTER_VALUE) _id_counter = 0 ; } diff --git a/libretroshare/src/pqi/pqiqos.h b/libretroshare/src/pqi/pqiqos.h index 438f45622..9a4bfbb15 100644 --- a/libretroshare/src/pqi/pqiqos.h +++ b/libretroshare/src/pqi/pqiqos.h @@ -45,141 +45,141 @@ class pqiQoS { +public: + pqiQoS(uint32_t max_levels,float alpha) ; + + struct ItemRecord + { + void *data ; + uint32_t current_offset ; + uint32_t size ; + uint32_t id ; + }; + + class ItemQueue + { public: - pqiQoS(uint32_t max_levels,float alpha) ; - - struct ItemRecord - { - void *data ; - uint32_t current_offset ; - uint32_t size ; - uint32_t id ; - }; - - class ItemQueue + ItemQueue() { - public: - ItemQueue() - { - _item_count =0 ; - } - void *pop() - { - if(_items.empty()) - return NULL ; + _item_count =0 ; + } + void *pop() + { + if(_items.empty()) + return NULL ; - void *item = _items.front().data ; + void *item = _items.front().data ; + _items.pop_front() ; + --_item_count ; + + return item ; + } + + void *slice(uint32_t max_size,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) + { + if(_items.empty()) + return NULL ; + + ItemRecord& rec(_items.front()) ; + packet_id = rec.id ; + + // readily get rid of the item if it can be sent as a whole + + if(rec.current_offset == 0 && rec.size < max_size) + { + offset = 0 ; + starts = true ; + ends = true ; + size = rec.size ; + + return pop() ; + } + starts = (rec.current_offset == 0) ; + ends = (rec.current_offset + max_size >= rec.size) ; + + if(rec.size <= rec.current_offset) + { + std::cerr << "(EE) severe error in slicing in QoS." << std::endl; + pop() ; + return NULL ; + } + + size = std::min(max_size, uint32_t((int)rec.size - (int)rec.current_offset)) ; + void *mem = rs_malloc(size) ; + + if(!mem) + { + std::cerr << "(EE) memory allocation error in QoS." << std::endl; + pop() ; + return NULL ; + } + + memcpy(mem,&((unsigned char*)rec.data)[rec.current_offset],size) ; + + if(ends) // we're taking the whole stuff. So we can delete the entry. + { + free(rec.data) ; _items.pop_front() ; - --_item_count ; - - return item ; } - - void *slice(uint32_t max_size,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) - { - if(_items.empty()) - return NULL ; - - ItemRecord& rec(_items.front()) ; - packet_id = rec.id ; - - // readily get rid of the item if it can be sent as a whole - - if(rec.current_offset == 0 && rec.size < max_size) - { - offset = 0 ; - starts = true ; - ends = true ; - size = rec.size ; - - return pop() ; - } - starts = (rec.current_offset == 0) ; - ends = (rec.current_offset + max_size > rec.size) ; - - if(rec.size < rec.current_offset) - { - std::cerr << "(EE) severe error in slicing in QoS." << std::endl; - pop() ; - return NULL ; - } - - size = std::min(max_size, uint32_t((int)rec.size - (int)rec.current_offset)) ; - void *mem = rs_malloc(size) ; - - if(!mem) - { - std::cerr << "(EE) memory allocation error in QoS." << std::endl; - pop() ; - return NULL ; - } - - memcpy(mem,&((unsigned char*)rec.data)[rec.current_offset],size) ; - - if(ends) // we're taking the whole stuff. So we can delete the entry. - { - free(rec.data) ; - _items.pop_front() ; - } - else - rec.current_offset += size ; - - return mem ; - } + else + rec.current_offset += size ; // by construction, !ends implies rec.current_offset < rec.size - void push(void *item,uint32_t size,uint32_t id) - { - ItemRecord rec ; - - rec.data = item ; - rec.current_offset = 0 ; - rec.size = size ; - rec.id = id ; - - _items.push_back(rec) ; - ++_item_count ; - } + return mem ; + } - uint32_t size() const { return _item_count ; } + void push(void *item,uint32_t size,uint32_t id) + { + ItemRecord rec ; - float _threshold ; - float _counter ; - float _inc ; - uint32_t _item_count ; - - std::list _items ; - }; + rec.data = item ; + rec.current_offset = 0 ; + rec.size = size ; + rec.id = id ; - // This function pops items from the queue, y order of priority - // - void *out_rsItem() ; - void *out_rsItem(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) ; + _items.push_back(rec) ; + } - // This function is used to queue items. - // - void in_rsItem(void *item, int size, int priority) ; + uint32_t size() const { return _item_count ; } - void print() const ; - uint64_t qos_queue_size() const { return _nb_items ; } + float _threshold ; + float _counter ; + float _inc ; + uint32_t _item_count ; - // kills all waiting items. - void clear() ; + std::list _items ; + }; - // get some stats about what's going on. service_packets will contain the number of - // packets per service, and queue_sizes will contain the size of the different priority queues. + // This function pops items from the queue, y order of priority + // + void *out_rsItem() ; + void *out_rsItem(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) ; - //int gatherStatistics(std::vector& per_service_count,std::vector& per_priority_count) const ; + // This function is used to queue items. + // + void in_rsItem(void *item, int size, int priority) ; - void computeTotalItemSize() const ; - int debug_computeTotalItemSize() const ; - private: - // This vector stores the lists of items with equal priorities. - // - std::vector _item_queues ; - float _alpha ; - uint64_t _nb_items ; - - uint32_t _id_counter ; + void print() const ; + uint64_t qos_queue_size() const { return _nb_items ; } + + // kills all waiting items. + void clear() ; + + // get some stats about what's going on. service_packets will contain the number of + // packets per service, and queue_sizes will contain the size of the different priority queues. + + //int gatherStatistics(std::vector& per_service_count,std::vector& per_priority_count) const ; + + void computeTotalItemSize() const ; + int debug_computeTotalItemSize() const ; +private: + // This vector stores the lists of items with equal priorities. + // + std::vector _item_queues ; + float _alpha ; + uint64_t _nb_items ; + uint32_t _id_counter ; + + static const uint32_t MAX_PACKET_COUNTER_VALUE ; }; diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index e1bac3245..1922cb823 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -45,8 +45,8 @@ static const float PQISTREAM_AVG_FRAC = 0.8; // for bandpass filter over static const int PQISTREAM_OPTIMAL_PACKET_SIZE = 512; // It is believed that this value should be lower than TCP slices and large enough as compare to encryption padding. // most importantly, it should be constant, so as to allow correct QoS. static const int PQISTREAM_OPTIMAL_SLICE_OFFSET_UNIT = 16 ; // slices offset in units of 16 bits. That allows bigger numbers encoded in 4 less bits. -static const int PQISTREAM_SLICE_FLAG_ENDS = 0x01; // these flags should be kept in the range 0x01-0x08 -static const int PQISTREAM_SLICE_FLAG_STARTS = 0x02; // +static const int PQISTREAM_SLICE_FLAG_STARTS = 0x01; // +static const int PQISTREAM_SLICE_FLAG_ENDS = 0x02; // these flags should be kept in the range 0x01-0x08 static const int PQISTREAM_SLICE_PROTOCOL_VERSION_ID = 0x10; // Protocol version ID. Should hold on the 4 lower bits. static const int PQISTREAM_PARTIAL_PACKET_HEADER_SIZE = 8; // Same size than normal header, to make the code simpler. @@ -527,6 +527,7 @@ int pqistreamer::handleoutgoing_locked() if(slice_starts && slice_ends) // good old method. Send the packet as is, since it's a full packet. { + std::cerr << "sending full slice, old style" << std::endl; mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size) ; memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size],dta,slice_size) ; free(dta); @@ -535,6 +536,8 @@ int pqistreamer::handleoutgoing_locked() } else // partial packet. We make a special header for it and insert it in the stream { + std::cerr << "sending partial slice, packet ID=" << std::hex << slice_packet_id << std::dec << ", offset=" << slice_offset << ", size=" << slice_size << std::endl; + mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE) ; memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE],dta,slice_size) ; free(dta); @@ -723,16 +726,17 @@ continue_packet: bool is_packet_starting = (((char*)block)[0] == 0x11) ; bool is_packet_ending = (((char*)block)[0] == 0x12) ; - int extralen =0; - int slice_offset = 0 ; - int slice_packet_id =0; + uint32_t extralen =0; + uint32_t slice_offset = 0 ; + uint32_t slice_packet_id =0; if( ((char*)block)[0] == 0x10 || ((char*)block)[0] == 0x11 || ((char*)block)[0] == 0x12) { - extralen = (int(((char*)block)[6]) << 8) + (int(((char*)block)[7])); - slice_offset = (int(((char*)block)[5]) << 4) + (int(((char*)block)[4]) << 12); - slice_packet_id = (int(((char*)block)[3]) << 0) + (int(((char*)block)[2]) << 8) + (int(((char*)block)[1]) << 16); + extralen = (uint32_t(((uint8_t*)block)[6]) << 8) + (uint32_t(((uint8_t*)block)[7])); + slice_offset = (uint32_t(((uint8_t*)block)[5]) << 4) + (uint32_t(((uint8_t*)block)[4]) << 12); + slice_packet_id = (uint32_t(((uint8_t*)block)[3]) << 0) + (uint32_t(((uint8_t*)block)[2]) << 8) + (uint32_t(((uint8_t*)block)[1]) << 16); + std::cerr << "Reading from mem block " << RsUtil::BinToHex((char*)block,8) << ": packet_id=" << slice_packet_id << ", len=" << extralen << ", offset=" << slice_offset << std::endl; is_partial_packet = true ; } else @@ -902,19 +906,17 @@ continue_packet: std::cerr << "[" << (void*)pthread_self() << "] " << "deserializing. Size=" << pktlen << std::endl ; #endif RsItem *pkt ; + std::cerr << "Got packet with header " << RsUtil::BinToHex((char*)block,8) << std::endl; if(is_partial_packet) { std::cerr << "Inputing partial packet " << RsUtil::BinToHex((char*)block,8) << std::endl; - pkt = addPartialPacket(block,extralen,slice_offset,slice_packet_id,is_packet_starting,is_packet_ending) ; + pkt = addPartialPacket(block,pktlen,slice_offset,slice_packet_id,is_packet_starting,is_packet_ending) ; } else pkt = mRsSerialiser->deserialise(block, &pktlen); - - - std::cerr << "Got packet with header " << RsUtil::BinToHex((char*)block,8) << std::endl; - + if ((pkt != NULL) && (0 < handleincomingitem_locked(pkt,pktlen))) { #ifdef DEBUG_PQISTREAMER @@ -953,15 +955,19 @@ continue_packet: return 0; } -RsItem *pqistreamer::addPartialPacket(void *block,uint32_t len,uint32_t slice_offset,uint32_t slice_packet_id,bool is_packet_starting,bool is_packet_ending) +RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t slice_offset,uint32_t slice_packet_id,bool is_packet_starting,bool is_packet_ending) { + std::cerr << "Receiving partial packet. size=" << len << ", offset=" << slice_offset << ". ID=" << std::hex << slice_packet_id << std::dec << ", starting:" << is_packet_starting << ", ending:" << is_packet_ending ; + if(is_packet_starting && is_packet_ending) { std::cerr << "(EE) unexpected situation. Got in addPartialPacket() a full packet both starting and ending" << std::endl; - free(block) ; return NULL ; } + uint32_t slice_length = len - PQISTREAM_PARTIAL_PACKET_HEADER_SIZE ; + unsigned char *slice_data = &((unsigned char*)block)[PQISTREAM_PARTIAL_PACKET_HEADER_SIZE] ; + std::map::iterator it = mPartialPackets.find(slice_packet_id) ; if(it == mPartialPackets.end()) @@ -971,44 +977,57 @@ RsItem *pqistreamer::addPartialPacket(void *block,uint32_t len,uint32_t slice_of if(!is_packet_starting) { std::cerr << "(EE) dropping non starting packet that has no record." << std::endl; - free(block) ; + return NULL ; + } + PartialPacketRecord& rec = mPartialPackets[slice_packet_id] ; + + rec.mem = rs_malloc(slice_length) ; + + if(!rec.mem) + { + std::cerr << "(EE) Cannot allowcate memory for slice of size " << slice_length << std::endl; return NULL ; } - mPartialPackets[slice_packet_id].mem = block ; - mPartialPackets[slice_packet_id].size = len ; + memcpy(rec.mem, slice_data, slice_length) ; ; + rec.size = slice_length ; + + std::cerr << " => stored in new record." << std::endl; return NULL ; // no need to check for ending } else { + PartialPacketRecord& rec = it->second ; + if(is_packet_starting) { std::cerr << "(WW) dropping unfinished existing packet that gets to be replaced by new starting packet." << std::endl; - free(it->second.mem) ; - it->second.mem = block ; - it->second.size = len ; - - return NULL ; + free(rec.mem); + rec.size = 0 ; } // make sure this is a continuing packet, otherwise this is an error. - it->second.mem = realloc(it->second.mem, it->second.size + len) ; - memcpy( &((char*)it->second.mem)[it->second.size],block,len) ; - it->second.size += len ; - free(block) ; + rec.mem = realloc(rec.mem, rec.size + slice_length) ; + memcpy( &((char*)rec.mem)[rec.size],slice_data,slice_length) ; + rec.size += slice_length ; + + std::cerr << " => added to existing record " ; if(is_packet_ending) { - RsItem *item = mRsSerialiser->deserialise(it->second.mem, &it->second.size); + std::cerr << " => deserialising: mem=" << RsUtil::BinToHex((char*)rec.mem,std::min(20u,rec.size)) << std::endl; + RsItem *item = mRsSerialiser->deserialise(rec.mem, &rec.size); - free(it->second.mem) ; + free(rec.mem) ; mPartialPackets.erase(it) ; - return item ; } else + { + std::cerr << std::endl; return NULL ; + } } } diff --git a/libretroshare/src/pqi/pqistreamer.h b/libretroshare/src/pqi/pqistreamer.h index 79c5fbefc..27badc004 100644 --- a/libretroshare/src/pqi/pqistreamer.h +++ b/libretroshare/src/pqi/pqistreamer.h @@ -162,7 +162,7 @@ class pqistreamer: public PQInterface time_t mStatisticsTimeStamp ; void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); - RsItem *addPartialPacket(void *block, uint32_t len, uint32_t slice_offset, uint32_t slice_packet_id,bool packet_starting,bool packet_ending); + RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_offset, uint32_t slice_packet_id,bool packet_starting,bool packet_ending); std::map mPartialPackets ; }; From dd81ce3bf37d038ee3d4a00761793b140c6dd5eb Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 25 Apr 2016 22:50:41 -0400 Subject: [PATCH 29/43] using BinToHex to display mem blocks in pqistreamer debug --- libretroshare/src/pqi/pqistreamer.cc | 496 +++++++++++++-------------- 1 file changed, 234 insertions(+), 262 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index c04c2ea9a..42b00bda3 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -624,335 +624,307 @@ int pqistreamer::handleoutgoing_locked() */ int pqistreamer::handleincoming_locked() { - int readbytes = 0; - static const int max_failed_read_attempts = 2000 ; + int readbytes = 0; + static const int max_failed_read_attempts = 2000 ; #ifdef DEBUG_PQISTREAMER - pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincoming_locked()"); + pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::handleincoming_locked()"); #endif - if(!(mBio->isactive())) - { - mReading_state = reading_state_initial ; - free_rpend_locked(); - return 0; - } + if(!(mBio->isactive())) + { + mReading_state = reading_state_initial ; + free_rpend_locked(); + return 0; + } else - allocate_rpend_locked(); + allocate_rpend_locked(); - // enough space to read any packet. - int maxlen = mPkt_rpend_size; - void *block = mPkt_rpending; + // enough space to read any packet. + int maxlen = mPkt_rpend_size; + void *block = mPkt_rpending; - // initial read size: basic packet. - int blen = getRsPktBaseSize(); // this is valid for both packet slices and normal un-sliced packets (same header size) + // initial read size: basic packet. + int blen = getRsPktBaseSize(); // this is valid for both packet slices and normal un-sliced packets (same header size) - int maxin = inAllowedBytes_locked(); + int maxin = inAllowedBytes_locked(); #ifdef DEBUG_PQISTREAMER - std::cerr << "[" << (void*)pthread_self() << "] " << "reading state = " << mReading_state << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "reading state = " << mReading_state << std::endl ; #endif - switch(mReading_state) - { - case reading_state_initial: /*std::cerr << "jumping to start" << std::endl; */ goto start_packet_read ; - case reading_state_packet_started: /*std::cerr << "jumping to middle" << std::endl;*/ goto continue_packet ; - } + switch(mReading_state) + { + case reading_state_initial: /*std::cerr << "jumping to start" << std::endl; */ goto start_packet_read ; + case reading_state_packet_started: /*std::cerr << "jumping to middle" << std::endl;*/ goto continue_packet ; + } start_packet_read: - { // scope to ensure variable visibility - // read the basic block (minimum packet size) - int tmplen; + { // scope to ensure variable visibility + // read the basic block (minimum packet size) + int tmplen; #ifdef DEBUG_PQISTREAMER - std::cerr << "[" << (void*)pthread_self() << "] " << "starting packet" << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "starting packet" << std::endl ; #endif - memset(block,0,blen) ; // reset the block, to avoid uninitialized memory reads. + memset(block,0,blen) ; // reset the block, to avoid uninitialized memory reads. - if (blen != (tmplen = mBio->readdata(block, blen))) - { - pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, "pqistreamer::handleincoming() Didn't read BasePkt!"); + if (blen != (tmplen = mBio->readdata(block, blen))) + { + pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, "pqistreamer::handleincoming() Didn't read BasePkt!"); - // error.... (either blocked or failure) - if (tmplen == 0) - { + // error.... (either blocked or failure) + if (tmplen == 0) + { #ifdef DEBUG_PQISTREAMER - // most likely blocked! - pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, "pqistreamer::handleincoming() read blocked"); - std::cerr << "[" << (void*)pthread_self() << "] " << "given up 1" << std::endl ; + // most likely blocked! + pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, "pqistreamer::handleincoming() read blocked"); + std::cerr << "[" << (void*)pthread_self() << "] " << "given up 1" << std::endl ; #endif - return 0; - } - else if (tmplen < 0) - { - // Most likely it is that the packet is pending but could not be read by pqissl because of stream flow. - // So we return without an error, and leave the machine state in 'start_read'. - // - //pqioutput(PQL_WARNING, pqistreamerzone, "pqistreamer::handleincoming() Error in bio read"); + return 0; + } + else if (tmplen < 0) + { + // Most likely it is that the packet is pending but could not be read by pqissl because of stream flow. + // So we return without an error, and leave the machine state in 'start_read'. + // + //pqioutput(PQL_WARNING, pqistreamerzone, "pqistreamer::handleincoming() Error in bio read"); #ifdef DEBUG_PQISTREAMER - std::cerr << "[" << (void*)pthread_self() << "] " << "given up 2, state = " << mReading_state << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "given up 2, state = " << mReading_state << std::endl ; #endif - return 0; - } - else // tmplen > 0 - { - // strange case....This should never happen as partial reads are handled by pqissl below. + return 0; + } + else // tmplen > 0 + { + // strange case....This should never happen as partial reads are handled by pqissl below. #ifdef DEBUG_PQISTREAMER - std::string out = "pqistreamer::handleincoming() Incomplete "; - rs_sprintf_append(out, "(Strange) read of %d bytes", tmplen); - pqioutput(PQL_ALERT, pqistreamerzone, out); + std::string out = "pqistreamer::handleincoming() Incomplete "; + rs_sprintf_append(out, "(Strange) read of %d bytes", tmplen); + pqioutput(PQL_ALERT, pqistreamerzone, out); - std::cerr << "[" << (void*)pthread_self() << "] " << "given up 3" << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "given up 3" << std::endl ; #endif - return -1; - } - } + return -1; + } + } #ifdef DEBUG_PQISTREAMER - std::cerr << "[" << (void*)pthread_self() << "] " << "block 0 : " << (int)(((unsigned char*)block)[0]) << " " << (int)(((unsigned char*)block)[1]) << " " << (int)(((unsigned char*)block)[2]) << " " - << (int)(((unsigned char*)block)[3]) << " " - << (int)(((unsigned char*)block)[4]) << " " - << (int)(((unsigned char*)block)[5]) << " " - << (int)(((unsigned char*)block)[6]) << " " - << (int)(((unsigned char*)block)[7]) << " " << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "block 0 : " << RsUtil::BinToHex(block,8) << std::endl; #endif - readbytes += blen; - mReading_state = reading_state_packet_started ; - mFailed_read_attempts = 0 ; // reset failed read, as the packet has been totally read. - } + readbytes += blen; + mReading_state = reading_state_packet_started ; + mFailed_read_attempts = 0 ; // reset failed read, as the packet has been totally read. + } continue_packet: - { - // workout how much more to read. - - bool is_partial_packet = false ; - bool is_packet_starting = (((char*)block)[0] == 0x11) ; - bool is_packet_ending = (((char*)block)[0] == 0x12) ; - - uint32_t extralen =0; - uint32_t slice_offset = 0 ; - uint32_t slice_packet_id =0; - - if( ((char*)block)[0] == 0x10 || ((char*)block)[0] == 0x11 || ((char*)block)[0] == 0x12) - { - extralen = (uint32_t(((uint8_t*)block)[6]) << 8) + (uint32_t(((uint8_t*)block)[7])); - slice_offset = (uint32_t(((uint8_t*)block)[5]) << 4) + (uint32_t(((uint8_t*)block)[4]) << 12); - slice_packet_id = (uint32_t(((uint8_t*)block)[3]) << 0) + (uint32_t(((uint8_t*)block)[2]) << 8) + (uint32_t(((uint8_t*)block)[1]) << 16); - - std::cerr << "Reading from mem block " << RsUtil::BinToHex((char*)block,8) << ": packet_id=" << slice_packet_id << ", len=" << extralen << ", offset=" << slice_offset << std::endl; - is_partial_packet = true ; - } - else - extralen = getRsItemSize(block) - blen; + { + // workout how much more to read. + + bool is_partial_packet = false ; + bool is_packet_starting = (((char*)block)[0] == 0x11) ; + bool is_packet_ending = (((char*)block)[0] == 0x12) ; + + uint32_t extralen =0; + uint32_t slice_offset = 0 ; + uint32_t slice_packet_id =0; + + if( ((char*)block)[0] == 0x10 || ((char*)block)[0] == 0x11 || ((char*)block)[0] == 0x12) + { + extralen = (uint32_t(((uint8_t*)block)[6]) << 8) + (uint32_t(((uint8_t*)block)[7])); + slice_offset = (uint32_t(((uint8_t*)block)[5]) << 4) + (uint32_t(((uint8_t*)block)[4]) << 12); + slice_packet_id = (uint32_t(((uint8_t*)block)[3]) << 0) + (uint32_t(((uint8_t*)block)[2]) << 8) + (uint32_t(((uint8_t*)block)[1]) << 16); + + std::cerr << "Reading from mem block " << RsUtil::BinToHex((char*)block,8) << ": packet_id=" << slice_packet_id << ", len=" << extralen << ", offset=" << slice_offset << std::endl; + is_partial_packet = true ; + } + else + extralen = getRsItemSize(block) - blen; #ifdef DEBUG_PQISTREAMER - std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet getRsItemSize(block) = " << getRsItemSize(block) << std::endl ; - std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet extralen = " << extralen << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet getRsItemSize(block) = " << getRsItemSize(block) << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet extralen = " << extralen << std::endl ; - std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet state=" << mReading_state << std::endl ; - std::cerr << "[" << (void*)pthread_self() << "] " << "block 1 : " << (int)(((unsigned char*)block)[0]) << " " << (int)(((unsigned char*)block)[1]) << " " << (int)(((unsigned char*)block)[2]) << " " << (int)(((unsigned char*)block)[3]) << " " - << (int)(((unsigned char*)block)[4]) << " " - << (int)(((unsigned char*)block)[5]) << " " - << (int)(((unsigned char*)block)[6]) << " " - << (int)(((unsigned char*)block)[7]) << " " << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet state=" << mReading_state << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "block 1 : " << RsUtil::BinToHex(block,8) << std::endl; #endif - if (extralen > maxlen - blen) - { - pqioutput(PQL_ALERT, pqistreamerzone, "ERROR: Read Packet too Big!"); + if (extralen > maxlen - blen) + { + pqioutput(PQL_ALERT, pqistreamerzone, "ERROR: Read Packet too Big!"); - p3Notify *notify = RsServer::notify(); - if (notify) - { - std::string title = - "Warning: Bad Packet Read"; + p3Notify *notify = RsServer::notify(); + if (notify) + { + std::string title = + "Warning: Bad Packet Read"; - std::string msg; - msg = " **** WARNING **** \n"; - msg += "Retroshare has caught a BAD Packet Read"; - msg += "\n"; - msg += "This is normally caused by connecting to an"; - msg += " OLD version of Retroshare"; - msg += "\n"; - rs_sprintf_append(msg, "(M:%d B:%d E:%d)\n", maxlen, blen, extralen); - msg += "\n"; - rs_sprintf_append(msg, "block = %d %d %d %d %d %d %d %d\n", - (int)(((unsigned char*)block)[0]), - (int)(((unsigned char*)block)[1]), - (int)(((unsigned char*)block)[2]), - (int)(((unsigned char*)block)[3]), - (int)(((unsigned char*)block)[4]), - (int)(((unsigned char*)block)[5]), - (int)(((unsigned char*)block)[6]), - (int)(((unsigned char*)block)[7])) ; - msg += "\n"; - msg += "Please get your friends to upgrade to the latest version"; - msg += "\n"; - msg += "\n"; - msg += "If you are sure the error was not caused by an old version"; - msg += "\n"; - msg += "Please report the problem to Retroshare's developers"; - msg += "\n"; + std::string msg; + msg = " **** WARNING **** \n"; + msg += "Retroshare has caught a BAD Packet Read"; + msg += "\n"; + msg += "This is normally caused by connecting to an"; + msg += " OLD version of Retroshare"; + msg += "\n"; + rs_sprintf_append(msg, "(M:%d B:%d E:%d)\n", maxlen, blen, extralen); + msg += "\n"; + msg += "block = " ; + msg += RsUtil::BinToHex((char*)block,8); - notify->AddLogMessage(0, RS_SYS_WARNING, title, msg); + msg += "\n"; + msg += "Please get your friends to upgrade to the latest version"; + msg += "\n"; + msg += "\n"; + msg += "If you are sure the error was not caused by an old version"; + msg += "\n"; + msg += "Please report the problem to Retroshare's developers"; + msg += "\n"; - std::cerr << "pqistreamer::handle_incoming() ERROR: Read Packet too Big" << std::endl; - std::cerr << msg; - std::cerr << std::endl; + notify->AddLogMessage(0, RS_SYS_WARNING, title, msg); - } - mBio->close(); - mReading_state = reading_state_initial ; // restart at state 1. - mFailed_read_attempts = 0 ; - return -1; + std::cerr << "pqistreamer::handle_incoming() ERROR: Read Packet too Big" << std::endl; + std::cerr << msg; + std::cerr << std::endl; - // Used to exit now! exit(1); - } + } + mBio->close(); + mReading_state = reading_state_initial ; // restart at state 1. + mFailed_read_attempts = 0 ; + return -1; - if (extralen > 0) - { - void *extradata = (void *) (((char *) block) + blen); - int tmplen ; + // Used to exit now! exit(1); + } - // Don't reset the block now! If pqissl is in the middle of a multiple-chunk - // packet (larger than 16384 bytes), and pqistreamer jumped directly yo - // continue_packet:, then readdata is going to write after the beginning of - // extradata, yet not exactly at start -> the start of the packet would be wiped out. - // - // so, don't do that: - // memset( extradata,0,extralen ) ; + if (extralen > 0) + { + void *extradata = (void *) (((char *) block) + blen); + int tmplen ; - if (extralen != (tmplen = mBio->readdata(extradata, extralen))) - { + // Don't reset the block now! If pqissl is in the middle of a multiple-chunk + // packet (larger than 16384 bytes), and pqistreamer jumped directly yo + // continue_packet:, then readdata is going to write after the beginning of + // extradata, yet not exactly at start -> the start of the packet would be wiped out. + // + // so, don't do that: + // memset( extradata,0,extralen ) ; + + if (extralen != (tmplen = mBio->readdata(extradata, extralen))) + { #ifdef DEBUG_PQISTREAMER - if(tmplen > 0) - std::cerr << "[" << (void*)pthread_self() << "] " << "Incomplete packet read ! This is a real problem ;-)" << std::endl ; + if(tmplen > 0) + std::cerr << "[" << (void*)pthread_self() << "] " << "Incomplete packet read ! This is a real problem ;-)" << std::endl ; #endif - if(++mFailed_read_attempts > max_failed_read_attempts) - { - std::string out; - rs_sprintf(out, "Error Completing Read (read %d/%d)", tmplen, extralen); - std::cerr << out << std::endl ; - pqioutput(PQL_ALERT, pqistreamerzone, out); + if(++mFailed_read_attempts > max_failed_read_attempts) + { + std::string out; + rs_sprintf(out, "Error Completing Read (read %d/%d)", tmplen, extralen); + std::cerr << out << std::endl ; + pqioutput(PQL_ALERT, pqistreamerzone, out); - p3Notify *notify = RsServer::notify(); - if (notify) - { - std::string title = "Warning: Error Completing Read"; + p3Notify *notify = RsServer::notify(); + if (notify) + { + std::string title = "Warning: Error Completing Read"; - std::string msgout; - msgout = " **** WARNING **** \n"; - msgout += "Retroshare has experienced an unexpected Read ERROR"; - msgout += "\n"; - rs_sprintf_append(msgout, "(M:%d B:%d E:%d R:%d)\n", maxlen, blen, extralen, tmplen); - msgout += "\n"; - msgout += "Note: this error might as well happen (rarely) when a peer disconnects in between a transmission of a large packet.\n"; - msgout += "If it happens manny time, please contact the developers, and send them these numbers:"; - msgout += "\n"; + std::string msgout; + msgout = " **** WARNING **** \n"; + msgout += "Retroshare has experienced an unexpected Read ERROR"; + msgout += "\n"; + rs_sprintf_append(msgout, "(M:%d B:%d E:%d R:%d)\n", maxlen, blen, extralen, tmplen); + msgout += "\n"; + msgout += "Note: this error might as well happen (rarely) when a peer disconnects in between a transmission of a large packet.\n"; + msgout += "If it happens manny time, please contact the developers, and send them these numbers:"; + msgout += "\n"; - rs_sprintf_append(msgout, "block = %d %d %d %d %d %d %d %d\n", - (int)(((unsigned char*)block)[0]), - (int)(((unsigned char*)block)[1]), - (int)(((unsigned char*)block)[2]), - (int)(((unsigned char*)block)[3]), - (int)(((unsigned char*)block)[4]), - (int)(((unsigned char*)block)[5]), - (int)(((unsigned char*)block)[6]), - (int)(((unsigned char*)block)[7])); + msgout += "block = " ; + msgout += RsUtil::BinToHex((char*)block,8) + "\n" ; - //notify->AddSysMessage(0, RS_SYS_WARNING, title, msgout.str()); + std::cerr << msgout << std::endl; + } - std::cerr << msgout << std::endl; - } - - mBio->close(); - mReading_state = reading_state_initial ; // restart at state 1. - mFailed_read_attempts = 0 ; - return -1; - } - else - { + mBio->close(); + mReading_state = reading_state_initial ; // restart at state 1. + mFailed_read_attempts = 0 ; + return -1; + } + else + { #ifdef DEBUG_PQISTREAMER - std::cerr << "[" << (void*)pthread_self() << "] " << "given up 5, state = " << mReading_state << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "given up 5, state = " << mReading_state << std::endl ; #endif - return 0 ; // this is just a SSL_WANT_READ error. Don't panic, we'll re-try the read soon. - // we assume readdata() returned either -1 or the complete read size. - } - } + return 0 ; // this is just a SSL_WANT_READ error. Don't panic, we'll re-try the read soon. + // we assume readdata() returned either -1 or the complete read size. + } + } #ifdef DEBUG_PQISTREAMER - std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet state=" << mReading_state << std::endl ; - std::cerr << "[" << (void*)pthread_self() << "] " << "block 2 : " << (int)(((unsigned char*)extradata)[0]) << " " << (int)(((unsigned char*)extradata)[1]) << " " << (int)(((unsigned char*)extradata)[2]) << " " << (int)(((unsigned char*)extradata)[3]) << " " - << (int)(((unsigned char*)extradata)[4]) << " " - << (int)(((unsigned char*)extradata)[5]) << " " - << (int)(((unsigned char*)extradata)[6]) << " " - << (int)(((unsigned char*)extradata)[7]) << " " << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet state=" << mReading_state << std::endl ; + std::cerr << "[" << (void*)pthread_self() << "] " << "block 2 : " << RsUtil::BinToHex(extradata,8) << std::endl; #endif - mFailed_read_attempts = 0 ; - readbytes += extralen; - } + mFailed_read_attempts = 0 ; + readbytes += extralen; + } - // create packet, based on header. + // create packet, based on header. #ifdef DEBUG_PQISTREAMER - { - std::string out; - rs_sprintf(out, "Read Data Block -> Incoming Pkt(%d)", blen + extralen); - //std::cerr << out ; - pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, out); - } + { + std::string out; + rs_sprintf(out, "Read Data Block -> Incoming Pkt(%d)", blen + extralen); + //std::cerr << out ; + pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, out); + } #endif - // std::cerr << "Deserializing packet of size " << pktlen <deserialise(block, &pktlen); - - if ((pkt != NULL) && (0 < handleincomingitem_locked(pkt,pktlen))) - { -#ifdef DEBUG_PQISTREAMER - pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, "Successfully Read a Packet!"); -#endif - inReadBytes_locked(pktlen); // only count deserialised packets, because that's what is actually been transfered. - } - else if (!is_partial_packet) - { -#ifdef DEBUG_PQISTREAMER - pqioutput(PQL_ALERT, pqistreamerzone, "Failed to handle Packet!"); -#endif - std::cerr << "Incoming Packet could not be deserialised:" << std::endl; - std::cerr << " Incoming peer id: " << PeerId() << std::endl; - if(pktlen >= 8) - std::cerr << " Packet header : " << RsUtil::BinToHex((unsigned char*)block,8) << std::endl; - if(pktlen > 8) - std::cerr << " Packet data : " << RsUtil::BinToHex((unsigned char*)block+8,std::min(50u,pktlen-8)) << ((pktlen>58)?"...":"") << std::endl; - } + RsItem *pkt ; + std::cerr << "Got packet with header " << RsUtil::BinToHex((char*)block,8) << std::endl; - mReading_state = reading_state_initial ; // restart at state 1. - mFailed_read_attempts = 0 ; // reset failed read, as the packet has been totally read. - } + if(is_partial_packet) + { + std::cerr << "Inputing partial packet " << RsUtil::BinToHex((char*)block,8) << std::endl; - if(maxin > readbytes && mBio->moretoread(0)) - goto start_packet_read ; + pkt = addPartialPacket(block,pktlen,slice_offset,slice_packet_id,is_packet_starting,is_packet_ending) ; + } + else + pkt = mRsSerialiser->deserialise(block, &pktlen); + + if ((pkt != NULL) && (0 < handleincomingitem_locked(pkt,pktlen))) + { +#ifdef DEBUG_PQISTREAMER + pqioutput(PQL_DEBUG_BASIC, pqistreamerzone, "Successfully Read a Packet!"); +#endif + inReadBytes_locked(pktlen); // only count deserialised packets, because that's what is actually been transfered. + } + else if (!is_partial_packet) + { +#ifdef DEBUG_PQISTREAMER + pqioutput(PQL_ALERT, pqistreamerzone, "Failed to handle Packet!"); +#endif + std::cerr << "Incoming Packet could not be deserialised:" << std::endl; + std::cerr << " Incoming peer id: " << PeerId() << std::endl; + if(pktlen >= 8) + std::cerr << " Packet header : " << RsUtil::BinToHex((unsigned char*)block,8) << std::endl; + if(pktlen > 8) + std::cerr << " Packet data : " << RsUtil::BinToHex((unsigned char*)block+8,std::min(50u,pktlen-8)) << ((pktlen>58)?"...":"") << std::endl; + } + + mReading_state = reading_state_initial ; // restart at state 1. + mFailed_read_attempts = 0 ; // reset failed read, as the packet has been totally read. + } + + if(maxin > readbytes && mBio->moretoread(0)) + goto start_packet_read ; #ifdef DEBUG_TRANSFERS - if (readbytes >= maxin) - { - std::cerr << "pqistreamer::handleincoming() Stopped reading as readbytes >= maxin. Read " << readbytes << " bytes "; - std::cerr << std::endl; - } + if (readbytes >= maxin) + { + std::cerr << "pqistreamer::handleincoming() Stopped reading as readbytes >= maxin. Read " << readbytes << " bytes "; + std::cerr << std::endl; + } #endif - return 0; + return 0; } RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t slice_offset,uint32_t slice_packet_id,bool is_packet_starting,bool is_packet_ending) From 3b685851d2863f960766ac2f2180a38885e27433 Mon Sep 17 00:00:00 2001 From: csoler Date: Mon, 25 Apr 2016 23:37:02 -0400 Subject: [PATCH 30/43] improved/simplified slicing protocol --- libretroshare/src/pqi/pqiqos.cc | 22 +--- libretroshare/src/pqi/pqiqos.h | 6 +- libretroshare/src/pqi/pqiqosstreamer.cc | 8 +- libretroshare/src/pqi/pqiqosstreamer.h | 2 +- libretroshare/src/pqi/pqistreamer.cc | 147 +++++++++++------------- libretroshare/src/pqi/pqistreamer.h | 4 +- 6 files changed, 80 insertions(+), 109 deletions(-) diff --git a/libretroshare/src/pqi/pqiqos.cc b/libretroshare/src/pqi/pqiqos.cc index a8678ca8c..b2472c0b7 100644 --- a/libretroshare/src/pqi/pqiqos.cc +++ b/libretroshare/src/pqi/pqiqos.cc @@ -86,20 +86,7 @@ void pqiQoS::in_rsItem(void *ptr,int size,int priority) // } -void *pqiQoS::out_rsItem() -{ - bool starts,ends ; - uint32_t packet_id,offset,size ; - - void *res = out_rsItem(~0u,16,offset,size,starts,ends,packet_id) ; - - if(!starts || !ends) - std::cerr << "(EE) protocol error in pqiQoS. Will eventually kill connection!" << std::endl; - - return res ; -} - -void *pqiQoS::out_rsItem(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) +void *pqiQoS::out_rsItem(uint32_t max_slice_size, uint32_t& size, bool& starts, bool& ends, uint32_t& packet_id) { // Go through the queues. Increment counters. @@ -127,16 +114,11 @@ void *pqiQoS::out_rsItem(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& // now chop a slice of this item - void *res = _item_queues[last].slice(max_slice_size,offset,size,starts,ends,packet_id) ; + void *res = _item_queues[last].slice(max_slice_size,size,starts,ends,packet_id) ; if(ends) --_nb_items ; - if( (offset % offset_unit) != 0) - std::cerr << "(EE) Severe error in pqiQoS::out_rsItem(). offset unit inconsistent with calculated offset." << std::endl; - - offset /= offset_unit ; - return res ; } else diff --git a/libretroshare/src/pqi/pqiqos.h b/libretroshare/src/pqi/pqiqos.h index 9a4bfbb15..a83ec1445 100644 --- a/libretroshare/src/pqi/pqiqos.h +++ b/libretroshare/src/pqi/pqiqos.h @@ -75,7 +75,7 @@ public: return item ; } - void *slice(uint32_t max_size,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) + void *slice(uint32_t max_size,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) { if(_items.empty()) return NULL ; @@ -87,7 +87,6 @@ public: if(rec.current_offset == 0 && rec.size < max_size) { - offset = 0 ; starts = true ; ends = true ; size = rec.size ; @@ -151,8 +150,7 @@ public: // This function pops items from the queue, y order of priority // - void *out_rsItem() ; - void *out_rsItem(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) ; + void *out_rsItem(uint32_t max_slice_size,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) ; // This function is used to queue items. // diff --git a/libretroshare/src/pqi/pqiqosstreamer.cc b/libretroshare/src/pqi/pqiqosstreamer.cc index c18388707..c9cc7d570 100644 --- a/libretroshare/src/pqi/pqiqosstreamer.cc +++ b/libretroshare/src/pqi/pqiqosstreamer.cc @@ -65,14 +65,16 @@ void pqiQoSstreamer::locked_clear_out_queue() _total_item_count = 0 ; } -void *pqiQoSstreamer::locked_pop_out_data(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) +void *pqiQoSstreamer::locked_pop_out_data(uint32_t max_slice_size, uint32_t& size, bool& starts, bool& ends, uint32_t& packet_id) { - void *out = pqiQoS::out_rsItem(max_slice_size,offset_unit,offset,size,starts,ends,packet_id) ; + void *out = pqiQoS::out_rsItem(max_slice_size,size,starts,ends,packet_id) ; if(out != NULL) { _total_item_size -= getRsItemSize(out) ; - --_total_item_count ; + + if(ends) + --_total_item_count ; } return out ; diff --git a/libretroshare/src/pqi/pqiqosstreamer.h b/libretroshare/src/pqi/pqiqosstreamer.h index 5ea3df65e..0144cbdb8 100644 --- a/libretroshare/src/pqi/pqiqosstreamer.h +++ b/libretroshare/src/pqi/pqiqosstreamer.h @@ -40,7 +40,7 @@ class pqiQoSstreamer: public pqithreadstreamer, public pqiQoS virtual int locked_out_queue_size() const { return _total_item_count ; } virtual void locked_clear_out_queue() ; virtual int locked_compute_out_pkt_size() const { return _total_item_size ; } - virtual void *locked_pop_out_data(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id); + virtual void *locked_pop_out_data(uint32_t max_slice_size,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id); //virtual int locked_gatherStatistics(std::vector& per_service_count,std::vector& per_priority_count) const; // extracting data. diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 42b00bda3..3f8f711b0 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -44,10 +44,9 @@ static const int PQISTREAM_AVG_PERIOD = 5; // update speed estimate every static const float PQISTREAM_AVG_FRAC = 0.8; // for bandpass filter over speed estimate. static const int PQISTREAM_OPTIMAL_PACKET_SIZE = 512; // It is believed that this value should be lower than TCP slices and large enough as compare to encryption padding. // most importantly, it should be constant, so as to allow correct QoS. -static const int PQISTREAM_OPTIMAL_SLICE_OFFSET_UNIT = 16 ; // slices offset in units of 16 bits. That allows bigger numbers encoded in 4 less bits. static const int PQISTREAM_SLICE_FLAG_STARTS = 0x01; // static const int PQISTREAM_SLICE_FLAG_ENDS = 0x02; // these flags should be kept in the range 0x01-0x08 -static const int PQISTREAM_SLICE_PROTOCOL_VERSION_ID = 0x10; // Protocol version ID. Should hold on the 4 lower bits. +static const int PQISTREAM_SLICE_PROTOCOL_VERSION_ID_01 = 0x10; // Protocol version ID. Should hold on the 4 lower bits. static const int PQISTREAM_PARTIAL_PACKET_HEADER_SIZE = 8; // Same size than normal header, to make the code simpler. /* This removes the print statements (which hammer pqidebug) */ @@ -411,7 +410,7 @@ time_t pqistreamer::getLastIncomingTS() // // Old : 02 0014 03 00000026 [data, 26 bytes] => [version 1B] [service 2B][subpacket 1B] [size 4B] // New1: fv 0014 03 xxxxx sss [data, sss bytes] => [flags 0.5B version 0.5B] [service 2B][subpacket 1B] [packet counter 2.5B size 1.5B] -// New2: fx xxxxxx oooo ssss [data, sss bytes] => [flags 0.5B] [2^28 packet count] [2^16 offset (in units of 16)] [size 2^16] +// New2: pp ff xxxxxxxx ssss [data, sss bytes] => [flags 1B] [protocol version 1B] [2^32 packet count] [2^16 size] // // Flags: 0x1 => incomplete packet continued after // Flags: 0x2 => packet ending a previously incomplete packet @@ -424,9 +423,9 @@ time_t pqistreamer::getLastIncomingTS() // - Encode length on 1.5 Bytes (10 bits) => max slice size = 1024 // - Encode packet ID on 2.5 Bytes (20 bits) => packet counter = [0...1056364] // Mode 2: -// - Encode flags on 0.5 Bytes ( 4 bits) -// - Encode packet ID on 3.5 Bytes (28 bits) => packet counter = [0...16777216] -// - Encode offset on 2.0 Bytes (16 bits) => 65536 * 16 = // ax packet size = 1048576 +// - Encode protocol on 1.0 Bytes ( 8 bits) +// - Encode flags on 1.0 Bytes ( 8 bits) +// - Encode packet ID on 4.0 Bytes (32 bits) => packet counter = [0...2^32] // - Encode size on 2.0 Bytes (16 bits) => 65536 // max slice size = 65536 // // - limit packet grouping to max size 1024. @@ -498,7 +497,6 @@ int pqistreamer::handleoutgoing_locked() mPkt_wpending_size = 0 ; int k=0; - uint32_t slice_offset =0 ; uint32_t slice_size=0; bool slice_starts=true ; bool slice_ends=true ; @@ -506,60 +504,53 @@ int pqistreamer::handleoutgoing_locked() do { - dta = locked_pop_out_data(PQISTREAM_OPTIMAL_PACKET_SIZE,PQISTREAM_OPTIMAL_SLICE_OFFSET_UNIT,slice_offset,slice_size,slice_starts,slice_ends,slice_packet_id) ; + dta = locked_pop_out_data(PQISTREAM_OPTIMAL_PACKET_SIZE,slice_size,slice_starts,slice_ends,slice_packet_id) ; if(!dta) break ; - if(slice_size > 0xffff) - { - std::cerr << "(EE) protocol error in pqitreamer: slice size is too large and cannot be encoded." ; - free(mPkt_wpending) ; - mPkt_wpending_size = 0; - } - - if(slice_offset > 0xfffff || (slice_offset & 0xff)!=0) // 5 f, on purpose. Not a bug. - { - std::cerr << "(EE) protocol error in pqitreamer: slice size is too large and cannot be encoded." ; - free(mPkt_wpending) ; - mPkt_wpending_size = 0; - } - - if(slice_starts && slice_ends) // good old method. Send the packet as is, since it's a full packet. - { - std::cerr << "sending full slice, old style" << std::endl; - mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size) ; - memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size],dta,slice_size) ; - free(dta); - mPkt_wpending_size += slice_size ; - ++k ; - } - else // partial packet. We make a special header for it and insert it in the stream - { - std::cerr << "sending partial slice, packet ID=" << std::hex << slice_packet_id << std::dec << ", offset=" << slice_offset << ", size=" << slice_size << std::endl; - - mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE) ; - memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE],dta,slice_size) ; - free(dta); - - // New2: fp xxxxxx oooo ssss [data, sss bytes] => [flags 0.5B] [protocol version 0.5B] [2^24 packet count] [2^16 offset (in units of 16)] [size 2^16] - - uint8_t partial_flags = PQISTREAM_SLICE_PROTOCOL_VERSION_ID ; // includes version. Flags are in the first half-byte - if(slice_starts) partial_flags |= PQISTREAM_SLICE_FLAG_STARTS ; - if(slice_ends ) partial_flags |= PQISTREAM_SLICE_FLAG_ENDS ; - - ((char*)mPkt_wpending)[mPkt_wpending_size+0x00] = partial_flags ; - ((char*)mPkt_wpending)[mPkt_wpending_size+0x01] = uint8_t(slice_packet_id >> 16) & 0xff ; - ((char*)mPkt_wpending)[mPkt_wpending_size+0x02] = uint8_t(slice_packet_id >> 8) & 0xff ; - ((char*)mPkt_wpending)[mPkt_wpending_size+0x03] = uint8_t(slice_packet_id >> 0) & 0xff ; - ((char*)mPkt_wpending)[mPkt_wpending_size+0x04] = uint8_t(slice_offset >> 12) & 0xff ; - ((char*)mPkt_wpending)[mPkt_wpending_size+0x05] = uint8_t(slice_offset >> 4) & 0xff ; // not a bug. The last 4 bits are discarded because they are always 0 - ((char*)mPkt_wpending)[mPkt_wpending_size+0x06] = uint8_t(slice_size >> 8) & 0xff ; - ((char*)mPkt_wpending)[mPkt_wpending_size+0x07] = uint8_t(slice_size >> 0) & 0xff ; - - mPkt_wpending_size += slice_size + PQISTREAM_PARTIAL_PACKET_HEADER_SIZE; - ++k ; - } + if(slice_size > 0xffff) + { + std::cerr << "(EE) protocol error in pqitreamer: slice size is too large and cannot be encoded." ; + free(mPkt_wpending) ; + mPkt_wpending_size = 0; + } + + if(slice_starts && slice_ends) // good old method. Send the packet as is, since it's a full packet. + { + std::cerr << "sending full slice, old style" << std::endl; + mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size) ; + memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size],dta,slice_size) ; + free(dta); + mPkt_wpending_size += slice_size ; + ++k ; + } + else // partial packet. We make a special header for it and insert it in the stream + { + std::cerr << "sending partial slice, packet ID=" << std::hex << slice_packet_id << std::dec << ", size=" << slice_size << std::endl; + + mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE) ; + memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE],dta,slice_size) ; + free(dta); + + // New2: pp ff xxxxxxxx ssss [data, sss bytes] => [flags 1B] [protocol version 1B] [2^32 packet count] [2^16 size] + + uint8_t partial_flags = 0 ; + if(slice_starts) partial_flags |= PQISTREAM_SLICE_FLAG_STARTS ; + if(slice_ends ) partial_flags |= PQISTREAM_SLICE_FLAG_ENDS ; + + ((char*)mPkt_wpending)[mPkt_wpending_size+0x00] = PQISTREAM_SLICE_PROTOCOL_VERSION_ID_01 ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x01] = partial_flags ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x02] = uint8_t(slice_packet_id >> 24) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x03] = uint8_t(slice_packet_id >> 16) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x04] = uint8_t(slice_packet_id >> 8) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x05] = uint8_t(slice_packet_id >> 0) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x06] = uint8_t(slice_size >> 8) & 0xff ; + ((char*)mPkt_wpending)[mPkt_wpending_size+0x07] = uint8_t(slice_size >> 0) & 0xff ; + + mPkt_wpending_size += slice_size + PQISTREAM_PARTIAL_PACKET_HEADER_SIZE; + ++k ; + } } while(mPkt_wpending_size < (uint32_t)maxbytes && mPkt_wpending_size < PQISTREAM_OPTIMAL_PACKET_SIZE ) ; @@ -574,9 +565,9 @@ int pqistreamer::handleoutgoing_locked() RsScopeTimer tmer("pqistreamer:"+PeerId().toStdString()) ; // write packet. -#ifdef DEBUG_PQISTREAMER +//#ifdef DEBUG_PQISTREAMER std::cout << "Sending Out Pkt of size " << mPkt_wpending_size << " !" << std::endl; -#endif +//#endif int ss=0; if (mPkt_wpending_size != (ss = mBio->senddata(mPkt_wpending, mPkt_wpending_size))) @@ -718,24 +709,23 @@ continue_packet: // workout how much more to read. bool is_partial_packet = false ; - bool is_packet_starting = (((char*)block)[0] == 0x11) ; - bool is_packet_ending = (((char*)block)[0] == 0x12) ; + bool is_packet_starting = (((char*)block)[1] == PQISTREAM_SLICE_FLAG_STARTS) ; + bool is_packet_ending = (((char*)block)[1] == PQISTREAM_SLICE_FLAG_ENDS) ; + bool is_packet_middle = (((char*)block)[1] == 0x00) ; uint32_t extralen =0; - uint32_t slice_offset = 0 ; uint32_t slice_packet_id =0; - if( ((char*)block)[0] == 0x10 || ((char*)block)[0] == 0x11 || ((char*)block)[0] == 0x12) + if( ((char*)block)[0] == PQISTREAM_SLICE_PROTOCOL_VERSION_ID_01 && ( is_packet_starting || is_packet_middle || is_packet_ending)) { - extralen = (uint32_t(((uint8_t*)block)[6]) << 8) + (uint32_t(((uint8_t*)block)[7])); - slice_offset = (uint32_t(((uint8_t*)block)[5]) << 4) + (uint32_t(((uint8_t*)block)[4]) << 12); - slice_packet_id = (uint32_t(((uint8_t*)block)[3]) << 0) + (uint32_t(((uint8_t*)block)[2]) << 8) + (uint32_t(((uint8_t*)block)[1]) << 16); + extralen = (uint32_t(((uint8_t*)block)[6]) << 8 ) + (uint32_t(((uint8_t*)block)[7])); + slice_packet_id = (uint32_t(((uint8_t*)block)[2]) << 24) + (uint32_t(((uint8_t*)block)[3]) << 16) + (uint32_t(((uint8_t*)block)[4]) << 8) + (uint32_t(((uint8_t*)block)[5]) << 0); - std::cerr << "Reading from mem block " << RsUtil::BinToHex((char*)block,8) << ": packet_id=" << slice_packet_id << ", len=" << extralen << ", offset=" << slice_offset << std::endl; + std::cerr << "Reading partial packet from mem block " << RsUtil::BinToHex((char*)block,8) << ": packet_id=" << std::hex << slice_packet_id << std::dec << ", len=" << extralen << std::endl; is_partial_packet = true ; } else - extralen = getRsItemSize(block) - blen; + extralen = getRsItemSize(block) - blen; // old style packet type #ifdef DEBUG_PQISTREAMER std::cerr << "[" << (void*)pthread_self() << "] " << "continuing packet getRsItemSize(block) = " << getRsItemSize(block) << std::endl ; @@ -884,7 +874,7 @@ continue_packet: { std::cerr << "Inputing partial packet " << RsUtil::BinToHex((char*)block,8) << std::endl; - pkt = addPartialPacket(block,pktlen,slice_offset,slice_packet_id,is_packet_starting,is_packet_ending) ; + pkt = addPartialPacket(block,pktlen,slice_packet_id,is_packet_starting,is_packet_ending) ; } else pkt = mRsSerialiser->deserialise(block, &pktlen); @@ -927,13 +917,13 @@ continue_packet: return 0; } -RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t slice_offset,uint32_t slice_packet_id,bool is_packet_starting,bool is_packet_ending) +RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t slice_packet_id,bool is_packet_starting,bool is_packet_ending) { - std::cerr << "Receiving partial packet. size=" << len << ", offset=" << slice_offset << ". ID=" << std::hex << slice_packet_id << std::dec << ", starting:" << is_packet_starting << ", ending:" << is_packet_ending ; + std::cerr << "Receiving partial packet. size=" << len << ", ID=" << std::hex << slice_packet_id << std::dec << ", starting:" << is_packet_starting << ", ending:" << is_packet_ending ; if(is_packet_starting && is_packet_ending) { - std::cerr << "(EE) unexpected situation. Got in addPartialPacket() a full packet both starting and ending" << std::endl; + std::cerr << " (EE) unexpected situation: both starting and ending" << std::endl; return NULL ; } @@ -948,7 +938,7 @@ RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t sl if(!is_packet_starting) { - std::cerr << "(EE) dropping non starting packet that has no record." << std::endl; + std::cerr << " (EE) non starting packet has no record. Dropping" << std::endl; return NULL ; } PartialPacketRecord& rec = mPartialPackets[slice_packet_id] ; @@ -957,14 +947,14 @@ RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t sl if(!rec.mem) { - std::cerr << "(EE) Cannot allowcate memory for slice of size " << slice_length << std::endl; + std::cerr << " (EE) Cannot allocate memory for slice of size " << slice_length << std::endl; return NULL ; } memcpy(rec.mem, slice_data, slice_length) ; ; rec.size = slice_length ; - std::cerr << " => stored in new record." << std::endl; + std::cerr << " => stored in new record (size=" << rec.size << std::endl; return NULL ; // no need to check for ending } @@ -984,11 +974,11 @@ RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t sl memcpy( &((char*)rec.mem)[rec.size],slice_data,slice_length) ; rec.size += slice_length ; - std::cerr << " => added to existing record " ; + std::cerr << " => added to existing record size=" << rec.size ; if(is_packet_ending) { - std::cerr << " => deserialising: mem=" << RsUtil::BinToHex((char*)rec.mem,std::min(20u,rec.size)) << std::endl; + std::cerr << " => deserialising: mem=" << RsUtil::BinToHex((char*)rec.mem,std::min(8u,rec.size)) << std::endl; RsItem *item = mRsSerialiser->deserialise(rec.mem, &rec.size); free(rec.mem) ; @@ -1232,9 +1222,8 @@ int pqistreamer::locked_gatherStatistics(std::list& out_lst,std:: return 1 ; } -void *pqistreamer::locked_pop_out_data(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) +void *pqistreamer::locked_pop_out_data(uint32_t max_slice_size,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id) { - offset = 0 ; size = 0 ; starts = true ; ends = true ; diff --git a/libretroshare/src/pqi/pqistreamer.h b/libretroshare/src/pqi/pqistreamer.h index 27badc004..47d3ac825 100644 --- a/libretroshare/src/pqi/pqistreamer.h +++ b/libretroshare/src/pqi/pqistreamer.h @@ -81,7 +81,7 @@ class pqistreamer: public PQInterface virtual int locked_out_queue_size() const ; virtual void locked_clear_out_queue() ; virtual int locked_compute_out_pkt_size() const ; - virtual void *locked_pop_out_data(uint32_t max_slice_size,uint32_t offset_unit,uint32_t& offset,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id); + virtual void *locked_pop_out_data(uint32_t max_slice_size,uint32_t& size,bool& starts,bool& ends,uint32_t& packet_id); virtual int locked_gatherStatistics(std::list& outqueue_stats,std::list& inqueue_stats); // extracting data. void updateRates() ; @@ -162,7 +162,7 @@ class pqistreamer: public PQInterface time_t mStatisticsTimeStamp ; void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); - RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_offset, uint32_t slice_packet_id,bool packet_starting,bool packet_ending); + RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_packet_id,bool packet_starting,bool packet_ending); std::map mPartialPackets ; }; From 153db5ca6417f5ee002775aacd246351f3a275c2 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 26 Apr 2016 09:22:24 -0400 Subject: [PATCH 31/43] added on/off mechanism for packet slicing to ensure packward compatibility --- libretroshare/src/pqi/pqistreamer.cc | 58 ++++++++++++++++++++++++---- libretroshare/src/pqi/pqistreamer.h | 2 + 2 files changed, 53 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 3f8f711b0..aad83a5de 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -48,6 +48,13 @@ static const int PQISTREAM_SLICE_FLAG_STARTS = 0x01; // static const int PQISTREAM_SLICE_FLAG_ENDS = 0x02; // these flags should be kept in the range 0x01-0x08 static const int PQISTREAM_SLICE_PROTOCOL_VERSION_ID_01 = 0x10; // Protocol version ID. Should hold on the 4 lower bits. static const int PQISTREAM_PARTIAL_PACKET_HEADER_SIZE = 8; // Same size than normal header, to make the code simpler. +static const int PQISTREAM_PACKET_SLICING_PROBE_DELAY = 60; // send every 60 secs. + +// This is a probe packet, that won't deserialise (it's empty) but will not cause problems to old peers either, since they will ignore +// it. This packet however will be understood by new peers as a signal to enable packet slicing. This should go when all peers use the +// same protocol. + +static uint8_t PACKET_SLICING_PROBE_BYTES[8] = { 0x02, 0xaa, 0xbb, 0xcc, 0x00, 0x00, 0x00, 0x08 } ; /* This removes the print statements (which hammer pqidebug) */ /*** @@ -71,6 +78,9 @@ pqistreamer::pqistreamer(RsSerialiser *rss, const RsPeerId& id, BinInterface *bi { RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + mAcceptsPacketSlicing = false ; // by default. Will be turned into true when everyone's ready. + mLastSentPacketSlicingProbe = 0 ; + mAvgLastUpdate = mCurrReadTS = mCurrSentTS = time(NULL); mIncomingSize = 0 ; @@ -103,6 +113,7 @@ pqistreamer::~pqistreamer() { RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ + std::cerr << "Closing pqistreamer." << std::endl; pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::~pqistreamer() Destruction!"); if (mBio_flags & BIN_FLAGS_NO_CLOSE) @@ -192,8 +203,8 @@ void pqistreamer::updateRates() { int64_t diff = int64_t(t) - int64_t(mAvgLastUpdate) ; - float avgReadpSec = getRate(true) * PQISTREAM_AVG_FRAC + (1.0 - PQISTREAM_AVG_FRAC) * mAvgReadCount/(1000.0 * float(diff)); - float avgSentpSec = getRate(false) * PQISTREAM_AVG_FRAC + (1.0 - PQISTREAM_AVG_FRAC) * mAvgSentCount/(1000.0 * float(diff)); + float avgReadpSec = getRate(true ) * PQISTREAM_AVG_FRAC + (1.0 - PQISTREAM_AVG_FRAC) * mAvgReadCount/(1000.0 * float(diff)); + float avgSentpSec = getRate(false) * PQISTREAM_AVG_FRAC + (1.0 - PQISTREAM_AVG_FRAC) * mAvgSentCount/(1000.0 * float(diff)); #ifdef DEBUG_PQISTREAMER std::cerr << "Peer " << PeerId() << ": Current speed estimates: " << avgReadpSec << " / " << avgSentpSec << std::endl; @@ -263,6 +274,7 @@ int pqistreamer::tick_send(uint32_t timeout) { handleoutgoing_locked(); } + return 1; } @@ -455,6 +467,9 @@ int pqistreamer::handleoutgoing_locked() { /* if we are not active - clear anything in the queues. */ locked_clear_out_queue() ; + + std::cerr << "(II) Switching off packet slicing." << std::endl; + mAcceptsPacketSlicing = false ; /* also remove the pending packets */ if (mPkt_wpending) @@ -497,6 +512,21 @@ int pqistreamer::handleoutgoing_locked() mPkt_wpending_size = 0 ; int k=0; + // Checks for inserting a packet slicing probe. We do that to send the other peer the information that packet slicing can be used. + // if so, we enable it for the session. This should be removed (because it's unnecessary) when all users have switched to the new version. + time_t now = time(NULL) ; + + if((!mAcceptsPacketSlicing) && now > mLastSentPacketSlicingProbe + PQISTREAM_PACKET_SLICING_PROBE_DELAY) + { + std::cerr << "(II) Inserting packet slicing probe in traffic" << std::endl; + + mPkt_wpending_size = 8 ; + mPkt_wpending = rs_malloc(8) ; + memcpy(mPkt_wpending,PACKET_SLICING_PROBE_BYTES,8) ; + + mLastSentPacketSlicingProbe = now ; + } + uint32_t slice_size=0; bool slice_starts=true ; bool slice_ends=true ; @@ -504,7 +534,9 @@ int pqistreamer::handleoutgoing_locked() do { - dta = locked_pop_out_data(PQISTREAM_OPTIMAL_PACKET_SIZE,slice_size,slice_starts,slice_ends,slice_packet_id) ; + int desired_packet_size = mAcceptsPacketSlicing?PQISTREAM_OPTIMAL_PACKET_SIZE:(getRsPktMaxSize()); + + dta = locked_pop_out_data(desired_packet_size,slice_size,slice_starts,slice_ends,slice_packet_id) ; if(!dta) break ; @@ -565,9 +597,9 @@ int pqistreamer::handleoutgoing_locked() RsScopeTimer tmer("pqistreamer:"+PeerId().toStdString()) ; // write packet. -//#ifdef DEBUG_PQISTREAMER - std::cout << "Sending Out Pkt of size " << mPkt_wpending_size << " !" << std::endl; -//#endif +#ifdef DEBUG_PQISTREAMER + std::cout << "Sending Out Pkt of size " << mPkt_wpending_size << " !" << std::endl; +#endif int ss=0; if (mPkt_wpending_size != (ss = mBio->senddata(mPkt_wpending, mPkt_wpending_size))) @@ -584,9 +616,13 @@ int pqistreamer::handleoutgoing_locked() // ensuring exactly the same data is written (openSSL requirement). return -1; } +#ifdef DEBUG_PQISTREAMER else std::cerr << PeerId() << ": sent " << ss << " bytes " << std::endl; +#endif + ++nsent; + outSentBytes_locked(mPkt_wpending_size); // this is the only time where we know exactly what was sent. #ifdef DEBUG_TRANSFERS @@ -703,13 +739,21 @@ start_packet_read: readbytes += blen; mReading_state = reading_state_packet_started ; mFailed_read_attempts = 0 ; // reset failed read, as the packet has been totally read. + + // Check for packet slicing probe (04/26/2016). To be removed when everyone uses it. + + if(!memcmp(block,PACKET_SLICING_PROBE_BYTES,8)) + { + mAcceptsPacketSlicing = true ; + std::cerr << "(II) Enabling packet slicing!" << std::endl; + } } continue_packet: { // workout how much more to read. bool is_partial_packet = false ; - bool is_packet_starting = (((char*)block)[1] == PQISTREAM_SLICE_FLAG_STARTS) ; + bool is_packet_starting = (((char*)block)[1] == PQISTREAM_SLICE_FLAG_STARTS) ; // STARTS and ENDS flags are actually never combined. bool is_packet_ending = (((char*)block)[1] == PQISTREAM_SLICE_FLAG_ENDS) ; bool is_packet_middle = (((char*)block)[1] == 0x00) ; diff --git a/libretroshare/src/pqi/pqistreamer.h b/libretroshare/src/pqi/pqistreamer.h index 47d3ac825..b50244622 100644 --- a/libretroshare/src/pqi/pqistreamer.h +++ b/libretroshare/src/pqi/pqistreamer.h @@ -161,6 +161,8 @@ class pqistreamer: public PQInterface std::list mCurrentStatsChunk_Out ; time_t mStatisticsTimeStamp ; + bool mAcceptsPacketSlicing ; + time_t mLastSentPacketSlicingProbe ; void locked_addTrafficClue(const RsItem *pqi, uint32_t pktsize, std::list &lst); RsItem *addPartialPacket(const void *block, uint32_t len, uint32_t slice_packet_id,bool packet_starting,bool packet_ending); From 87764abe4c72c079e4f62556f24d42aae73e3c23 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 26 Apr 2016 21:23:19 -0400 Subject: [PATCH 32/43] removed debug info in pqistreamer --- libretroshare/src/pqi/pqistreamer.cc | 41 +++++++++++++++++++++------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index aad83a5de..8e0878b08 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -61,6 +61,7 @@ static uint8_t PACKET_SLICING_PROBE_BYTES[8] = { 0x02, 0xaa, 0xbb, 0xcc, 0x00, #define RSITEM_DEBUG 1 #define DEBUG_TRANSFERS 1 #define DEBUG_PQISTREAMER 1 +#define DEBUG_PACKET_SLICING 1 ***/ #ifdef DEBUG_TRANSFERS @@ -112,8 +113,9 @@ pqistreamer::pqistreamer(RsSerialiser *rss, const RsPeerId& id, BinInterface *bi pqistreamer::~pqistreamer() { RsStackMutex stack(mStreamerMtx); /**** LOCKED MUTEX ****/ - +#ifdef DEBUG_PQISTREAMER std::cerr << "Closing pqistreamer." << std::endl; +#endif pqioutput(PQL_DEBUG_ALL, pqistreamerzone, "pqistreamer::~pqistreamer() Destruction!"); if (mBio_flags & BIN_FLAGS_NO_CLOSE) @@ -467,8 +469,9 @@ int pqistreamer::handleoutgoing_locked() { /* if we are not active - clear anything in the queues. */ locked_clear_out_queue() ; - +#ifdef DEBUG_PACKET_SLICING std::cerr << "(II) Switching off packet slicing." << std::endl; +#endif mAcceptsPacketSlicing = false ; /* also remove the pending packets */ @@ -493,12 +496,12 @@ int pqistreamer::handleoutgoing_locked() if ((!(mBio->cansend(0))) || (maxbytes < sentbytes)) { -//#ifdef DEBUG_PQISTREAMER +#ifdef DEBUG_PACKET_SLICING if (maxbytes < sentbytes) std::cerr << "pqistreamer::handleoutgoing_locked() Stopped sending: bio not ready. maxbytes=" << maxbytes << ", sentbytes=" << sentbytes << std::endl; else std::cerr << "pqistreamer::handleoutgoing_locked() Stopped sending: sentbytes=" << sentbytes << ", max=" << maxbytes << std::endl; -//#endif +#endif return 0; } @@ -518,7 +521,9 @@ int pqistreamer::handleoutgoing_locked() if((!mAcceptsPacketSlicing) && now > mLastSentPacketSlicingProbe + PQISTREAM_PACKET_SLICING_PROBE_DELAY) { +#ifdef DEBUG_PACKET_SLICING std::cerr << "(II) Inserting packet slicing probe in traffic" << std::endl; +#endif mPkt_wpending_size = 8 ; mPkt_wpending = rs_malloc(8) ; @@ -550,7 +555,9 @@ int pqistreamer::handleoutgoing_locked() if(slice_starts && slice_ends) // good old method. Send the packet as is, since it's a full packet. { - std::cerr << "sending full slice, old style" << std::endl; +#ifdef DEBUG_PACKET_SLICING + std::cerr << "sending full slice, old style. Size=" << slice_size << std::endl; +#endif mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size) ; memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size],dta,slice_size) ; free(dta); @@ -559,7 +566,9 @@ int pqistreamer::handleoutgoing_locked() } else // partial packet. We make a special header for it and insert it in the stream { +#ifdef DEBUG_PACKET_SLICING std::cerr << "sending partial slice, packet ID=" << std::hex << slice_packet_id << std::dec << ", size=" << slice_size << std::endl; +#endif mPkt_wpending = realloc(mPkt_wpending,slice_size+mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE) ; memcpy( &((char*)mPkt_wpending)[mPkt_wpending_size+PQISTREAM_PARTIAL_PACKET_HEADER_SIZE],dta,slice_size) ; @@ -745,7 +754,9 @@ start_packet_read: if(!memcmp(block,PACKET_SLICING_PROBE_BYTES,8)) { mAcceptsPacketSlicing = true ; +#ifdef DEBUG_PACKET_SLICING std::cerr << "(II) Enabling packet slicing!" << std::endl; +#endif } } continue_packet: @@ -765,7 +776,9 @@ continue_packet: extralen = (uint32_t(((uint8_t*)block)[6]) << 8 ) + (uint32_t(((uint8_t*)block)[7])); slice_packet_id = (uint32_t(((uint8_t*)block)[2]) << 24) + (uint32_t(((uint8_t*)block)[3]) << 16) + (uint32_t(((uint8_t*)block)[4]) << 8) + (uint32_t(((uint8_t*)block)[5]) << 0); +#ifdef DEBUG_PACKET_SLICING std::cerr << "Reading partial packet from mem block " << RsUtil::BinToHex((char*)block,8) << ": packet_id=" << std::hex << slice_packet_id << std::dec << ", len=" << extralen << std::endl; +#endif is_partial_packet = true ; } else @@ -905,19 +918,17 @@ continue_packet: } #endif - // std::cerr << "Deserializing packet of size " << pktlen < stored in new record (size=" << rec.size << std::endl; +#endif return NULL ; // no need to check for ending } @@ -1018,11 +1033,15 @@ RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t sl memcpy( &((char*)rec.mem)[rec.size],slice_data,slice_length) ; rec.size += slice_length ; +#ifdef DEBUG_PACKET_SLICING std::cerr << " => added to existing record size=" << rec.size ; +#endif if(is_packet_ending) { +#ifdef DEBUG_PACKET_SLICING std::cerr << " => deserialising: mem=" << RsUtil::BinToHex((char*)rec.mem,std::min(8u,rec.size)) << std::endl; +#endif RsItem *item = mRsSerialiser->deserialise(rec.mem, &rec.size); free(rec.mem) ; @@ -1031,7 +1050,9 @@ RsItem *pqistreamer::addPartialPacket(const void *block,uint32_t len,uint32_t sl } else { +#ifdef DEBUG_PACKET_SLICING std::cerr << std::endl; +#endif return NULL ; } } From a4430647de6074d181f7dc5a2ffb8308016522d0 Mon Sep 17 00:00:00 2001 From: csoler Date: Tue, 26 Apr 2016 23:42:44 -0400 Subject: [PATCH 33/43] various changes to measure RTTs more accurately --- libretroshare/src/rsserver/p3face-server.cc | 7 ++- libretroshare/src/services/p3rtt.cc | 66 +++++++-------------- libretroshare/src/services/p3rtt.h | 2 +- 3 files changed, 29 insertions(+), 46 deletions(-) diff --git a/libretroshare/src/rsserver/p3face-server.cc b/libretroshare/src/rsserver/p3face-server.cc index b76913693..9fc0758b5 100644 --- a/libretroshare/src/rsserver/p3face-server.cc +++ b/libretroshare/src/rsserver/p3face-server.cc @@ -47,6 +47,7 @@ int rsserverzone = 101; /**** #define DEBUG_TICK 1 ****/ +#define DEBUG_TICK 1 #define WARN_BIG_CYCLE_TIME (0.2) #ifdef WINDOWS_SYS @@ -69,8 +70,8 @@ static double getCurrentTS() return cts; } -const double RsServer::minTimeDelta = 0.1; // 25; -const double RsServer::maxTimeDelta = 0.5; +const double RsServer::minTimeDelta = 0.05; // 25; +const double RsServer::maxTimeDelta = 0.1; const double RsServer::kickLimit = 0.15; @@ -138,6 +139,8 @@ void RsServer::data_tick() double ts = getCurrentTS(); double delta = ts - mLastts; + + std::cerr << "[data tick] delta = " << delta << std::endl; /* for the fast ticked stuff */ if (delta > mTimeDelta) diff --git a/libretroshare/src/services/p3rtt.cc b/libretroshare/src/services/p3rtt.cc index 1e3c96b61..12aaf8e9a 100644 --- a/libretroshare/src/services/p3rtt.cc +++ b/libretroshare/src/services/p3rtt.cc @@ -23,6 +23,8 @@ * */ +#include + #include "util/rsdir.h" #include "retroshare/rsiface.h" #include "pqi/pqibin.h" @@ -37,6 +39,7 @@ /**** * #define DEBUG_RTT 1 ****/ +#define DEBUG_RTT 1 /* DEFINE INTERFACE POINTER! */ @@ -147,6 +150,7 @@ RsServiceInfo p3rtt::getServiceInfo() int p3rtt::tick() { + std::cerr << "p3rtt::tick() " << getCurrentTS() << std::endl; sendPackets(); return 0; @@ -168,7 +172,7 @@ int p3rtt::sendPackets() pt = mSentPingTime; } - if (now - pt > RTT_PING_PERIOD) + if (now >= pt+RTT_PING_PERIOD) { sendPingMeasurements(); @@ -190,19 +194,10 @@ void p3rtt::sendPingMeasurements() mServiceCtrl->getPeersConnected(getServiceInfo().mServiceType, idList); -#ifdef DEBUG_RTT - std::cerr << "p3rtt::sendPingMeasurements() @ts: " << ts; - std::cerr << std::endl; -#endif - /* prepare packets */ std::set::iterator it; for(it = idList.begin(); it != idList.end(); ++it) { -#ifdef DEBUG_RTT - std::cerr << "p3rtt::sendPingMeasurements() Pinging: " << *it; - std::cerr << std::endl; -#endif double ts = getCurrentTS(); /* create the packet */ @@ -214,11 +209,8 @@ void p3rtt::sendPingMeasurements() storePingAttempt(*it, ts, mCounter); #ifdef DEBUG_RTT - std::cerr << "p3rtt::sendPingMeasurements() With Packet:"; - std::cerr << std::endl; - pingPkt->print(std::cerr, 10); + std::cerr << "p3rtt::sendPingMeasurements() Pinging: " << *it << " [" << pingPkt->mSeqNo << "," << std::hex << pingPkt->mPingTS << std::dec << "]" << std::endl;; #endif - sendItem(pingPkt); } @@ -256,30 +248,26 @@ int p3rtt::handlePing(RsItem *item) /* cast to right type */ RsRttPingItem *ping = (RsRttPingItem *) item; + double ts = getCurrentTS(); #ifdef DEBUG_RTT - std::cerr << "p3rtt::handlePing() Recvd Packet from: " << ping->PeerId(); - std::cerr << std::endl; + std::cerr << "p3rtt::handlePing() from: " << ping->PeerId() << " - [" << ping->mSeqNo << "," << std::hex << ping->mPingTS << std::dec << "] " << std::endl; + std::cerr << "incoming ping travel time: " << ts - convert64bitsToTs(ping->mPingTS) << std::endl; #endif /* with a ping, we just respond as quickly as possible - they do all the analysis */ RsRttPongItem *pong = new RsRttPongItem(); - pong->PeerId(ping->PeerId()); pong->mPingTS = ping->mPingTS; pong->mSeqNo = ping->mSeqNo; // add our timestamp. - double ts = getCurrentTS(); pong->mPongTS = convertTsTo64bits(ts); - -#ifdef DEBUG_RTT - std::cerr << "p3rtt::handlePing() With Packet:"; - std::cerr << std::endl; - pong->print(std::cerr, 10); -#endif - + static double mLastResponseToPong = 0.0 ;// bad stuff + std::cerr << "Delay since last response to PONG: " << ts - mLastResponseToPong << std::endl; + + mLastResponseToPong = ts ; sendItem(pong); return true ; } @@ -291,9 +279,7 @@ int p3rtt::handlePong(RsItem *item) RsRttPongItem *pong = (RsRttPongItem *) item; #ifdef DEBUG_RTT - std::cerr << "p3rtt::handlePong() Recvd Packet from: " << pong->PeerId(); - std::cerr << std::endl; - pong->print(std::cerr, 10); + std::cerr << "p3rtt::handlePong() from: " << pong->PeerId() << " - [" << pong->mSeqNo << "," << std::hex << pong->mPingTS << " -> " << pong->mPongTS << std::dec << "] "<< std::endl; #endif /* with a pong, we do the maths! */ @@ -305,21 +291,12 @@ int p3rtt::handlePong(RsItem *item) double offset = pongTS - (recvTS - rtt / 2.0); // so to get to their time, we go ourTS + offset. #ifdef DEBUG_RTT - std::cerr << "p3rtt::handlePong() Timing:"; - std::cerr << std::endl; - std::cerr << "\tpingTS: " << pingTS; - std::cerr << std::endl; - std::cerr << "\tpongTS: " << pongTS; - std::cerr << std::endl; - std::cerr << "\trecvTS: " << recvTS; - std::cerr << std::endl; - std::cerr << "\t ==> rtt: " << rtt; - std::cerr << std::endl; - std::cerr << "\t ==> offset: " << offset; - std::cerr << std::endl; + std::cerr << "incoming pong travel time: " << recvTS - convert64bitsToTs(pong->mPongTS) << std::endl; + std::cerr << " RTT analysis: pingTS: " << std::setprecision(16) << pingTS << ", pongTS: " << pongTS + << ", recvTS: " << std::setprecision(16) << recvTS << " ==> rtt: " << rtt << ", offset: " << offset << std::endl; #endif - storePongResult(pong->PeerId(), pong->mSeqNo, pingTS, rtt, offset); + storePongResult(pong->PeerId(), pong->mSeqNo, recvTS, rtt, offset); return true ; } @@ -333,6 +310,7 @@ int p3rtt::storePingAttempt(const RsPeerId& id, double ts, uint32_t seqno) /* find corresponding local data */ RttPeerInfo *peerInfo = locked_GetPeerInfo(id); + std::cerr << "Delay since previous ping attempt: " << ts - peerInfo->mCurrentPingTS << std::endl; peerInfo->mCurrentPingTS = ts; peerInfo->mCurrentPingCounter = seqno; @@ -349,7 +327,7 @@ int p3rtt::storePingAttempt(const RsPeerId& id, double ts, uint32_t seqno) -int p3rtt::storePongResult(const RsPeerId& id, uint32_t counter, double ts, double rtt, double offset) +int p3rtt::storePongResult(const RsPeerId& id, uint32_t counter, double recv_ts, double rtt, double offset) { RsStackMutex stack(mRttMtx); /****** LOCKED MUTEX *******/ @@ -366,8 +344,10 @@ int p3rtt::storePongResult(const RsPeerId& id, uint32_t counter, double ts, doub { peerInfo->mCurrentPongRecvd = true; } + if(!peerInfo->mPongResults.empty()) + std::cerr << "Delay since last pong: " << recv_ts - peerInfo->mPongResults.back().mTS << std::endl; - peerInfo->mPongResults.push_back(RsRttPongResult(ts, rtt, offset)); + peerInfo->mPongResults.push_back(RsRttPongResult(recv_ts, rtt, offset)); while(peerInfo->mPongResults.size() > MAX_PONG_RESULTS) diff --git a/libretroshare/src/services/p3rtt.h b/libretroshare/src/services/p3rtt.h index 0b7e61f46..9b881fba8 100644 --- a/libretroshare/src/services/p3rtt.h +++ b/libretroshare/src/services/p3rtt.h @@ -84,7 +84,7 @@ virtual bool recvItem(RsItem *item); // Overloaded from p3FastService. int handlePong(RsItem *item); int storePingAttempt(const RsPeerId& id, double ts, uint32_t mCounter); - int storePongResult(const RsPeerId& id, uint32_t counter, double ts, double rtt, double offset); + int storePongResult(const RsPeerId& id, uint32_t counter, double recv_ts, double rtt, double offset); /*! From 654e0fc1d83a56f7a8e25abc269ead27587687ce Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 29 Apr 2016 17:53:21 -0400 Subject: [PATCH 34/43] added missing call to SSL_pending, which caused decrypted data to wait in the SSL buffer when the bio is not receiving anything more --- libretroshare/src/pqi/pqissl.cc | 6 +++++- libretroshare/src/pqi/pqistreamer.cc | 3 +++ libretroshare/src/rsserver/p3face-server.cc | 3 --- libretroshare/src/services/p3rtt.cc | 1 - 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libretroshare/src/pqi/pqissl.cc b/libretroshare/src/pqi/pqissl.cc index 15c088627..e729534bc 100644 --- a/libretroshare/src/pqi/pqissl.cc +++ b/libretroshare/src/pqi/pqissl.cc @@ -1857,7 +1857,11 @@ bool pqissl::moretoread(uint32_t usec) #endif return 1; } - else + else if(SSL_pending(ssl_connection) > 0) + { + return 1 ; + } + else { #ifdef PQISSL_DEBUG rslog(RSL_DEBUG_ALL, pqisslzone, diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 8e0878b08..1a43fde02 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -63,6 +63,7 @@ static uint8_t PACKET_SLICING_PROBE_BYTES[8] = { 0x02, 0xaa, 0xbb, 0xcc, 0x00, #define DEBUG_PQISTREAMER 1 #define DEBUG_PACKET_SLICING 1 ***/ +#define DEBUG_PACKET_SLICING 1 #ifdef DEBUG_TRANSFERS #include "util/rsprint.h" @@ -780,6 +781,8 @@ continue_packet: std::cerr << "Reading partial packet from mem block " << RsUtil::BinToHex((char*)block,8) << ": packet_id=" << std::hex << slice_packet_id << std::dec << ", len=" << extralen << std::endl; #endif is_partial_packet = true ; + + mAcceptsPacketSlicing = true ; // this is needed } else extralen = getRsItemSize(block) - blen; // old style packet type diff --git a/libretroshare/src/rsserver/p3face-server.cc b/libretroshare/src/rsserver/p3face-server.cc index 9fc0758b5..a5b1c83ef 100644 --- a/libretroshare/src/rsserver/p3face-server.cc +++ b/libretroshare/src/rsserver/p3face-server.cc @@ -47,7 +47,6 @@ int rsserverzone = 101; /**** #define DEBUG_TICK 1 ****/ -#define DEBUG_TICK 1 #define WARN_BIG_CYCLE_TIME (0.2) #ifdef WINDOWS_SYS @@ -140,8 +139,6 @@ void RsServer::data_tick() double ts = getCurrentTS(); double delta = ts - mLastts; - std::cerr << "[data tick] delta = " << delta << std::endl; - /* for the fast ticked stuff */ if (delta > mTimeDelta) { diff --git a/libretroshare/src/services/p3rtt.cc b/libretroshare/src/services/p3rtt.cc index 12aaf8e9a..74dc0e1e1 100644 --- a/libretroshare/src/services/p3rtt.cc +++ b/libretroshare/src/services/p3rtt.cc @@ -150,7 +150,6 @@ RsServiceInfo p3rtt::getServiceInfo() int p3rtt::tick() { - std::cerr << "p3rtt::tick() " << getCurrentTS() << std::endl; sendPackets(); return 0; From 5ab9aed37199d0ba7c98f06a17db108673c6cdd8 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 29 Apr 2016 18:40:28 -0400 Subject: [PATCH 35/43] added placeholder service ID for packet slicing probe --- libretroshare/src/serialiser/rsserviceids.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/libretroshare/src/serialiser/rsserviceids.h b/libretroshare/src/serialiser/rsserviceids.h index b7d7b2fa1..7aadd3dea 100644 --- a/libretroshare/src/serialiser/rsserviceids.h +++ b/libretroshare/src/serialiser/rsserviceids.h @@ -96,6 +96,9 @@ const uint16_t RS_SERVICE_TYPE_PLUGIN_ARADO_ID = 0x2001; const uint16_t RS_SERVICE_TYPE_PLUGIN_QCHESS_ID = 0x2002; const uint16_t RS_SERVICE_TYPE_PLUGIN_FEEDREADER = 0x2003; +// Reserved for packet slicing probes. +const uint16_t RS_SERVICE_TYPE_PACKET_SLICING_PROBE = 0xAABB; + // Nabu's services. const uint16_t RS_SERVICE_TYPE_PLUGIN_FIDO_GW = 0xF1D0; const uint16_t RS_SERVICE_TYPE_PLUGIN_ZERORESERVE = 0xBEEF; From 3bc3989e790de985d4f6cbf71536cbae6949a554 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 29 Apr 2016 18:41:10 -0400 Subject: [PATCH 36/43] removed debug info for pqistreamer --- libretroshare/src/pqi/pqistreamer.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 1a43fde02..7f46d454f 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -63,7 +63,6 @@ static uint8_t PACKET_SLICING_PROBE_BYTES[8] = { 0x02, 0xaa, 0xbb, 0xcc, 0x00, #define DEBUG_PQISTREAMER 1 #define DEBUG_PACKET_SLICING 1 ***/ -#define DEBUG_PACKET_SLICING 1 #ifdef DEBUG_TRANSFERS #include "util/rsprint.h" From 6d2bfcf04e4ef053cbefac4d8a8b8c2cf17c3f4f Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 29 Apr 2016 18:52:58 -0400 Subject: [PATCH 37/43] restored average kicking rate to a more reasonnable value. To be tested (for too much CPU use). Dynamically changing this value is probably needed for VOIP --- libretroshare/src/rsserver/p3face-server.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/libretroshare/src/rsserver/p3face-server.cc b/libretroshare/src/rsserver/p3face-server.cc index a5b1c83ef..037ef3a44 100644 --- a/libretroshare/src/rsserver/p3face-server.cc +++ b/libretroshare/src/rsserver/p3face-server.cc @@ -69,8 +69,12 @@ static double getCurrentTS() return cts; } +// These values should be tunable from the GUI, to offer a compromise between speed and CPU use. +// In some cases (VOIP) it's likely that we will need to set them temporarily to a very low +// value, in order to favor a fast feedback + const double RsServer::minTimeDelta = 0.05; // 25; -const double RsServer::maxTimeDelta = 0.1; +const double RsServer::maxTimeDelta = 0.2; const double RsServer::kickLimit = 0.15; From 4bdd46486152fe975e8fd9759a11e6f07d862681 Mon Sep 17 00:00:00 2001 From: csoler Date: Fri, 29 Apr 2016 18:53:25 -0400 Subject: [PATCH 38/43] removed a few debug output and timer measurement --- libretroshare/src/pqi/pqistreamer.cc | 2 -- libretroshare/src/services/p3rtt.cc | 8 ++++++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 7f46d454f..8bf2fafb5 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -603,8 +603,6 @@ int pqistreamer::handleoutgoing_locked() if (mPkt_wpending) { - RsScopeTimer tmer("pqistreamer:"+PeerId().toStdString()) ; - // write packet. #ifdef DEBUG_PQISTREAMER std::cout << "Sending Out Pkt of size " << mPkt_wpending_size << " !" << std::endl; diff --git a/libretroshare/src/services/p3rtt.cc b/libretroshare/src/services/p3rtt.cc index 74dc0e1e1..751eccba4 100644 --- a/libretroshare/src/services/p3rtt.cc +++ b/libretroshare/src/services/p3rtt.cc @@ -39,8 +39,6 @@ /**** * #define DEBUG_RTT 1 ****/ -#define DEBUG_RTT 1 - /* DEFINE INTERFACE POINTER! */ RsRtt *rsRtt = NULL; @@ -264,7 +262,9 @@ int p3rtt::handlePing(RsItem *item) pong->mPongTS = convertTsTo64bits(ts); static double mLastResponseToPong = 0.0 ;// bad stuff +#ifdef DEBUG_RTT std::cerr << "Delay since last response to PONG: " << ts - mLastResponseToPong << std::endl; +#endif mLastResponseToPong = ts ; sendItem(pong); @@ -309,7 +309,9 @@ int p3rtt::storePingAttempt(const RsPeerId& id, double ts, uint32_t seqno) /* find corresponding local data */ RttPeerInfo *peerInfo = locked_GetPeerInfo(id); +#ifdef DEBUG_RTT std::cerr << "Delay since previous ping attempt: " << ts - peerInfo->mCurrentPingTS << std::endl; +#endif peerInfo->mCurrentPingTS = ts; peerInfo->mCurrentPingCounter = seqno; @@ -343,8 +345,10 @@ int p3rtt::storePongResult(const RsPeerId& id, uint32_t counter, double recv_ts, { peerInfo->mCurrentPongRecvd = true; } +#ifdef DEBUG_RTT if(!peerInfo->mPongResults.empty()) std::cerr << "Delay since last pong: " << recv_ts - peerInfo->mPongResults.back().mTS << std::endl; +#endif peerInfo->mPongResults.push_back(RsRttPongResult(recv_ts, rtt, offset)); From 0a6ae559fd1dea5cc4b94561785657e968e0c8e0 Mon Sep 17 00:00:00 2001 From: zeners Date: Sat, 30 Apr 2016 14:21:09 +0200 Subject: [PATCH 39/43] webui: redefined building webui on linux: 'make' don't build libresapi every time anymore, it now knows when, what and how to 'build' webui on windows: builds old style --- libresapi/src/libresapi.pro | 83 +++++++++++++++++++---- libresapi/src/webui-src/make-src/build.sh | 67 +++++++++++------- 2 files changed, 113 insertions(+), 37 deletions(-) diff --git a/libresapi/src/libresapi.pro b/libresapi/src/libresapi.pro index 39fca750e..a0d840754 100644 --- a/libresapi/src/libresapi.pro +++ b/libresapi/src/libresapi.pro @@ -15,19 +15,75 @@ INCLUDEPATH += ../../libretroshare/src unix { webui_files.path = "$${DATA_DIR}/webui" - webui_files.files = webui/* - INSTALLS += webui_files + webui_files.files = webui/app.js webui/app.css webui/index.html + INSTALLS += webui_files - webui_img_files.path = "$${DATA_DIR}/webui/img" - webui_img_files.files = ../../retroshare-gui/src/gui/images/logo/logo_splash.png - INSTALLS += webui_img_files + webui_img_files.path = "$${DATA_DIR}/webui/img" + webui_img_files.files = ../../retroshare-gui/src/gui/images/logo/logo_splash.png + INSTALLS += webui_img_files - create_webfiles.commands = sh $$_PRO_FILE_PWD_/webui-src/make-src/build.sh $$_PRO_FILE_PWD_ - QMAKE_EXTRA_TARGETS += create_webfiles - PRE_TARGETDEPS += create_webfiles + #create_webfiles.commands = sh $$_PRO_FILE_PWD_/webui-src/make-src/build.sh $$_PRO_FILE_PWD_ + #QMAKE_EXTRA_TARGETS += create_webfiles + #POST_TARGETDEPS += create_webfiles - # create dummy files - system(webui-src/make-src/init.sh .) + #QMAKE_POST_LINK=sh $$_PRO_FILE_PWD_/webui-src/make-src/build.sh $$_PRO_FILE_PWD_ + + WEBUI_SRC_SCRIPT = webui-src/make-src/build.sh + + WEBUI_SRC_HTML = $$WEBUI_SRC_SCRIPT + WEBUI_SRC_HTML += webui-src/app/assets/index.html + + WEBUI_SRC_JS = $$WEBUI_SRC_SCRIPT + WEBUI_SRC_JS += webui-src/app/accountselect.js + WEBUI_SRC_JS += webui-src/app/adddownloads.js + WEBUI_SRC_JS += webui-src/app/addidentity.js + WEBUI_SRC_JS += webui-src/app/addpeer.js + WEBUI_SRC_JS += webui-src/app/chat.js + WEBUI_SRC_JS += webui-src/app/createlogin.js + WEBUI_SRC_JS += webui-src/app/downloads.js + WEBUI_SRC_JS += webui-src/app/forums.js + WEBUI_SRC_JS += webui-src/app/home.js + WEBUI_SRC_JS += webui-src/app/identities.js + WEBUI_SRC_JS += webui-src/app/main.js + WEBUI_SRC_JS += webui-src/app/menudef.js + WEBUI_SRC_JS += webui-src/app/menu.js + WEBUI_SRC_JS += webui-src/app/mithril.js + WEBUI_SRC_JS += webui-src/app/mithril.min.js + WEBUI_SRC_JS += webui-src/app/peers.js + WEBUI_SRC_JS += webui-src/app/retroshare.js + WEBUI_SRC_JS += webui-src/app/search.js + WEBUI_SRC_JS += webui-src/app/searchresult.js + WEBUI_SRC_JS += webui-src/app/servicecontrol.js + WEBUI_SRC_JS += webui-src/app/settings.js + WEBUI_SRC_JS += webui-src/app/waiting.js + + WEBUI_SRC_CSS = $$WEBUI_SRC_SCRIPT + WEBUI_SRC_CSS += webui-src/app/green-black.scss + WEBUI_SRC_CSS += webui-src/app/_reset.scss + WEBUI_SRC_CSS += webui-src/app/_chat.sass + WEBUI_SRC_CSS += webui-src/app/main.sass + + + create_webfiles_html.output = webui/index.html + create_webfiles_html.input = WEBUI_SRC_HTML + create_webfiles_html.commands = sh $$_PRO_FILE_PWD_/webui-src/make-src/build.sh $$_PRO_FILE_PWD_ index.html . + create_webfiles_html.variable_out = OBJECTS + create_webfiles_html.CONFIG = combine no_link + + create_webfiles_js.output = webui/app.js + create_webfiles_js.input = WEBUI_SRC_JS + create_webfiles_js.commands = sh $$_PRO_FILE_PWD_/webui-src/make-src/build.sh $$_PRO_FILE_PWD_ app.js . + create_webfiles_js.variable_out = OBJECTS + create_webfiles_js.CONFIG = combine no_link + + create_webfiles_css.output = webui/app.css + create_webfiles_css.input = WEBUI_SRC_CSS + create_webfiles_css.commands = sh $$_PRO_FILE_PWD_/webui-src/make-src/build.sh $$_PRO_FILE_PWD_ app.css . + create_webfiles_css.variable_out = OBJECTS + create_webfiles_css.CONFIG = combine no_link + + + QMAKE_EXTRA_COMPILERS += create_webfiles_html create_webfiles_js create_webfiles_css } win32{ @@ -44,9 +100,10 @@ win32{ MAKE_SRC=$$PRO_PATH\\webui-src\\make-src } - create_webfiles.commands = $$MAKE_SRC\\build.bat $$PRO_PATH - QMAKE_EXTRA_TARGETS += create_webfiles - PRE_TARGETDEPS += create_webfiles + #create_webfiles.commands = $$MAKE_SRC\\build.bat $$PRO_PATH + #QMAKE_EXTRA_TARGETS += create_webfiles + #PRE_TARGETDEPS += create_webfiles + QMAKE_POST_LINK=$$MAKE_SRC\\build.bat $$PRO_PATH # create dummy files system($$MAKE_SRC\\init.bat .) diff --git a/libresapi/src/webui-src/make-src/build.sh b/libresapi/src/webui-src/make-src/build.sh index f96674cae..425119a72 100755 --- a/libresapi/src/webui-src/make-src/build.sh +++ b/libresapi/src/webui-src/make-src/build.sh @@ -2,7 +2,7 @@ # create webfiles from sources at compile time (works without npm/node.js) -if [ "$1" = "" ];then +if [ "$1" = "" ]; then publicdest=../../webui src=.. else @@ -10,34 +10,53 @@ else src=$1/webui-src fi -if [ -d "$publicdest" ]; then - echo remove existing $publicdest - rm $publicdest -R +if [ "$2" = "" ]; then + + if [ -d "$publicdest" ]; then + echo remove existing $publicdest + rm $publicdest -R + fi fi -echo mkdir $publicdest -mkdir $publicdest +if [ ! -d "$publicdest" ]; then + echo mkdir $publicdest + mkdir $publicdest +fi -echo building app.js -echo - copy template.js ... -cp $src/make-src/template.js $publicdest/app.js +if [ "$2" = "" ]||[ "$2" = "app.js" ]; then + echo building app.js + echo - copy template.js ... + cp $src/make-src/template.js $publicdest/app.js -for filename in $src/app/*.js; do - fname=$(basename "$filename") - fname="${fname%.*}" - echo - adding $fname ... - echo require.register\(\"$fname\", function\(exports, require, module\) { >> $publicdest/app.js - cat $filename >> $publicdest/app.js - echo >> $publicdest/app.js - echo }\)\; >> $publicdest/app.js -done + for filename in $src/app/*.js; do + fname=$(basename "$filename") + fname="${fname%.*}" + echo - adding $fname ... + echo require.register\(\"$fname\", function\(exports, require, module\) { >> $publicdest/app.js + cat $filename >> $publicdest/app.js + echo >> $publicdest/app.js + echo }\)\; >> $publicdest/app.js + done +fi -echo building app.css -cat $src/app/green-black.scss >> $publicdest/app.css -cat $src/make-src/main.css >> $publicdest/app.css -cat $src/make-src/chat.css >> $publicdest/app.css +if [ "$2" = "" ]||[ "$2" = "app.css" ]; then + echo building app.css + cat $src/app/green-black.scss >> $publicdest/app.css + cat $src/make-src/main.css >> $publicdest/app.css + cat $src/make-src/chat.css >> $publicdest/app.css +fi -echo copy index.html -cp $src/app/assets/index.html $publicdest/index.html +if [ "$2" = "" ]||[ "$2" = "index.html" ]; then + echo copy index.html + cp $src/app/assets/index.html $publicdest/index.html +fi +if [ "$2" != "" ]&&[ "$3" != "" ]; then + if [ ! -d "$3/webui" ]; then + echo mkdir $3/webui + mkdir $3/webui + fi + echo copy $2 nach $3/webui/$2 + cp $publicdest/$2 $3/webui/$2 +fi echo build.sh complete From 1f7a94d49e1bd33010436829c0a4c19dab4c8f51 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 30 Apr 2016 11:13:51 -0400 Subject: [PATCH 40/43] fixed bug causing crash in pqistreamer --- libretroshare/src/pqi/pqistreamer.cc | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 8bf2fafb5..6a63b8f00 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -546,13 +546,6 @@ int pqistreamer::handleoutgoing_locked() if(!dta) break ; - if(slice_size > 0xffff) - { - std::cerr << "(EE) protocol error in pqitreamer: slice size is too large and cannot be encoded." ; - free(mPkt_wpending) ; - mPkt_wpending_size = 0; - } - if(slice_starts && slice_ends) // good old method. Send the packet as is, since it's a full packet. { #ifdef DEBUG_PACKET_SLICING @@ -566,6 +559,13 @@ int pqistreamer::handleoutgoing_locked() } else // partial packet. We make a special header for it and insert it in the stream { + if(slice_size > 0xffff || !mAcceptsPacketSlicing) + { + std::cerr << "(EE) protocol error in pqitreamer: slice size is too large and cannot be encoded." ; + free(mPkt_wpending) ; + mPkt_wpending_size = 0; + return -1 ; + } #ifdef DEBUG_PACKET_SLICING std::cerr << "sending partial slice, packet ID=" << std::hex << slice_packet_id << std::dec << ", size=" << slice_size << std::endl; #endif From 01605e5a59fc096c1820e3a25507a5ae5887c939 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 30 Apr 2016 11:19:21 -0400 Subject: [PATCH 41/43] updated description f packet slicing in comments section of pqistreamer --- libretroshare/src/pqi/pqistreamer.cc | 40 +++++++++++++--------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/libretroshare/src/pqi/pqistreamer.cc b/libretroshare/src/pqi/pqistreamer.cc index 6a63b8f00..acbaffcb7 100644 --- a/libretroshare/src/pqi/pqistreamer.cc +++ b/libretroshare/src/pqi/pqistreamer.cc @@ -423,33 +423,29 @@ time_t pqistreamer::getLastIncomingTS() // Packet slicing: // // Old : 02 0014 03 00000026 [data, 26 bytes] => [version 1B] [service 2B][subpacket 1B] [size 4B] -// New1: fv 0014 03 xxxxx sss [data, sss bytes] => [flags 0.5B version 0.5B] [service 2B][subpacket 1B] [packet counter 2.5B size 1.5B] -// New2: pp ff xxxxxxxx ssss [data, sss bytes] => [flags 1B] [protocol version 1B] [2^32 packet count] [2^16 size] +// New2: pp ff xxxxxxxx ssss [data, sss bytes] => [protocol version 1B] [flags 1B] [2^32 packet count] [2^16 size] // -// Flags: 0x1 => incomplete packet continued after -// Flags: 0x2 => packet ending a previously incomplete packet +// Encode protocol on 1.0 Bytes ( 8 bits) +// Encode flags on 1.0 Bytes ( 8 bits) +// 0x01 => incomplete packet continued after +// 0x02 => packet ending a previously incomplete packet // -// - backward compatibility: -// * send one packet with service + subpacket = ffffff. Old peers will silently ignore such packets. +// Encode packet ID on 4.0 Bytes (32 bits) => packet counter = [0...2^32] +// Encode size on 2.0 Bytes (16 bits) => 65536 // max slice size = 65536 +// +// Backward compatibility: +// * send one packet with service + subpacket = aabbcc. Old peers will silently ignore such packets. Full packet header is: 02aabbcc 00000008 // * if received, mark the peer as able to decode the new packet type +// In pqiQoS: +// - limit packet grouping to max size 512. +// - new peers need to read flux, and properly extract partial sizes, and combine packets based on packet counter. +// - on sending, RS grabs slices of max size 1024 from pqiQoS. If smaller, possibly pack them together. +// pqiQoS keeps track of sliced packets and makes sure the output is consistent: +// * when a large packet needs to be send, only takes a slice and return it, and update the remaining part +// * always consider priority when taking new slices => a newly arrived fast packet will always get through. // -// Mode 1: -// - Encode length on 1.5 Bytes (10 bits) => max slice size = 1024 -// - Encode packet ID on 2.5 Bytes (20 bits) => packet counter = [0...1056364] -// Mode 2: -// - Encode protocol on 1.0 Bytes ( 8 bits) -// - Encode flags on 1.0 Bytes ( 8 bits) -// - Encode packet ID on 4.0 Bytes (32 bits) => packet counter = [0...2^32] -// - Encode size on 2.0 Bytes (16 bits) => 65536 // max slice size = 65536 +// Max slice size should be customisable, depending on bandwidth. To be tested... // -// - limit packet grouping to max size 1024. -// - new peers need to read flux, and properly extract partial sizes, and combine packets based on packet counter. -// - on sending, RS should grab slices of max size 1024 from pqiQoS. If smaller, possibly pack them together. -// pqiQoS keeps track of sliced packets and makes sure the output is consistent: -// * when a large packet needs to be send, only takes a slice and return it, and update the remaining part -// * always consider priority when taking new slices => a newly arrived fast packet will always get through. -// -// Max slice size should be customisable, depending on bandwidth. int pqistreamer::handleoutgoing_locked() { From be7e87cb1e72effdc182058fa910ee05c615d73d Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 30 Apr 2016 17:25:24 -0400 Subject: [PATCH 42/43] use RsTemporaryMemory in signature verification --- libretroshare/src/gxs/gxssecurity.cc | 51 +++++++++++++++------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index e9dfc5567..d061444e1 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -44,13 +44,12 @@ static RsGxsId getRsaKeyFingerprint(RSA *pubkey) int lenn = BN_num_bytes(pubkey -> n); int lene = BN_num_bytes(pubkey -> e); - unsigned char *tmp = new unsigned char[lenn+lene]; + RsTemporaryMemory tmp(lenn+lene) ; BN_bn2bin(pubkey -> n, tmp); BN_bn2bin(pubkey -> e, &tmp[lenn]); - Sha1CheckSum s = RsDirUtil::sha1sum(tmp,lenn+lene) ; - delete[] tmp ; + Sha1CheckSum s = RsDirUtil::sha1sum(tmp,lenn+lene) ; // Copy first CERTSIGNLEN bytes from the hash of the public modulus and exponent // We should not be using strings here, but a real ID. To be done later. @@ -363,35 +362,41 @@ bool GxsSecurity::validateNxsMsg(const RsNxsMsg& msg, const RsTlvKeySignature& s RsGxsMessageId msgId = msgMeta.mMsgId, origMsgId = msgMeta.mOrigMsgId; msgMeta.mOrigMsgId.clear(); msgMeta.mMsgId.clear(); + int signOk = 0 ; - uint32_t metaDataLen = msgMeta.serial_size(); - uint32_t allMsgDataLen = metaDataLen + msg.msg.bin_len; - char* metaData = new char[metaDataLen]; - char* allMsgData = new char[allMsgDataLen]; // msgData + metaData + { + EVP_PKEY *signKey = EVP_PKEY_new(); + EVP_PKEY_assign_RSA(signKey, rsakey); + EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); - msgMeta.serialise(metaData, &metaDataLen); + uint32_t metaDataLen = msgMeta.serial_size(); + uint32_t allMsgDataLen = metaDataLen + msg.msg.bin_len; - // copy msg data and meta in allmsgData buffer - memcpy(allMsgData, msg.msg.bin_data, msg.msg.bin_len); - memcpy(allMsgData+(msg.msg.bin_len), metaData, metaDataLen); + RsTemporaryMemory metaData(metaDataLen) ; + RsTemporaryMemory allMsgData(allMsgDataLen) ; - delete[] metaData ; + if(!metaData || !allMsgData) + { + std::cerr << "(EE) Cannot allocate temporary memory for signature checking. Sizes=" << metaDataLen << ", " << allMsgDataLen << ". Out of memory??" << std::endl; + return false ; + } + msgMeta.serialise(metaData, &metaDataLen); - EVP_PKEY *signKey = EVP_PKEY_new(); - EVP_PKEY_assign_RSA(signKey, rsakey); + // copy msg data and meta in allmsgData buffer + memcpy(allMsgData, msg.msg.bin_data, msg.msg.bin_len); + memcpy(allMsgData+(msg.msg.bin_len), metaData, metaDataLen); - /* calc and check signature */ - EVP_MD_CTX *mdctx = EVP_MD_CTX_create(); + /* calc and check signature */ - EVP_VerifyInit(mdctx, EVP_sha1()); - EVP_VerifyUpdate(mdctx, allMsgData, allMsgDataLen); - int signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey); + EVP_VerifyInit(mdctx, EVP_sha1()); + EVP_VerifyUpdate(mdctx, allMsgData, allMsgDataLen); - delete[] allMsgData ; + signOk = EVP_VerifyFinal(mdctx, sigbuf, siglen, signKey); - /* clean up */ - EVP_PKEY_free(signKey); - EVP_MD_CTX_destroy(mdctx); + /* clean up */ + EVP_PKEY_free(signKey); + EVP_MD_CTX_destroy(mdctx); + } msgMeta.mOrigMsgId = origMsgId; msgMeta.mMsgId = msgId; From 7914aeb60a123680a54e4ce33b7a29fb8810c8b9 Mon Sep 17 00:00:00 2001 From: csoler Date: Sat, 30 Apr 2016 17:33:47 -0400 Subject: [PATCH 43/43] use RsTemporaryMemory in signature verification --- libretroshare/src/gxs/gxssecurity.cc | 4 +--- libretroshare/src/util/rsmemory.cc | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/libretroshare/src/gxs/gxssecurity.cc b/libretroshare/src/gxs/gxssecurity.cc index d061444e1..7440a9b40 100644 --- a/libretroshare/src/gxs/gxssecurity.cc +++ b/libretroshare/src/gxs/gxssecurity.cc @@ -376,10 +376,8 @@ bool GxsSecurity::validateNxsMsg(const RsNxsMsg& msg, const RsTlvKeySignature& s RsTemporaryMemory allMsgData(allMsgDataLen) ; if(!metaData || !allMsgData) - { - std::cerr << "(EE) Cannot allocate temporary memory for signature checking. Sizes=" << metaDataLen << ", " << allMsgDataLen << ". Out of memory??" << std::endl; return false ; - } + msgMeta.serialise(metaData, &metaDataLen); // copy msg data and meta in allmsgData buffer diff --git a/libretroshare/src/util/rsmemory.cc b/libretroshare/src/util/rsmemory.cc index 46ffd4a58..6aab9dfef 100644 --- a/libretroshare/src/util/rsmemory.cc +++ b/libretroshare/src/util/rsmemory.cc @@ -13,7 +13,7 @@ void *rs_malloc(size_t size) if(size > SAFE_MEMALLOC_THRESHOLD) { - std::cerr << "(EE) Memory allocation error. A chunk of size 0 was requested. Callstack:" << std::endl; + std::cerr << "(EE) Memory allocation error. A chunk of size larger than " << SAFE_MEMALLOC_THRESHOLD << " was requested. Callstack:" << std::endl; print_stacktrace() ; return NULL ; }