From 148d1310a2c8d93f0ea0d819a8089b6842639f42 Mon Sep 17 00:00:00 2001 From: thunder2 Date: Fri, 10 Aug 2012 18:06:29 +0000 Subject: [PATCH] Added new button to open the link of the message in browser or copy the link of the message. Added "RSS: " for the forum feeds. Parse the feedburner:origLink in the rss feed. Moved download functions to a new class CURLWrapper for easy use. Added two new functions (currently only for local feeds for testing): - embed images into the message (works for Qt 4.7 and higher) - save complete web page git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs-b1@5399 b45a01b8-16f6-495d-af2f-9b41ad6348cc --- plugins/FeedReader/FeedReader.pro | 4 +- plugins/FeedReader/gui/AddFeedDialog.cpp | 32 +- plugins/FeedReader/gui/AddFeedDialog.h | 1 + plugins/FeedReader/gui/AddFeedDialog.ui | 31 +- plugins/FeedReader/gui/FeedReaderDialog.cpp | 60 ++- plugins/FeedReader/gui/FeedReaderDialog.h | 4 +- plugins/FeedReader/gui/FeedReaderDialog.ui | 22 +- plugins/FeedReader/gui/FeedReader_images.qrc | 1 + plugins/FeedReader/gui/images/Link.png | Bin 0 -> 2813 bytes plugins/FeedReader/interface/rsFeedReader.h | 6 +- plugins/FeedReader/services/p3FeedReader.cc | 181 +++++-- plugins/FeedReader/services/p3FeedReader.h | 5 +- .../FeedReader/services/p3FeedReaderThread.cc | 500 ++++++++++++------ .../FeedReader/services/p3FeedReaderThread.h | 6 +- .../FeedReader/services/rsFeedReaderItems.h | 4 +- .../FeedReader/services/util/CURLWrapper.cc | 136 +++++ .../FeedReader/services/util/CURLWrapper.h | 50 ++ 17 files changed, 806 insertions(+), 237 deletions(-) create mode 100644 plugins/FeedReader/gui/images/Link.png create mode 100644 plugins/FeedReader/services/util/CURLWrapper.cc create mode 100644 plugins/FeedReader/services/util/CURLWrapper.h diff --git a/plugins/FeedReader/FeedReader.pro b/plugins/FeedReader/FeedReader.pro index 64aaa1612..769797a66 100644 --- a/plugins/FeedReader/FeedReader.pro +++ b/plugins/FeedReader/FeedReader.pro @@ -6,6 +6,7 @@ SOURCES = FeedReaderPlugin.cpp \ services/p3FeedReader.cc \ services/p3FeedReaderThread.cc \ services/rsFeedReaderItems.cc \ + services/util/CURLWrapper.cc \ gui/FeedReaderDialog.cpp \ gui/AddFeedDialog.cpp \ gui/FeedReaderNotify.cpp \ @@ -16,6 +17,7 @@ HEADERS = FeedReaderPlugin.h \ services/p3FeedReader.h \ services/p3FeedReaderThread.h \ services/rsFeedReaderItems.h \ + services/util/CURLWrapper.h \ gui/FeedReaderDialog.h \ gui/AddFeedDialog.h \ gui/FeedReaderNotify.h \ @@ -39,7 +41,7 @@ linux-* { } win32 { - DEFINES += CURL_STATICLIB + DEFINES += CURL_STATICLIB LIBXML_STATIC CURL_DIR = ../../../curl-7.26.0 LIBXML2_DIR = ../../../libxml2-2.8.0 diff --git a/plugins/FeedReader/gui/AddFeedDialog.cpp b/plugins/FeedReader/gui/AddFeedDialog.cpp index 5013eaf5a..ac0b6106d 100644 --- a/plugins/FeedReader/gui/AddFeedDialog.cpp +++ b/plugins/FeedReader/gui/AddFeedDialog.cpp @@ -46,12 +46,17 @@ AddFeedDialog::AddFeedDialog(RsFeedReader *feedReader, QWidget *parent) connect(ui->useStandardProxyCheckBox, SIGNAL(toggled(bool)), this, SLOT(useStandardProxyToggled())); connect(ui->typeForumRadio, SIGNAL(toggled(bool)), this, SLOT(typeForumToggled())); + /* currently only for loacl feeds */ + connect(ui->saveCompletePageCheckBox, SIGNAL(toggled(bool)), this, SLOT(denyForumToggled())); + connect(ui->embedImagesCheckBox, SIGNAL(toggled(bool)), this, SLOT(denyForumToggled())); + connect(ui->urlLineEdit, SIGNAL(textChanged(QString)), this, SLOT(validate())); connect(ui->nameLineEdit, SIGNAL(textChanged(QString)), this, SLOT(validate())); connect(ui->useInfoFromFeedCheckBox, SIGNAL(toggled(bool)), this, SLOT(validate())); + connect(ui->typeLocalRadio, SIGNAL(toggled(bool)), this, SLOT(validate())); + connect(ui->typeForumRadio, SIGNAL(toggled(bool)), this, SLOT(validate())); ui->activatedCheckBox->setChecked(true); - ui->typeLocalRadio->setChecked(true); ui->forumComboBox->setEnabled(false); ui->useInfoFromFeedCheckBox->setChecked(true); ui->updateForumInfoCheckBox->setEnabled(false); @@ -124,6 +129,16 @@ void AddFeedDialog::typeForumToggled() ui->updateForumInfoCheckBox->setEnabled(checked); } +void AddFeedDialog::denyForumToggled() +{ + if (ui->saveCompletePageCheckBox->isChecked() || ui->embedImagesCheckBox->isChecked()) { + ui->typeForumRadio->setEnabled(false); + ui->typeLocalRadio->setChecked(true); + } else { + ui->typeForumRadio->setEnabled(true); + } +} + void AddFeedDialog::validate() { bool ok = true; @@ -134,6 +149,9 @@ void AddFeedDialog::validate() if (ui->nameLineEdit->text().isEmpty() && !ui->useInfoFromFeedCheckBox->isChecked()) { ok = false; } + if (!ui->typeLocalRadio->isChecked() && !ui->typeForumRadio->isChecked()) { + ok = false; + } ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(ok); } @@ -196,6 +214,8 @@ bool AddFeedDialog::fillFeed(const std::string &feedId) ui->useInfoFromFeedCheckBox->setChecked(feedInfo.flag.infoFromFeed); ui->updateForumInfoCheckBox->setChecked(feedInfo.flag.updateForumInfo); ui->activatedCheckBox->setChecked(!feedInfo.flag.deactivated); + ui->embedImagesCheckBox->setChecked(feedInfo.flag.embedImages); + ui->saveCompletePageCheckBox->setChecked(feedInfo.flag.saveCompletePage); ui->descriptionPlainTextEdit->setPlainText(QString::fromUtf8(feedInfo.description.c_str())); @@ -206,6 +226,8 @@ bool AddFeedDialog::fillFeed(const std::string &feedId) if (feedInfo.flag.forum) { ui->typeForumRadio->setChecked(true); + ui->saveCompletePageCheckBox->setEnabled(false); + ui->embedImagesCheckBox->setEnabled(false); if (feedInfo.forumId.empty()) { ui->forumNameLabel->setText(tr("Not yet created")); @@ -259,13 +281,17 @@ void AddFeedDialog::createFeed() feedInfo.flag.infoFromFeed = ui->useInfoFromFeedCheckBox->isChecked(); feedInfo.flag.updateForumInfo = ui->updateForumInfoCheckBox->isChecked() && ui->updateForumInfoCheckBox->isEnabled(); feedInfo.flag.deactivated = !ui->activatedCheckBox->isChecked(); + feedInfo.flag.embedImages = ui->embedImagesCheckBox->isChecked(); + feedInfo.flag.saveCompletePage = ui->saveCompletePageCheckBox->isChecked(); feedInfo.description = ui->descriptionPlainTextEdit->toPlainText().toUtf8().constData(); feedInfo.flag.forum = ui->typeForumRadio->isChecked(); if (mFeedId.empty()) { - /* set forum (only when create a new feed) */ - feedInfo.forumId = ui->forumComboBox->itemData(ui->forumComboBox->currentIndex()).toString().toStdString(); + if (feedInfo.flag.forum) { + /* set forum (only when create a new feed) */ + feedInfo.forumId = ui->forumComboBox->itemData(ui->forumComboBox->currentIndex()).toString().toStdString(); + } } feedInfo.flag.authentication = ui->useAuthenticationCheckBox->isChecked(); diff --git a/plugins/FeedReader/gui/AddFeedDialog.h b/plugins/FeedReader/gui/AddFeedDialog.h index c117cd789..f08d64611 100644 --- a/plugins/FeedReader/gui/AddFeedDialog.h +++ b/plugins/FeedReader/gui/AddFeedDialog.h @@ -50,6 +50,7 @@ private slots: void useStandardUpdateIntervalToggled(); void useStandardProxyToggled(); void typeForumToggled(); + void denyForumToggled(); void validate(); void createFeed(); diff --git a/plugins/FeedReader/gui/AddFeedDialog.ui b/plugins/FeedReader/gui/AddFeedDialog.ui index ed2587dfa..b09be53a8 100644 --- a/plugins/FeedReader/gui/AddFeedDialog.ui +++ b/plugins/FeedReader/gui/AddFeedDialog.ui @@ -7,7 +7,7 @@ 0 0 715 - 559 + 605 @@ -289,13 +289,6 @@ p, li { white-space: pre-wrap; } 6 - - - - Local Feed - - - @@ -323,6 +316,13 @@ p, li { white-space: pre-wrap; } + + + + Local Feed + + + @@ -366,6 +366,20 @@ p, li { white-space: pre-wrap; } + + + + Embed images (experimental for local feeds) + + + + + + + Save complete web page (experimental for local feeds) + + + @@ -416,7 +430,6 @@ p, li { white-space: pre-wrap; } urlLineEdit nameLineEdit descriptionPlainTextEdit - typeLocalRadio typeForumRadio forumComboBox activatedCheckBox diff --git a/plugins/FeedReader/gui/FeedReaderDialog.cpp b/plugins/FeedReader/gui/FeedReaderDialog.cpp index d93bf83b8..ef4438e1a 100644 --- a/plugins/FeedReader/gui/FeedReaderDialog.cpp +++ b/plugins/FeedReader/gui/FeedReaderDialog.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include "FeedReaderDialog.h" #include "ui_FeedReaderDialog.h" @@ -118,6 +119,7 @@ FeedReaderDialog::FeedReaderDialog(RsFeedReader *feedReader, QWidget *parent) connect(ui->filterLineEdit, SIGNAL(textChanged(QString)), this, SLOT(filterItems(QString))); connect(ui->filterColumnComboBox, SIGNAL(currentIndexChanged(int)), this, SLOT(filterColumnChanged())); + connect(ui->linkButton, SIGNAL(clicked()), this, SLOT(openLinkMsg())); connect(ui->expandButton, SIGNAL(clicked()), this, SLOT(toggleMsgText())); mFeedCompareRole = new RSTreeWidgetItemCompareRole; @@ -168,6 +170,18 @@ FeedReaderDialog::FeedReaderDialog(RsFeedReader *feedReader, QWidget *parent) /* initialize feed list */ ui->feedTreeWidget->sortItems(COLUMN_FEED_NAME, Qt::AscendingOrder); + /* build menu for link button */ + QMenu *menu = new QMenu(this); + QAction *action = menu->addAction(tr("Open link in browser"), this, SLOT(openLinkMsg())); + menu->addAction(tr("Copy link to clipboard"), this, SLOT(copyLinkMsg())); + + QFont font = action->font(); + font.setBold(true); + action->setFont(font); + + ui->linkButton->setMenu(menu); + ui->linkButton->setEnabled(false); + ui->msgTreeWidget->installEventFilter(this); } @@ -351,7 +365,7 @@ void FeedReaderDialog::msgTreeCustomPopupMenu(QPoint /*point*/) contextMnu.addSeparator(); - action = contextMnu.addAction(QIcon(""), tr("Copy link"), this, SLOT(copyLinkMsg())); + action = contextMnu.addAction(QIcon(""), tr("Copy link"), this, SLOT(copyLinskMsg())); action->setEnabled(!selectedItems.empty()); action = contextMnu.addAction(QIcon(""), tr("Remove"), this, SLOT(removeMsg())); @@ -522,10 +536,14 @@ void FeedReaderDialog::updateFeedItem(QTreeWidgetItem *item, FeedInfo &info) case FeedInfo::WAITING: break; case FeedInfo::WAITING_TO_DOWNLOAD: + workState = tr("waiting for download"); + break; case FeedInfo::DOWNLOADING: - workState = tr("loading"); + workState = tr("downloading"); break; case FeedInfo::WAITING_TO_PROCESS: + workState = tr("waiting for process"); + break; case FeedInfo::PROCESSING: workState = tr("processing"); break; @@ -791,6 +809,7 @@ void FeedReaderDialog::msgItemChanged() ui->msgTitle->clear(); // ui->msgLink->clear(); ui->msgText->clear(); + ui->linkButton->setEnabled(false); return; } @@ -800,6 +819,7 @@ void FeedReaderDialog::msgItemChanged() ui->msgTitle->clear(); // ui->msgLink->clear(); ui->msgText->clear(); + ui->linkButton->setEnabled(false); return; } @@ -809,6 +829,7 @@ void FeedReaderDialog::msgItemChanged() ui->msgTitle->clear(); // ui->msgLink->clear(); ui->msgText->clear(); + ui->linkButton->setEnabled(false); return; } @@ -832,7 +853,8 @@ void FeedReaderDialog::msgItemChanged() ui->msgText->setHtml(msgTxt); ui->msgTitle->setText(QString::fromUtf8(msgInfo.title.c_str())); -// ui->msgLink->setHtml(RsHtml().formatText(NULL, QString::fromUtf8(msgInfo.link.c_str()), RSHTML_FORMATTEXT_EMBED_LINKS)); + + ui->linkButton->setEnabled(!msgInfo.link.empty()); } void FeedReaderDialog::setMsgAsReadUnread(QList &rows, bool read) @@ -1049,7 +1071,7 @@ void FeedReaderDialog::markAllAsReadMsg() setMsgAsReadUnread(items, true); } -void FeedReaderDialog::copyLinkMsg() +void FeedReaderDialog::copyLinksMsg() { QString links; @@ -1086,3 +1108,33 @@ void FeedReaderDialog::removeMsg() } mFeedReader->removeMsgs(feedId, msgIds); } + +void FeedReaderDialog::copyLinkMsg() +{ + QTreeWidgetItem *item = ui->msgTreeWidget->currentItem(); + if (!item) { + return; + } + + QString link = item->data(COLUMN_MSG_DATA, ROLE_MSG_LINK).toString(); + if (link.isEmpty()) { + return; + } + + QApplication::clipboard()->setText(link); +} + +void FeedReaderDialog::openLinkMsg() +{ + QTreeWidgetItem *item = ui->msgTreeWidget->currentItem(); + if (!item) { + return; + } + + QString link = item->data(COLUMN_MSG_DATA, ROLE_MSG_LINK).toString(); + if (link.isEmpty()) { + return; + } + + QDesktopServices::openUrl(QUrl(link)); +} diff --git a/plugins/FeedReader/gui/FeedReaderDialog.h b/plugins/FeedReader/gui/FeedReaderDialog.h index 1bb6c14b8..7eae19f3e 100644 --- a/plugins/FeedReader/gui/FeedReaderDialog.h +++ b/plugins/FeedReader/gui/FeedReaderDialog.h @@ -64,8 +64,10 @@ private slots: void markAsReadMsg(); void markAsUnreadMsg(); void markAllAsReadMsg(); - void copyLinkMsg(); + void copyLinksMsg(); void removeMsg(); + void openLinkMsg(); + void copyLinkMsg(); /* FeedReaderNotify */ void feedChanged(const QString &feedId, int type); diff --git a/plugins/FeedReader/gui/FeedReaderDialog.ui b/plugins/FeedReader/gui/FeedReaderDialog.ui index 95c076e29..7dbaeea91 100644 --- a/plugins/FeedReader/gui/FeedReaderDialog.ui +++ b/plugins/FeedReader/gui/FeedReaderDialog.ui @@ -268,8 +268,8 @@ border: 1px solid #CCCCCC;} - - + + @@ -289,7 +289,7 @@ border: 1px solid #CCCCCC;} - + QLabel#msgTitle{ @@ -305,7 +305,21 @@ background: white;} - + + + + + :/images/Link.png:/images/Link.png + + + QToolButton::MenuButtonPopup + + + true + + + + diff --git a/plugins/FeedReader/gui/FeedReader_images.qrc b/plugins/FeedReader/gui/FeedReader_images.qrc index 6acbedfab..e1a6c5918 100644 --- a/plugins/FeedReader/gui/FeedReader_images.qrc +++ b/plugins/FeedReader/gui/FeedReader_images.qrc @@ -8,6 +8,7 @@ images/FeedErrorOverlay.png images/FolderAdd.png images/FeedAdd.png + images/Link.png images/Update.png diff --git a/plugins/FeedReader/gui/images/Link.png b/plugins/FeedReader/gui/images/Link.png new file mode 100644 index 0000000000000000000000000000000000000000..8f8fd3bc1bbb57a5d28affaaa6837f5198fcb188 GIT binary patch literal 2813 zcmVPx#AY({UO#lFTCIA3{ga82g0001h=l}q9FaQARU;qF* zm;eA5aGbhPJOBUy24YJ`L;wH)0002_L%V+f000SaNLh0L002k;002k;M#*bF000V9 zNklqy?vji#xENkpn;NaAQL+@&i?JseQ~(jWbN zeBx`*@vnb;&quN=e%6KG{;Q|R;(k%e%kBLK-Qks$aHM784MwO%Bl1RBn0S6eP2Rb# z*NRG95ZHHQx_SN0wX>7erA7Yj_kQdDw6A}f;*m%HmP5k_d1v9D`%^i%qdDz;I&I9~ z-8K7e(wJUdrFqk6*-aHiL%Fp~7PS}KQn|C;#qo2eR}=iC=k(fBC}i ze6$Rde;@h0c#H#XQE%>h`j(>m`sZXic(@<)V=Kv0Yo&caC9zFaA3?Fm$zH->CV29k z=lZUV-h1EeZH+tK?bCyaeQYwT9<@b%QA!!U`K90g2?FYa2j_qJ7ui=GPdA&-pPIE# z|6%N2_|0Zk-PG1cGA~aEFFHacDMh-Uk(y&JOSoJnoUI@#Cqhi6w|OqDJXFryZ6d$6 zWZIY4rkAVryS{jLIQ{KsKb*jKz8fh&_7IKofZ32QJ}_>-@)h4d^I(#XIz7=5P4Ov| zxs8>4B-%-fjbxSeq{qNXE>((QNDw68AQmo$vKF-+wz30fjy5}k1(DhF7tZ1)Ial|U;V=7FMMv5Yp*{6fCRYX zfxltlFMfkf`Qpm1I{n3J>GXZ+{7$Nc7pX8}LEp&DS)b#rKtJ7&N>=IVoMjVe<{FhK zM2O%#gg^*_=_q5eK8M>lM0w?4Yw*T_hlg)BKfiqQWBqDS0^i;vaP;J#kDG?O-7^eN;)tXZU~6LJAEMna4P1W^hQf*2&h z8MJFrUGB5-N@H%i<$o=kcb}Zp1AXNFr}w_0Y$%#{Oc*_%AMs}AGq!nc$*8HA)f(#( z<}{>@A+;H^Emw$o87UeBp|Ku92m}uyMq-Q*BOwMN5noHjI}H+HIdXDW?oNhBc4o!H zwVJ$;cj8)h_)F+0JZq09I_vt6$C}%0Ec9erP&tDUlCE*A2ZapF2yqi_(VziLLu6nh zMIEr#V_g6r5I*94B=~@xDXzS>pTj4n-1h0~462Rn;+DH-d(#~M?(hEf+NnSNDmq4w zO6mEUyZkKleHpQv8Nzl*+L~#j$3=B3yYeUvmC_9X1I&D6TnB7naizsp4s1X=PjC_M z0?tNUB}f_*RBH#5j71xHxCrU(xBtPPJ)JF3X+$Iv7M%;nTveMeF_^8BWbP!{;Q*?` zmcNIo$Z6S%p^-=dHbAUB6c#_LsS5|CN7Vu9K(GN{2ZSt9$qtipKp6^z$yzb>KbI{| z+T^y<8)sjqzjAawSij(`lWNpKUps~m$8kwcZ8TMEP{oR>bQmKMLJ;D+1WHFWvDk@2 z7M@sms64KUxGK=-PiXaa@v+86PZ7(c@Os%v-L-1Er8|cX%Me`GJJ*U6h1}^tU4q)e zPnIZm6Jji|rJ%|c+DN3_>)^o#Lg}G!h^a+SEOF{#7N}>QYUarIRiGR;Xh!9f&FMr8 zhM_l|s#fRdtLR`IsAz(BDsDI7YM<)Rc}z5faUVaNq1YkTmTH<(CmN~bUib15QF}z` zQB#MXI^xt3CXQ<2s3(r`0IqIVq;bIJWXxJ&#dx4qIcuo3NsSIw6-AcSk?2Dq@#hwh z$q>JP7g-e48(mE8$T~Y%)x^mhq4vrkLqytusQ0oE3P+ea?ATI{EcMJ0MsQ)h!n+-& z)0$b)jDC`OS7<*uGml<-`+wr{t#^)u6jKC*nI>L5fG{OeO$eo@sugam5XK>NiO>=y zfe5&Wu%1|YLg{f6OEs#gCl+rlaby^tJ47We;3omKb}UNY7>8;$)32RCAHDY!#qMN7 zD!HyS$B9D5Sq9g-Fjpe-v!E)XOc1t*ta=FFKtzQgLWp~Bq4M}5V5g3HX7P1RjFwPJ z+?k`ONkmowIhuE*)fFq(hf1{2)u@2LrcCtPB=VEdi44GxQ>+rWeXmgNe+^~8sC{(Q z3i|rT2sXak`RxGLalhj)$`EkarljyN@N?j|BN(J+a%dNWq7QPgcTLET%&zqEAYxRwYs@t%-Bjm@d9R zQ4I;zUazmMrC^JqeV*9ZMh)f(lML_(;St&+l|yKU)B%|UR01j$!sNOTLi~rT?Y9n| zxOGyDXQ+PX6Nng*h;i0T8(EgznHt^Bj1pPDBFhxcZS0GQRWl*HF8IXE=JxL(J|w;o2L6 z?Zl|ViIK;Q7ABQ>Gz~#)4n>S^uiicqy?`JI1Z;B~J9ia#;1VpXW9VRfk0eV-n+a(n zA#10kO|yv*|5#V<(e7fmD8o^<-ulMgM0Dl+6C6ozCsuAGwAdC>+<;&OFO>E!Ok*wVy>*VC!Mob!dR#($!wl2Q@52qjh`Wtq(>#n}`z3@}V^1B}R6QYlq z(Bc89`LdGglvJ`U=ch8BI3Im~iiSba&D5FcB(neQl-Wg=g-l=q|4!DhAqN*|N z6qR;RIwv2kl5F2V66Z0dgKnll7+IF(=eXTl~ zPzh1Q+J$nroDMc7o0l&QufKD(*xA@5_~H8;Pk @@ -68,6 +68,8 @@ public: flag.deactivated = false; flag.forum = false; flag.updateForumInfo = false; + flag.embedImages = false; + flag.saveCompletePage = false; } std::string feedId; @@ -98,6 +100,8 @@ public: bool deactivated : 1; bool forum : 1; bool updateForumInfo : 1; + bool embedImages : 1; + bool saveCompletePage : 1; } flag; }; diff --git a/plugins/FeedReader/services/p3FeedReader.cc b/plugins/FeedReader/services/p3FeedReader.cc index 0ae7f6ec0..76b16ea65 100644 --- a/plugins/FeedReader/services/p3FeedReader.cc +++ b/plugins/FeedReader/services/p3FeedReader.cc @@ -30,6 +30,7 @@ RsFeedReader *rsFeedReader = NULL; #define FEEDREADER_CLEAN_INTERVAL 1 * 60 * 60 // check every hour +#define FEEDREADER_FORUM_PREFIX L"RSS: " /********* * #define FEEDREADER_DEBUG @@ -91,6 +92,8 @@ static void feedToInfo(const RsFeedReaderFeed *feed, FeedInfo &info) info.flag.deactivated = (feed->flag & RS_FEED_FLAG_DEACTIVATED); info.flag.forum = (feed->flag & RS_FEED_FLAG_FORUM); info.flag.updateForumInfo = (feed->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO); + info.flag.embedImages = (feed->flag & RS_FEED_FLAG_EMBED_IMAGES); + info.flag.saveCompletePage = (feed->flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE); switch (feed->workstate) { case RsFeedReaderFeed::WAITING: @@ -151,6 +154,12 @@ static void infoToFeed(const FeedInfo &info, RsFeedReaderFeed *feed, bool add) if (info.flag.deactivated) { feed->flag |= RS_FEED_FLAG_DEACTIVATED; } + if (info.flag.embedImages) { + feed->flag |= RS_FEED_FLAG_EMBED_IMAGES; + } + if (info.flag.saveCompletePage) { + feed->flag |= RS_FEED_FLAG_SAVE_COMPLETE_PAGE; + } if (add) { /* only set when adding a new feed */ if (info.flag.folder) { @@ -950,6 +959,11 @@ int p3FeedReader::tick() } else { updateInterval = fi->updateInterval; } + + if (updateInterval == 0) { + continue; + } + if (fi->lastUpdate == 0 || fi->lastUpdate + (long) updateInterval <= currentTime) { /* add to download list */ feedToDownload.push_back(fi->feedId); @@ -1340,7 +1354,6 @@ void p3FeedReader::onDownloadError(const std::string &feedId, p3FeedReaderThread case p3FeedReaderThread::DOWNLOAD_ERROR: fi->errorState = RS_FEED_ERRORSTATE_DOWNLOAD_ERROR; break; - case p3FeedReaderThread::DOWNLOAD_ERROR_INIT: case p3FeedReaderThread::DOWNLOAD_INTERNAL_ERROR: fi->errorState = RS_FEED_ERRORSTATE_DOWNLOAD_INTERNAL_ERROR; break; @@ -1428,15 +1441,13 @@ bool p3FeedReader::getFeedToProcess(RsFeedReaderFeed &feed) return true; } -void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list &msgs) +bool p3FeedReader::onProcessSuccess_filterMsg(const std::string &feedId, std::list &msgs) { #ifdef FEEDREADER_DEBUG - std::cerr << "p3FeedReader::onProcessSuccess - feed " << feedId << " got " << msgs.size() << " messages" << std::endl; + std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - feed " << feedId << " got " << msgs.size() << " messages" << std::endl; #endif - std::list addedMsgs; - std::string forumId; - std::list forumMsgs; + bool result = true; { RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/ @@ -1446,9 +1457,9 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listsecond; @@ -1458,6 +1469,10 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listforumId.empty()) { /* create new forum */ + std::wstring forumName; + librs::util::ConvertUtf8ToUtf16(fi->name, forumName); + forumName.insert(0, FEEDREADER_FORUM_PREFIX); + long todo; // search for existing forum? /* search for existing own forum */ @@ -1472,22 +1487,18 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listname, forumName); std::wstring forumDescription; librs::util::ConvertUtf8ToUtf16(fi->description, forumDescription); /* create anonymous public forum */ fi->forumId = rsForums->createForum(forumName, forumDescription, RS_DISTRIB_PUBLIC | RS_DISTRIB_AUTHEN_ANON); - forumId = fi->forumId; - if (fi->forumId.empty()) { errorState = RS_FEED_ERRORSTATE_FORUM_CREATE; #ifdef FEEDREADER_DEBUG - std::cerr << "p3FeedReader::onProcessSuccess - can't create forum for feed " << feedId << " (" << it->second->name << ") - ignore all messages" << std::endl; + std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - can't create forum for feed " << feedId << " (" << it->second->name << ") - ignore all messages" << std::endl; } else { - std::cerr << "p3FeedReader::onProcessSuccess - forum " << forumId << " (" << fi->name << ") created" << std::endl; + std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - forum " << forumId << " (" << fi->name << ") created" << std::endl; #endif } } else { @@ -1498,8 +1509,6 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listforumId; } } else { errorState = RS_FEED_ERRORSTATE_FORUM_NOT_FOUND; @@ -1508,15 +1517,11 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list::iterator newMsgIt; for (newMsgIt = msgs.begin(); newMsgIt != msgs.end(); ) { RsFeedReaderMsg *miNew = *newMsgIt; - /* search for exisiting msg */ + /* search for existing msg */ std::map::iterator msgIt; for (msgIt = fi->mMsgs.begin(); msgIt != fi->mMsgs.end(); ++msgIt) { RsFeedReaderMsg *mi = msgIt->second; @@ -1525,38 +1530,103 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listmMsgs.end()) { - /* add new msg */ - rs_sprintf(miNew->msgId, "%lu", mNextMsgId++); - if (forum) { - miNew->flag = RS_FEEDMSG_FLAG_DELETED; - forumMsgs.push_back(*miNew); -// miNew->description.clear(); - } else { - miNew->flag = RS_FEEDMSG_FLAG_NEW; - addedMsgs.push_back(miNew->msgId); - } - fi->mMsgs[miNew->msgId] = miNew; + if (msgIt != fi->mMsgs.end()) { + /* msg exists */ + delete(*newMsgIt); newMsgIt = msgs.erase(newMsgIt); - -#ifdef FEEDREADER_DEBUG - ++newMsgs; -#endif } else { - /* msg was updated */ ++newMsgIt; } + } + } else { + result = false; + } + + fi->content.clear(); + fi->errorState = errorState; + fi->errorString.clear(); + + IndicateConfigChanged(); + } + + if (mNotify) { + mNotify->feedChanged(feedId, NOTIFY_TYPE_MOD); + } + + return result; +} + +void p3FeedReader::onProcessSuccess_addMsgs(const std::string &feedId, bool result, std::list &msgs) +{ #ifdef FEEDREADER_DEBUG - std::cerr << "p3FeedReader::onProcessSuccess - feed " << fi->feedId << " (" << fi->name << ") added " << newMsgs << "/" << msgs.size() << " messages" << std::endl; + std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << feedId << " got " << msgs.size() << " messages" << std::endl; +#endif + + std::list addedMsgs; + std::string forumId; + std::list forumMsgs; + + { + RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/ + + /* find feed */ + std::map::iterator it = mFeeds.find(feedId); + if (it == mFeeds.end()) { + /* feed not found */ +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << feedId << " not found" << std::endl; +#endif + return; + } + + RsFeedReaderFeed *fi = it->second; + bool forum = (fi->flag & RS_FEED_FLAG_FORUM); + + if (forum) { + forumId = fi->forumId; + if (forumId.empty()) { + /* don't process messages without forum id */ + result = false; +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << fi->feedId << " (" << fi->name << ") don't process messages without forum id" << std::endl; #endif } } + /* process msgs */ +#ifdef FEEDREADER_DEBUG + uint32_t newMsgs = 0; +#endif + + if (result) { + std::list::iterator newMsgIt; + for (newMsgIt = msgs.begin(); newMsgIt != msgs.end(); ) { + RsFeedReaderMsg *miNew = *newMsgIt; + /* add new msg */ + rs_sprintf(miNew->msgId, "%lu", mNextMsgId++); + if (forum) { + miNew->flag = RS_FEEDMSG_FLAG_DELETED; + forumMsgs.push_back(*miNew); +// miNew->description.clear(); + } else { + miNew->flag = RS_FEEDMSG_FLAG_NEW; + addedMsgs.push_back(miNew->msgId); + } + fi->mMsgs[miNew->msgId] = miNew; + newMsgIt = msgs.erase(newMsgIt); + +#ifdef FEEDREADER_DEBUG + ++newMsgs; +#endif + } +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << fi->feedId << " (" << fi->name << ") added " << newMsgs << "/" << msgs.size() << " messages" << std::endl; +#endif + } + fi->workstate = RsFeedReaderFeed::WAITING; fi->content.clear(); fi->lastUpdate = time(NULL); - fi->errorState = errorState; - fi->errorString.clear(); IndicateConfigChanged(); } @@ -1584,7 +1654,7 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listsetMessageStatus(forumMsgInfo.forumId, forumMsgInfo.msgId, 0, FORUM_MSG_STATUS_MASK); } else { #ifdef FEEDREADER_DEBUG - std::cerr << "p3FeedReader::onProcessSuccess - can't add forum message " << mi.title << " for feed " << forumId << std::endl; + std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - can't add forum message " << mi.title << " for feed " << forumId << std::endl; #endif } } @@ -1652,7 +1722,7 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam { bool changed = false; std::string forumId; - ForumInfo forumInfo; + ForumInfo forumInfoNew; { RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/ @@ -1683,11 +1753,12 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam changed = true; } - if (changed && (fi->flag & RS_FEED_FLAG_FORUM) && (fi->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO) && !fi->forumId.empty()) { + if ((fi->flag & RS_FEED_FLAG_FORUM) && (fi->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO) && !fi->forumId.empty()) { /* change forum too */ forumId = fi->forumId; - librs::util::ConvertUtf8ToUtf16(fi->name, forumInfo.forumName); - librs::util::ConvertUtf8ToUtf16(fi->description, forumInfo.forumDesc); + librs::util::ConvertUtf8ToUtf16(fi->name, forumInfoNew.forumName); + librs::util::ConvertUtf8ToUtf16(fi->description, forumInfoNew.forumDesc); + forumInfoNew.forumName.insert(0, FEEDREADER_FORUM_PREFIX); } } @@ -1700,10 +1771,22 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam } if (!forumId.empty()) { - /* name or description changed, update forum */ - if (!rsForums->setForumInfo(forumId, forumInfo)) { + ForumInfo forumInfo; + if (rsForums->getForumInfo(forumId, forumInfo)) { + if (forumInfo.forumName != forumInfoNew.forumName || forumInfo.forumDesc != forumInfoNew.forumDesc) { + /* name or description changed, update forum */ #ifdef FEEDREADER_DEBUG - std::cerr << "p3FeedReader::setFeed - can't change forum " << forumId << std::endl; + std::cerr << "p3FeedReader::setFeed - change forum " << forumId << std::endl; +#endif + if (!rsForums->setForumInfo(forumId, forumInfoNew)) { +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReader::setFeed - can't change forum " << forumId << std::endl; +#endif + } + } + } else { +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReader::setFeed - can't get forum info " << forumId << std::endl; #endif } } diff --git a/plugins/FeedReader/services/p3FeedReader.h b/plugins/FeedReader/services/p3FeedReader.h index 2e62e16da..e4525a65b 100644 --- a/plugins/FeedReader/services/p3FeedReader.h +++ b/plugins/FeedReader/services/p3FeedReader.h @@ -19,7 +19,7 @@ * Boston, MA 02110-1301, USA. ****************************************************************/ - #ifndef P3_FEEDREADER +#ifndef P3_FEEDREADER #define P3_FEEDREADER #include "retroshare/rsplugin.h" @@ -71,7 +71,8 @@ public: bool getFeedToDownload(RsFeedReaderFeed &feed); void onDownloadSuccess(const std::string &feedId, const std::string &content, std::string &icon); void onDownloadError(const std::string &feedId, p3FeedReaderThread::DownloadResult result, const std::string &error); - void onProcessSuccess(const std::string &feedId, std::list &msgs); + bool onProcessSuccess_filterMsg(const std::string &feedId, std::list &msgs); + void onProcessSuccess_addMsgs(const std::string &feedId, bool result, std::list &msgs); void onProcessError(const std::string &feedId, p3FeedReaderThread::ProcessResult result); bool getFeedToProcess(RsFeedReaderFeed &feed); diff --git a/plugins/FeedReader/services/p3FeedReaderThread.cc b/plugins/FeedReader/services/p3FeedReaderThread.cc index cbc03fe8b..b95589c01 100644 --- a/plugins/FeedReader/services/p3FeedReaderThread.cc +++ b/plugins/FeedReader/services/p3FeedReaderThread.cc @@ -22,9 +22,11 @@ #include "p3FeedReaderThread.h" #include "rsFeedReaderItems.h" #include "util/rsstring.h" +#include "util/CURLWrapper.h" -#include #include +#include +#include #include enum FeedFormat { FORMAT_RSS, FORMAT_RDF }; @@ -32,6 +34,7 @@ enum FeedFormat { FORMAT_RSS, FORMAT_RDF }; /********* * #define FEEDREADER_DEBUG *********/ +#define FEEDREADER_DEBUG p3FeedReaderThread::p3FeedReaderThread(p3FeedReader *feedReader, Type type) : RsThread(), mFeedReader(feedReader), mType(type) { @@ -88,21 +91,34 @@ void p3FeedReaderThread::run() { RsFeedReaderFeed feed; if (mFeedReader->getFeedToProcess(feed)) { - std::list entries; + std::list msgs; std::string error; + std::list::iterator it; - ProcessResult result = process(feed, entries, error); + ProcessResult result = process(feed, msgs, error); if (result == PROCESS_SUCCESS) { - mFeedReader->onProcessSuccess(feed.feedId, entries); + /* first, filter the messages */ + bool result = mFeedReader->onProcessSuccess_filterMsg(feed.feedId, msgs); + if (isRunning()) { + if (result) { + long todo; // process new items + /* second, process the descriptions */ + for (it = msgs.begin(); it != msgs.end(); ++it) { + RsFeedReaderMsg *mi = *it; + processMsg(feed, mi); + } + } + /* third, add messages */ + mFeedReader->onProcessSuccess_addMsgs(feed.feedId, result, msgs); + } } else { mFeedReader->onProcessError(feed.feedId, result); } - std::list::iterator it; - for (it = entries.begin(); it != entries.end(); ++it) { + for (it = msgs.begin(); it != msgs.end(); ++it) { delete (*it); } - entries.clear(); + msgs.clear(); } } break; @@ -114,121 +130,110 @@ void p3FeedReaderThread::run() /****************************** Download ***********************************/ /***************************************************************************/ -static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream) +static bool isContentType(const std::string &contentType, const char *type) { - std::string *s = (std::string*) stream; - s->append ((char*) ptr, size * nmemb); - - return nmemb * size; + return (strncasecmp(contentType.c_str(), type, strlen(type)) == 0); } -static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream) +static bool toBase64(const std::vector &data, std::string &base64) { - std::vector *bytes = (std::vector*) stream; + bool result = false; - std::vector newBytes; - newBytes.resize(size * nmemb); - memcpy(newBytes.data(), ptr, newBytes.size()); - - bytes->insert(bytes->end(), newBytes.begin(), newBytes.end()); - - return nmemb * size; -} - -static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/) -{ - p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp; - - if (!thread->isRunning()) { - /* thread was stopped */ - return 1; + /* Set up a base64 encoding BIO that writes to a memory BIO */ + BIO *b64 = BIO_new(BIO_f_base64()); + if (b64) { + BIO_set_flags(b64, BIO_FLAGS_BASE64_NO_NL); + BIO *bmem = BIO_new(BIO_s_mem()); + if (bmem) { + BIO_set_flags(bmem, BIO_CLOSE); // probably redundant + b64 = BIO_push(b64, bmem); + /* Send the data */ + BIO_write(b64, data.data(), data.size()); + /* Collect the encoded data */ + BIO_flush(b64); + char* temp; + int count = BIO_get_mem_data(bmem, &temp); + if (count && temp) { + base64.assign(temp, count); + result = true; + } + } + BIO_free_all(b64); } - long todo; // show progress in gui - - return 0; + return result; } -static bool getFavicon (std::string url, const std::string &proxy, std::string &icon) +static std::string getBaseLink(std::string link) +{ + size_t found = link.rfind('/'); + if (found != std::string::npos) { + link.erase(found + 1); + } + + return link; +} + +static std::string calculateLink(const std::string &baseLink, const std::string &link) +{ + if (link.substr(0, 7) == "http://") { + /* absolute link */ + return link; + } + + /* calculate link of base link */ + std::string resultLink = baseLink; + + /* link should begin with "http://" */ + if (resultLink.substr(0, 7) != "http://") { + resultLink.insert(0, "http://"); + } + + if (link.empty()) { + /* no link */ + return resultLink; + } + + if (*link.begin() == '/') { + /* link begins with "/" */ + size_t found = resultLink.find('/', 7); + if (found != std::string::npos) { + resultLink.erase(found); + } + } else { + /* check for "/" at the end */ + std::string::reverse_iterator it = resultLink.rend (); + it--; + if (*it != '/') { + resultLink += "/"; + } + } + + resultLink += link; + + return resultLink; +} + +static bool getFavicon(CURLWrapper &CURL, const std::string &url, std::string &icon) { icon.clear(); - if (url.substr(0, 7) == "http://") { - int found = url.find("/", 7); - if (found >= 0) { - url.erase(found, url.length() - found); - } - } else { - return false; - } - bool result = false; - CURL *curl = curl_easy_init(); - if (curl) { - url += "/favicon.ico"; - - std::vector vicon; - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1); - curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionBinary); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &vicon); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); - curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1); - - if (!proxy.empty()) { - curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str()); - } - - CURLcode code = curl_easy_perform(curl); - if (code == CURLE_OK) { - long response; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - if (response == 200) { - char *contentType = NULL; - curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &contentType); - if (contentType && - (strncasecmp(contentType, "image/x-icon", 12) == 0 || - strncasecmp(contentType, "application/octet-stream", 24) == 0 || - strncasecmp(contentType, "text/plain", 10) == 0)) { - if (!vicon.empty()) { - long todo; // check it - // Set up a base64 encoding BIO that writes to a memory BIO - BIO *b64 = BIO_new(BIO_f_base64()); - if (b64) { - BIO *out = BIO_new(BIO_s_mem()); - if (out) { - BIO_set_flags(out, BIO_CLOSE); // probably redundant - b64 = BIO_push(b64, out); - // Send the data - BIO_write(b64, vicon.data(), vicon.size()); - // Collect the encoded data - BIO_flush(b64); - char* temp; - int count = BIO_get_mem_data(out, &temp); - if (count && temp) { - icon = temp; - result = true; - } - } - BIO_free_all(b64); - } - } -// char *encode = NULL; -// size_t encodeSize = 0; -// code = Curl_base64_encode(curl, (const char*) vicon.data(), vicon.size(), &encode, &encodeSize); -// if (code == CURLE_OK && encodeSize) { -// icon = encode; -// free(encode); -// encode = NULL; -// result = true; -// } + std::vector vicon; + CURLcode code = CURL.downloadBinary(calculateLink(url, "/favicon.ico"), vicon); + if (code == CURLE_OK) { + if (CURL.responseCode() == 200) { + std::string contentType = CURL.contentType(); + if (isContentType(contentType, "image/x-icon") || + isContentType(contentType, "application/octet-stream") || + isContentType(contentType, "text/plain")) { + if (!vicon.empty()) { + long todo; // check it + result = toBase64(vicon, icon); } } } - - curl_easy_cleanup(curl); - curl = NULL; } return result; @@ -245,68 +250,42 @@ p3FeedReaderThread::DownloadResult p3FeedReaderThread::download(const RsFeedRead DownloadResult result; - CURL *curl = curl_easy_init(); - if (curl) { - std::string proxy; - if (feed.flag & RS_FEED_FLAG_STANDARD_PROXY) { - std::string standardProxyAddress; - uint16_t standardProxyPort; - if (mFeedReader->getStandardProxy(standardProxyAddress, standardProxyPort)) { - rs_sprintf(proxy, "%s:%u", standardProxyAddress.c_str(), standardProxyPort); - } - } else { - if (!feed.proxyAddress.empty() && feed.proxyPort) { - rs_sprintf(proxy, "%s:%u", feed.proxyAddress.c_str(), feed.proxyPort); - } - } + std::string proxy = getProxyForFeed(feed); + CURLWrapper CURL(proxy); + CURLcode code = CURL.downloadText(feed.url, content); - curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0); - curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback); - curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this); - curl_easy_setopt(curl, CURLOPT_URL, feed.url.c_str()); - curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionString); - curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content); - curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1); + if (code == CURLE_OK) { + long responseCode = CURL.responseCode(); - if (!proxy.empty()) { - curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str()); - } + switch (responseCode) { + case 200: + { + std::string contentType = CURL.contentType(); - CURLcode code = curl_easy_perform(curl); - if (code == CURLE_OK) { - long response; - curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response); - if (response == 200) { - char *contentType = NULL; - curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &contentType); - if (contentType && - (strncasecmp(contentType, "text/xml", 8) == 0 || - strncasecmp(contentType, "application/rss+xml", 19) == 0 || - strncasecmp(contentType, "application/xml", 15) == 0 || - strncasecmp(contentType, "application/xhtml+xml", 21) == 0)) { + if (isContentType(contentType, "text/xml") || + isContentType(contentType, "application/rss+xml") || + isContentType(contentType, "application/xml") || + isContentType(contentType, "application/xhtml+xml")) { /* ok */ result = DOWNLOAD_SUCCESS; } else { result = DOWNLOAD_UNKNOWN_CONTENT_TYPE; - error = contentType ? contentType : ""; + error = contentType; } - } else if (response == 404) { - result = DOWNLOAD_NOT_FOUND; - } else { - result = DOWNLOAD_UNKOWN_RESPONSE_CODE; - rs_sprintf(error, "%ld", response); } - } else { - result = DOWNLOAD_ERROR; - error = curl_easy_strerror(code); + break; + case 404: + result = DOWNLOAD_NOT_FOUND; + break; + default: + result = DOWNLOAD_UNKOWN_RESPONSE_CODE; + rs_sprintf(error, "%ld", responseCode); } - curl_easy_cleanup(curl); - curl = NULL; - - getFavicon(feed.url, proxy, icon); + getFavicon(CURL, feed.url, icon); } else { - result = DOWNLOAD_ERROR_INIT; + result = DOWNLOAD_ERROR; + error = curl_easy_strerror(code); } #ifdef FEEDREADER_DEBUG @@ -320,12 +299,12 @@ p3FeedReaderThread::DownloadResult p3FeedReaderThread::download(const RsFeedRead /****************************** Process ************************************/ /***************************************************************************/ -static bool convertOutput(xmlCharEncodingHandlerPtr charEncodingHandler, const xmlChar *output, std::string &text) +static bool convertToString(xmlCharEncodingHandlerPtr charEncodingHandler, const xmlChar *xmlText, std::string &text) { bool result = false; + xmlBufferPtr in = xmlBufferCreateStatic((void*) xmlText, xmlStrlen(xmlText)); xmlBufferPtr out = xmlBufferCreate(); - xmlBufferPtr in = xmlBufferCreateStatic((void*) output, xmlStrlen(output)); int ret = xmlCharEncOutFunc(charEncodingHandler, out, in); if (ret >= 0) { result = true; @@ -338,6 +317,24 @@ static bool convertOutput(xmlCharEncodingHandlerPtr charEncodingHandler, const x return result; } +static bool convertFromString(xmlCharEncodingHandlerPtr charEncodingHandler, const char *text, xmlChar *&xmlText) +{ + bool result = false; + + xmlBufferPtr in = xmlBufferCreateStatic((void*) text, strlen(text)); + xmlBufferPtr out = xmlBufferCreate(); + int ret = xmlCharEncOutFunc(charEncodingHandler, out, in); + if (ret >= 0) { + result = true; + xmlText = xmlBufferDetach(out); + } + + xmlBufferFree(in); + xmlBufferFree(out); + + return result; +} + static xmlNodePtr findNode(xmlNodePtr node, const char *name, bool children = false) { if (node->name) { @@ -419,12 +416,46 @@ static bool getChildText(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler } if (child->children->content) { - return convertOutput((xmlCharEncodingHandlerPtr) charEncodingHandler, child->children->content, text); + return convertToString((xmlCharEncodingHandlerPtr) charEncodingHandler, child->children->content, text); } return true; } +static std::string xmlGetAttr(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler, xmlNodePtr node, const char *name) +{ + if (!node || !name) { + return ""; + } + + std::string value; + + xmlChar *xmlValue = xmlGetProp(node, (const xmlChar*) name); + if (xmlValue) { + convertToString((xmlCharEncodingHandlerPtr) charEncodingHandler, xmlValue, value); + xmlFree(xmlValue); + } + + return value; +} + +static bool xmlSetAttr(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler, xmlNodePtr node, const char *name, const char *value) +{ + if (!node || !name) { + return false; + } + + xmlChar *xmlValue = NULL; + if (!convertFromString((xmlCharEncodingHandlerPtr) charEncodingHandler, value, xmlValue)) { + return false; + } + + xmlAttrPtr xmlAttr = xmlSetProp (node, (const xmlChar*) name, xmlValue); + xmlFree(xmlValue); + + return xmlAttr != NULL; +} + static void splitString(std::string s, std::vector &v, const char d) { v.clear(); @@ -689,7 +720,7 @@ static time_t parseRFC822Date(const std::string &pubDate) // broken mail-/news-clients omit the time zone if (*dateString) { if ((strncasecmp(dateString, "gmt", 3) == 0) || - (strncasecmp(dateString, "utc", 3) == 0)) + (strncasecmp(dateString, "utc", 3) == 0)) { dateString += 3; while(*dateString && isspace(*dateString)) @@ -942,7 +973,10 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader item->feedId = feed.feedId; item->title = title; - getChildText(mCharEncodingHandler, node, "link", item->link); + /* try feedburner:origLink */ + if (!getChildText(mCharEncodingHandler, node, "origLink", item->link) || item->link.empty()) { + getChildText(mCharEncodingHandler, node, "link", item->link); + } long todo; // remove sid // // remove sid= @@ -969,6 +1003,7 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader // } getChildText(mCharEncodingHandler, node, "author", item->author); + getChildText(mCharEncodingHandler, node, "description", item->description); std::string pubDate; @@ -1007,3 +1042,148 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader return result; } + +std::string p3FeedReaderThread::getProxyForFeed(const RsFeedReaderFeed &feed) +{ + std::string proxy; + if (feed.flag & RS_FEED_FLAG_STANDARD_PROXY) { + std::string standardProxyAddress; + uint16_t standardProxyPort; + if (mFeedReader->getStandardProxy(standardProxyAddress, standardProxyPort)) { + rs_sprintf(proxy, "%s:%u", standardProxyAddress.c_str(), standardProxyPort); + } + } else { + if (!feed.proxyAddress.empty() && feed.proxyPort) { + rs_sprintf(proxy, "%s:%u", feed.proxyAddress.c_str(), feed.proxyPort); + } + } + return proxy; +} + +bool p3FeedReaderThread::processMsg(const RsFeedReaderFeed &feed, RsFeedReaderMsg *msg) +{ + if (!msg) { + return false; + } + + std::string proxy = getProxyForFeed(feed); + + std::string url; + if (feed.flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE) { +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download page " << msg->link << std::endl; +#endif + std::string content; + CURLWrapper CURL(proxy); + CURLcode code = CURL.downloadText(msg->link, content); + + if (code == CURLE_OK && CURL.responseCode() == 200 && isContentType(CURL.contentType(), "text/html")) { + /* ok */ + msg->description = content; + } else { +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot download page, CURLCode = " << code << ", responseCode = " << CURL.responseCode() << ", contentType = " << CURL.contentType() << std::endl; +#endif + return false; + } + + /* get effective url (moved location) */ + std::string effectiveUrl = CURL.effectiveUrl(); + url = getBaseLink(effectiveUrl.empty() ? msg->link : effectiveUrl); + } + + /* check if string contains xml chars (very simple test) */ + if (msg->description.find('<') == std::string::npos) { + return true; + } + + bool result = true; + + /* process description */ + long todo; // encoding + htmlDocPtr document = htmlReadMemory(msg->description.c_str(), msg->description.length(), url.c_str(), "", HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING | HTML_PARSE_COMPACT); + if (document) { + xmlNodePtr root = xmlDocGetRootElement(document); + if (root) { + /* process all children */ + std::list parents; + parents.push_back(root); + + while (!parents.empty()) { + if (!isRunning()) { + break; + } + xmlNodePtr node = parents.front(); + parents.pop_front(); + + if (node->type == XML_ELEMENT_NODE) { + /* check for image */ + if (strcasecmp((char*) node->name, "img") == 0) { + bool removeImage = true; + + if (feed.flag & RS_FEED_FLAG_EMBED_IMAGES) { + /* embed image */ + std::string src = xmlGetAttr(mCharEncodingHandler, node, "src"); + if (!src.empty()) { + /* download image */ +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download image " << src << std::endl; +#endif + std::vector data; + CURLWrapper CURL(proxy); + CURLcode code = CURL.downloadBinary(calculateLink(url, src), data); + if (code == CURLE_OK && CURL.responseCode() == 200) { + std::string contentType = CURL.contentType(); + if (isContentType(contentType, "image/")) { + std::string base64; + if (toBase64(data, base64)) { + std::string imageBase64; + rs_sprintf(imageBase64, "data:%s;base64,%s", contentType.c_str(), base64.c_str()); + if (xmlSetAttr(mCharEncodingHandler, node, "src", imageBase64.c_str())) { + removeImage = false; + } + } + } + } + } + } + + if (removeImage) { + /* remove image */ + xmlUnlinkNode(node); + xmlFreeNode(node); + continue; + } + } + + xmlNodePtr child; + for (child = node->children; child; child = child->next) { + parents.push_back(child); + } + } + } + + xmlChar *html = NULL; + int htmlSize = 0; + htmlDocDumpMemoryFormat(document, &html, &htmlSize, 0); + if (html) { + convertToString((xmlCharEncodingHandlerPtr) mCharEncodingHandler, html, msg->description); + xmlFree(html); + } else { +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot dump html" << std::endl; +#endif + result = false; + } + } else { +#ifdef FEEDREADER_DEBUG + std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") no root element" << std::endl; +#endif + result = false; + } + + xmlFreeDoc(document); + } + + return result; +} diff --git a/plugins/FeedReader/services/p3FeedReaderThread.h b/plugins/FeedReader/services/p3FeedReaderThread.h index 051cd0325..b405b8822 100644 --- a/plugins/FeedReader/services/p3FeedReaderThread.h +++ b/plugins/FeedReader/services/p3FeedReaderThread.h @@ -19,7 +19,7 @@ * Boston, MA 02110-1301, USA. ****************************************************************/ - #ifndef P3_FEEDREADERTHREAD +#ifndef P3_FEEDREADERTHREAD #define P3_FEEDREADERTHREAD #include "util/rsthreads.h" @@ -40,7 +40,6 @@ public: enum DownloadResult { DOWNLOAD_SUCCESS, - DOWNLOAD_ERROR_INIT, DOWNLOAD_ERROR, DOWNLOAD_UNKNOWN_CONTENT_TYPE, DOWNLOAD_NOT_FOUND, @@ -64,6 +63,9 @@ private: DownloadResult download(const RsFeedReaderFeed &feed, std::string &content, std::string &icon, std::string &error); ProcessResult process(const RsFeedReaderFeed &feed, std::list &entries, std::string &error); + std::string getProxyForFeed(const RsFeedReaderFeed &feed); + bool processMsg(const RsFeedReaderFeed &feed, RsFeedReaderMsg *msg); + p3FeedReader *mFeedReader; Type mType; /*xmlCharEncodingHandlerPtr*/ void *mCharEncodingHandler; diff --git a/plugins/FeedReader/services/rsFeedReaderItems.h b/plugins/FeedReader/services/rsFeedReaderItems.h index 92c911864..7f6703290 100644 --- a/plugins/FeedReader/services/rsFeedReaderItems.h +++ b/plugins/FeedReader/services/rsFeedReaderItems.h @@ -19,7 +19,7 @@ * Boston, MA 02110-1301, USA. ****************************************************************/ - #ifndef RS_FEEDREADER_ITEMS_H +#ifndef RS_FEEDREADER_ITEMS_H #define RS_FEEDREADER_ITEMS_H #include "serialiser/rsserial.h" @@ -55,6 +55,8 @@ const uint8_t RS_PKT_SUBTYPE_FEEDREADER_MSG = 0x03; #define RS_FEED_FLAG_DEACTIVATED 0x040 #define RS_FEED_FLAG_FORUM 0x080 #define RS_FEED_FLAG_UPDATE_FORUM_INFO 0x100 +#define RS_FEED_FLAG_EMBED_IMAGES 0x200 +#define RS_FEED_FLAG_SAVE_COMPLETE_PAGE 0x400 class RsFeedReaderFeed : public RsItem { diff --git a/plugins/FeedReader/services/util/CURLWrapper.cc b/plugins/FeedReader/services/util/CURLWrapper.cc new file mode 100644 index 000000000..e39d5e99c --- /dev/null +++ b/plugins/FeedReader/services/util/CURLWrapper.cc @@ -0,0 +1,136 @@ +/**************************************************************** + * RetroShare GUI is distributed under the following license: + * + * Copyright (C) 2012 by Thunder + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + ****************************************************************/ + +#include "CURLWrapper.h" + +//static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/) +//{ +// p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp; + +// if (!thread->isRunning()) { +// /* thread was stopped */ +// return 1; +// } + +// long todo; // show progress in gui + +// return 0; +//} + +CURLWrapper::CURLWrapper(const std::string &proxy) +{ + mCurl = curl_easy_init(); + if (mCurl) { + curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 0); +// curl_easy_setopt(mCurl, CURLOPT_PROGRESSFUNCTION, progressCallback); +// curl_easy_setopt(mCurl, CURLOPT_PROGRESSDATA, feedReader); + curl_easy_setopt(mCurl, CURLOPT_FOLLOWLOCATION, 1); + curl_easy_setopt(mCurl, CURLOPT_CONNECTTIMEOUT, 60); + curl_easy_setopt(mCurl, CURLOPT_TIMEOUT, 120); + + if (!proxy.empty()) { + curl_easy_setopt(mCurl, CURLOPT_PROXY, proxy.c_str()); + } + } +} + +CURLWrapper::~CURLWrapper() +{ + if (mCurl) { + curl_easy_cleanup(mCurl); + } +} + +static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream) +{ + std::string *s = (std::string*) stream; + s->append ((char*) ptr, size * nmemb); + + return nmemb * size; +} + +CURLcode CURLWrapper::downloadText(const std::string &link, std::string &data) +{ + data.clear(); + + if (!mCurl) { + return CURLE_FAILED_INIT; + } + + curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str()); + curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionString); + curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data); + + return curl_easy_perform(mCurl); +} + +static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream) +{ + std::vector *bytes = (std::vector*) stream; + + std::vector newBytes; + newBytes.resize(size * nmemb); + memcpy(newBytes.data(), ptr, newBytes.size()); + + bytes->insert(bytes->end(), newBytes.begin(), newBytes.end()); + + return nmemb * size; +} + +CURLcode CURLWrapper::downloadBinary(const std::string &link, std::vector &data) +{ + data.clear(); + + if (!mCurl) { + return CURLE_FAILED_INIT; + } + + curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 1); + curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str()); + curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionBinary); + curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data); + + return curl_easy_perform(mCurl); +} + +long CURLWrapper::longInfo(CURLINFO info) +{ + if (!mCurl) { + return 0; + } + + long value; + curl_easy_getinfo(mCurl, info, &value); + + return value; +} + +std::string CURLWrapper::stringInfo(CURLINFO info) +{ + if (!mCurl) { + return ""; + } + + char *value; + curl_easy_getinfo(mCurl, info, &value); + + return value ? value : ""; +} diff --git a/plugins/FeedReader/services/util/CURLWrapper.h b/plugins/FeedReader/services/util/CURLWrapper.h new file mode 100644 index 000000000..66ac68f53 --- /dev/null +++ b/plugins/FeedReader/services/util/CURLWrapper.h @@ -0,0 +1,50 @@ +/**************************************************************** + * RetroShare GUI is distributed under the following license: + * + * Copyright (C) 2012 by Thunder + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, + * Boston, MA 02110-1301, USA. + ****************************************************************/ + +#ifndef CURLWRAPPER +#define CURLWRAPPER + +#include +#include +#include + +class CURLWrapper +{ +public: + CURLWrapper(const std::string &proxy); + ~CURLWrapper(); + + CURLcode downloadText(const std::string &link, std::string &data); + CURLcode downloadBinary(const std::string &link, std::vector &data); + + long responseCode() { return longInfo(CURLINFO_RESPONSE_CODE); } + std::string contentType() { return stringInfo(CURLINFO_CONTENT_TYPE); } + std::string effectiveUrl() { return stringInfo(CURLINFO_EFFECTIVE_URL); } + +protected: + long longInfo(CURLINFO info); + std::string stringInfo(CURLINFO info); + +private: + CURL *mCurl; +}; + +#endif