diff --git a/plugins/FeedReader/FeedReader.pro b/plugins/FeedReader/FeedReader.pro
index 64aaa1612..769797a66 100644
--- a/plugins/FeedReader/FeedReader.pro
+++ b/plugins/FeedReader/FeedReader.pro
@@ -6,6 +6,7 @@ SOURCES = FeedReaderPlugin.cpp \
services/p3FeedReader.cc \
services/p3FeedReaderThread.cc \
services/rsFeedReaderItems.cc \
+ services/util/CURLWrapper.cc \
gui/FeedReaderDialog.cpp \
gui/AddFeedDialog.cpp \
gui/FeedReaderNotify.cpp \
@@ -16,6 +17,7 @@ HEADERS = FeedReaderPlugin.h \
services/p3FeedReader.h \
services/p3FeedReaderThread.h \
services/rsFeedReaderItems.h \
+ services/util/CURLWrapper.h \
gui/FeedReaderDialog.h \
gui/AddFeedDialog.h \
gui/FeedReaderNotify.h \
@@ -39,7 +41,7 @@ linux-* {
}
win32 {
- DEFINES += CURL_STATICLIB
+ DEFINES += CURL_STATICLIB LIBXML_STATIC
CURL_DIR = ../../../curl-7.26.0
LIBXML2_DIR = ../../../libxml2-2.8.0
diff --git a/plugins/FeedReader/gui/AddFeedDialog.cpp b/plugins/FeedReader/gui/AddFeedDialog.cpp
index 5013eaf5a..ac0b6106d 100644
--- a/plugins/FeedReader/gui/AddFeedDialog.cpp
+++ b/plugins/FeedReader/gui/AddFeedDialog.cpp
@@ -46,12 +46,17 @@ AddFeedDialog::AddFeedDialog(RsFeedReader *feedReader, QWidget *parent)
connect(ui->useStandardProxyCheckBox, SIGNAL(toggled(bool)), this, SLOT(useStandardProxyToggled()));
connect(ui->typeForumRadio, SIGNAL(toggled(bool)), this, SLOT(typeForumToggled()));
+ /* currently only for loacl feeds */
+ connect(ui->saveCompletePageCheckBox, SIGNAL(toggled(bool)), this, SLOT(denyForumToggled()));
+ connect(ui->embedImagesCheckBox, SIGNAL(toggled(bool)), this, SLOT(denyForumToggled()));
+
connect(ui->urlLineEdit, SIGNAL(textChanged(QString)), this, SLOT(validate()));
connect(ui->nameLineEdit, SIGNAL(textChanged(QString)), this, SLOT(validate()));
connect(ui->useInfoFromFeedCheckBox, SIGNAL(toggled(bool)), this, SLOT(validate()));
+ connect(ui->typeLocalRadio, SIGNAL(toggled(bool)), this, SLOT(validate()));
+ connect(ui->typeForumRadio, SIGNAL(toggled(bool)), this, SLOT(validate()));
ui->activatedCheckBox->setChecked(true);
- ui->typeLocalRadio->setChecked(true);
ui->forumComboBox->setEnabled(false);
ui->useInfoFromFeedCheckBox->setChecked(true);
ui->updateForumInfoCheckBox->setEnabled(false);
@@ -124,6 +129,16 @@ void AddFeedDialog::typeForumToggled()
ui->updateForumInfoCheckBox->setEnabled(checked);
}
+void AddFeedDialog::denyForumToggled()
+{
+ if (ui->saveCompletePageCheckBox->isChecked() || ui->embedImagesCheckBox->isChecked()) {
+ ui->typeForumRadio->setEnabled(false);
+ ui->typeLocalRadio->setChecked(true);
+ } else {
+ ui->typeForumRadio->setEnabled(true);
+ }
+}
+
void AddFeedDialog::validate()
{
bool ok = true;
@@ -134,6 +149,9 @@ void AddFeedDialog::validate()
if (ui->nameLineEdit->text().isEmpty() && !ui->useInfoFromFeedCheckBox->isChecked()) {
ok = false;
}
+ if (!ui->typeLocalRadio->isChecked() && !ui->typeForumRadio->isChecked()) {
+ ok = false;
+ }
ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(ok);
}
@@ -196,6 +214,8 @@ bool AddFeedDialog::fillFeed(const std::string &feedId)
ui->useInfoFromFeedCheckBox->setChecked(feedInfo.flag.infoFromFeed);
ui->updateForumInfoCheckBox->setChecked(feedInfo.flag.updateForumInfo);
ui->activatedCheckBox->setChecked(!feedInfo.flag.deactivated);
+ ui->embedImagesCheckBox->setChecked(feedInfo.flag.embedImages);
+ ui->saveCompletePageCheckBox->setChecked(feedInfo.flag.saveCompletePage);
ui->descriptionPlainTextEdit->setPlainText(QString::fromUtf8(feedInfo.description.c_str()));
@@ -206,6 +226,8 @@ bool AddFeedDialog::fillFeed(const std::string &feedId)
if (feedInfo.flag.forum) {
ui->typeForumRadio->setChecked(true);
+ ui->saveCompletePageCheckBox->setEnabled(false);
+ ui->embedImagesCheckBox->setEnabled(false);
if (feedInfo.forumId.empty()) {
ui->forumNameLabel->setText(tr("Not yet created"));
@@ -259,13 +281,17 @@ void AddFeedDialog::createFeed()
feedInfo.flag.infoFromFeed = ui->useInfoFromFeedCheckBox->isChecked();
feedInfo.flag.updateForumInfo = ui->updateForumInfoCheckBox->isChecked() && ui->updateForumInfoCheckBox->isEnabled();
feedInfo.flag.deactivated = !ui->activatedCheckBox->isChecked();
+ feedInfo.flag.embedImages = ui->embedImagesCheckBox->isChecked();
+ feedInfo.flag.saveCompletePage = ui->saveCompletePageCheckBox->isChecked();
feedInfo.description = ui->descriptionPlainTextEdit->toPlainText().toUtf8().constData();
feedInfo.flag.forum = ui->typeForumRadio->isChecked();
if (mFeedId.empty()) {
- /* set forum (only when create a new feed) */
- feedInfo.forumId = ui->forumComboBox->itemData(ui->forumComboBox->currentIndex()).toString().toStdString();
+ if (feedInfo.flag.forum) {
+ /* set forum (only when create a new feed) */
+ feedInfo.forumId = ui->forumComboBox->itemData(ui->forumComboBox->currentIndex()).toString().toStdString();
+ }
}
feedInfo.flag.authentication = ui->useAuthenticationCheckBox->isChecked();
diff --git a/plugins/FeedReader/gui/AddFeedDialog.h b/plugins/FeedReader/gui/AddFeedDialog.h
index c117cd789..f08d64611 100644
--- a/plugins/FeedReader/gui/AddFeedDialog.h
+++ b/plugins/FeedReader/gui/AddFeedDialog.h
@@ -50,6 +50,7 @@ private slots:
void useStandardUpdateIntervalToggled();
void useStandardProxyToggled();
void typeForumToggled();
+ void denyForumToggled();
void validate();
void createFeed();
diff --git a/plugins/FeedReader/gui/AddFeedDialog.ui b/plugins/FeedReader/gui/AddFeedDialog.ui
index ed2587dfa..b09be53a8 100644
--- a/plugins/FeedReader/gui/AddFeedDialog.ui
+++ b/plugins/FeedReader/gui/AddFeedDialog.ui
@@ -7,7 +7,7 @@
0
0
715
- 559
+ 605
@@ -289,13 +289,6 @@ p, li { white-space: pre-wrap; }
6
- -
-
-
- Local Feed
-
-
-
-
-
@@ -323,6 +316,13 @@ p, li { white-space: pre-wrap; }
+ -
+
+
+ Local Feed
+
+
+
-
@@ -366,6 +366,20 @@ p, li { white-space: pre-wrap; }
+ -
+
+
+ Embed images (experimental for local feeds)
+
+
+
+ -
+
+
+ Save complete web page (experimental for local feeds)
+
+
+
@@ -416,7 +430,6 @@ p, li { white-space: pre-wrap; }
urlLineEdit
nameLineEdit
descriptionPlainTextEdit
- typeLocalRadio
typeForumRadio
forumComboBox
activatedCheckBox
diff --git a/plugins/FeedReader/gui/FeedReaderDialog.cpp b/plugins/FeedReader/gui/FeedReaderDialog.cpp
index d93bf83b8..ef4438e1a 100644
--- a/plugins/FeedReader/gui/FeedReaderDialog.cpp
+++ b/plugins/FeedReader/gui/FeedReaderDialog.cpp
@@ -27,6 +27,7 @@
#include
#include
#include
+#include
#include "FeedReaderDialog.h"
#include "ui_FeedReaderDialog.h"
@@ -118,6 +119,7 @@ FeedReaderDialog::FeedReaderDialog(RsFeedReader *feedReader, QWidget *parent)
connect(ui->filterLineEdit, SIGNAL(textChanged(QString)), this, SLOT(filterItems(QString)));
connect(ui->filterColumnComboBox, SIGNAL(currentIndexChanged(int)), this, SLOT(filterColumnChanged()));
+ connect(ui->linkButton, SIGNAL(clicked()), this, SLOT(openLinkMsg()));
connect(ui->expandButton, SIGNAL(clicked()), this, SLOT(toggleMsgText()));
mFeedCompareRole = new RSTreeWidgetItemCompareRole;
@@ -168,6 +170,18 @@ FeedReaderDialog::FeedReaderDialog(RsFeedReader *feedReader, QWidget *parent)
/* initialize feed list */
ui->feedTreeWidget->sortItems(COLUMN_FEED_NAME, Qt::AscendingOrder);
+ /* build menu for link button */
+ QMenu *menu = new QMenu(this);
+ QAction *action = menu->addAction(tr("Open link in browser"), this, SLOT(openLinkMsg()));
+ menu->addAction(tr("Copy link to clipboard"), this, SLOT(copyLinkMsg()));
+
+ QFont font = action->font();
+ font.setBold(true);
+ action->setFont(font);
+
+ ui->linkButton->setMenu(menu);
+ ui->linkButton->setEnabled(false);
+
ui->msgTreeWidget->installEventFilter(this);
}
@@ -351,7 +365,7 @@ void FeedReaderDialog::msgTreeCustomPopupMenu(QPoint /*point*/)
contextMnu.addSeparator();
- action = contextMnu.addAction(QIcon(""), tr("Copy link"), this, SLOT(copyLinkMsg()));
+ action = contextMnu.addAction(QIcon(""), tr("Copy link"), this, SLOT(copyLinskMsg()));
action->setEnabled(!selectedItems.empty());
action = contextMnu.addAction(QIcon(""), tr("Remove"), this, SLOT(removeMsg()));
@@ -522,10 +536,14 @@ void FeedReaderDialog::updateFeedItem(QTreeWidgetItem *item, FeedInfo &info)
case FeedInfo::WAITING:
break;
case FeedInfo::WAITING_TO_DOWNLOAD:
+ workState = tr("waiting for download");
+ break;
case FeedInfo::DOWNLOADING:
- workState = tr("loading");
+ workState = tr("downloading");
break;
case FeedInfo::WAITING_TO_PROCESS:
+ workState = tr("waiting for process");
+ break;
case FeedInfo::PROCESSING:
workState = tr("processing");
break;
@@ -791,6 +809,7 @@ void FeedReaderDialog::msgItemChanged()
ui->msgTitle->clear();
// ui->msgLink->clear();
ui->msgText->clear();
+ ui->linkButton->setEnabled(false);
return;
}
@@ -800,6 +819,7 @@ void FeedReaderDialog::msgItemChanged()
ui->msgTitle->clear();
// ui->msgLink->clear();
ui->msgText->clear();
+ ui->linkButton->setEnabled(false);
return;
}
@@ -809,6 +829,7 @@ void FeedReaderDialog::msgItemChanged()
ui->msgTitle->clear();
// ui->msgLink->clear();
ui->msgText->clear();
+ ui->linkButton->setEnabled(false);
return;
}
@@ -832,7 +853,8 @@ void FeedReaderDialog::msgItemChanged()
ui->msgText->setHtml(msgTxt);
ui->msgTitle->setText(QString::fromUtf8(msgInfo.title.c_str()));
-// ui->msgLink->setHtml(RsHtml().formatText(NULL, QString::fromUtf8(msgInfo.link.c_str()), RSHTML_FORMATTEXT_EMBED_LINKS));
+
+ ui->linkButton->setEnabled(!msgInfo.link.empty());
}
void FeedReaderDialog::setMsgAsReadUnread(QList &rows, bool read)
@@ -1049,7 +1071,7 @@ void FeedReaderDialog::markAllAsReadMsg()
setMsgAsReadUnread(items, true);
}
-void FeedReaderDialog::copyLinkMsg()
+void FeedReaderDialog::copyLinksMsg()
{
QString links;
@@ -1086,3 +1108,33 @@ void FeedReaderDialog::removeMsg()
}
mFeedReader->removeMsgs(feedId, msgIds);
}
+
+void FeedReaderDialog::copyLinkMsg()
+{
+ QTreeWidgetItem *item = ui->msgTreeWidget->currentItem();
+ if (!item) {
+ return;
+ }
+
+ QString link = item->data(COLUMN_MSG_DATA, ROLE_MSG_LINK).toString();
+ if (link.isEmpty()) {
+ return;
+ }
+
+ QApplication::clipboard()->setText(link);
+}
+
+void FeedReaderDialog::openLinkMsg()
+{
+ QTreeWidgetItem *item = ui->msgTreeWidget->currentItem();
+ if (!item) {
+ return;
+ }
+
+ QString link = item->data(COLUMN_MSG_DATA, ROLE_MSG_LINK).toString();
+ if (link.isEmpty()) {
+ return;
+ }
+
+ QDesktopServices::openUrl(QUrl(link));
+}
diff --git a/plugins/FeedReader/gui/FeedReaderDialog.h b/plugins/FeedReader/gui/FeedReaderDialog.h
index 1bb6c14b8..7eae19f3e 100644
--- a/plugins/FeedReader/gui/FeedReaderDialog.h
+++ b/plugins/FeedReader/gui/FeedReaderDialog.h
@@ -64,8 +64,10 @@ private slots:
void markAsReadMsg();
void markAsUnreadMsg();
void markAllAsReadMsg();
- void copyLinkMsg();
+ void copyLinksMsg();
void removeMsg();
+ void openLinkMsg();
+ void copyLinkMsg();
/* FeedReaderNotify */
void feedChanged(const QString &feedId, int type);
diff --git a/plugins/FeedReader/gui/FeedReaderDialog.ui b/plugins/FeedReader/gui/FeedReaderDialog.ui
index 95c076e29..7dbaeea91 100644
--- a/plugins/FeedReader/gui/FeedReaderDialog.ui
+++ b/plugins/FeedReader/gui/FeedReaderDialog.ui
@@ -268,8 +268,8 @@ border: 1px solid #CCCCCC;}
-
-
-
-
+
+
-
@@ -289,7 +289,7 @@ border: 1px solid #CCCCCC;}
- -
+
-
QLabel#msgTitle{
@@ -305,7 +305,21 @@ background: white;}
- -
+
-
+
+
+
+ :/images/Link.png:/images/Link.png
+
+
+ QToolButton::MenuButtonPopup
+
+
+ true
+
+
+
+ -
diff --git a/plugins/FeedReader/gui/FeedReader_images.qrc b/plugins/FeedReader/gui/FeedReader_images.qrc
index 6acbedfab..e1a6c5918 100644
--- a/plugins/FeedReader/gui/FeedReader_images.qrc
+++ b/plugins/FeedReader/gui/FeedReader_images.qrc
@@ -8,6 +8,7 @@
images/FeedErrorOverlay.png
images/FolderAdd.png
images/FeedAdd.png
+ images/Link.png
images/Update.png
diff --git a/plugins/FeedReader/gui/images/Link.png b/plugins/FeedReader/gui/images/Link.png
new file mode 100644
index 000000000..8f8fd3bc1
Binary files /dev/null and b/plugins/FeedReader/gui/images/Link.png differ
diff --git a/plugins/FeedReader/interface/rsFeedReader.h b/plugins/FeedReader/interface/rsFeedReader.h
index 82d650db9..42be3e148 100644
--- a/plugins/FeedReader/interface/rsFeedReader.h
+++ b/plugins/FeedReader/interface/rsFeedReader.h
@@ -19,7 +19,7 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
- #ifndef RETROSHARE_FEEDREADER_GUI_INTERFACE_H
+#ifndef RETROSHARE_FEEDREADER_GUI_INTERFACE_H
#define RETROSHARE_FEEDREADER_GUI_INTERFACE_H
#include
@@ -68,6 +68,8 @@ public:
flag.deactivated = false;
flag.forum = false;
flag.updateForumInfo = false;
+ flag.embedImages = false;
+ flag.saveCompletePage = false;
}
std::string feedId;
@@ -98,6 +100,8 @@ public:
bool deactivated : 1;
bool forum : 1;
bool updateForumInfo : 1;
+ bool embedImages : 1;
+ bool saveCompletePage : 1;
} flag;
};
diff --git a/plugins/FeedReader/services/p3FeedReader.cc b/plugins/FeedReader/services/p3FeedReader.cc
index 0ae7f6ec0..76b16ea65 100644
--- a/plugins/FeedReader/services/p3FeedReader.cc
+++ b/plugins/FeedReader/services/p3FeedReader.cc
@@ -30,6 +30,7 @@
RsFeedReader *rsFeedReader = NULL;
#define FEEDREADER_CLEAN_INTERVAL 1 * 60 * 60 // check every hour
+#define FEEDREADER_FORUM_PREFIX L"RSS: "
/*********
* #define FEEDREADER_DEBUG
@@ -91,6 +92,8 @@ static void feedToInfo(const RsFeedReaderFeed *feed, FeedInfo &info)
info.flag.deactivated = (feed->flag & RS_FEED_FLAG_DEACTIVATED);
info.flag.forum = (feed->flag & RS_FEED_FLAG_FORUM);
info.flag.updateForumInfo = (feed->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO);
+ info.flag.embedImages = (feed->flag & RS_FEED_FLAG_EMBED_IMAGES);
+ info.flag.saveCompletePage = (feed->flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE);
switch (feed->workstate) {
case RsFeedReaderFeed::WAITING:
@@ -151,6 +154,12 @@ static void infoToFeed(const FeedInfo &info, RsFeedReaderFeed *feed, bool add)
if (info.flag.deactivated) {
feed->flag |= RS_FEED_FLAG_DEACTIVATED;
}
+ if (info.flag.embedImages) {
+ feed->flag |= RS_FEED_FLAG_EMBED_IMAGES;
+ }
+ if (info.flag.saveCompletePage) {
+ feed->flag |= RS_FEED_FLAG_SAVE_COMPLETE_PAGE;
+ }
if (add) {
/* only set when adding a new feed */
if (info.flag.folder) {
@@ -950,6 +959,11 @@ int p3FeedReader::tick()
} else {
updateInterval = fi->updateInterval;
}
+
+ if (updateInterval == 0) {
+ continue;
+ }
+
if (fi->lastUpdate == 0 || fi->lastUpdate + (long) updateInterval <= currentTime) {
/* add to download list */
feedToDownload.push_back(fi->feedId);
@@ -1340,7 +1354,6 @@ void p3FeedReader::onDownloadError(const std::string &feedId, p3FeedReaderThread
case p3FeedReaderThread::DOWNLOAD_ERROR:
fi->errorState = RS_FEED_ERRORSTATE_DOWNLOAD_ERROR;
break;
- case p3FeedReaderThread::DOWNLOAD_ERROR_INIT:
case p3FeedReaderThread::DOWNLOAD_INTERNAL_ERROR:
fi->errorState = RS_FEED_ERRORSTATE_DOWNLOAD_INTERNAL_ERROR;
break;
@@ -1428,15 +1441,13 @@ bool p3FeedReader::getFeedToProcess(RsFeedReaderFeed &feed)
return true;
}
-void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list &msgs)
+bool p3FeedReader::onProcessSuccess_filterMsg(const std::string &feedId, std::list &msgs)
{
#ifdef FEEDREADER_DEBUG
- std::cerr << "p3FeedReader::onProcessSuccess - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
+ std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
#endif
- std::list addedMsgs;
- std::string forumId;
- std::list forumMsgs;
+ bool result = true;
{
RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
@@ -1446,9 +1457,9 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listsecond;
@@ -1458,6 +1469,10 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listforumId.empty()) {
/* create new forum */
+ std::wstring forumName;
+ librs::util::ConvertUtf8ToUtf16(fi->name, forumName);
+ forumName.insert(0, FEEDREADER_FORUM_PREFIX);
+
long todo; // search for existing forum?
/* search for existing own forum */
@@ -1472,22 +1487,18 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listname, forumName);
std::wstring forumDescription;
librs::util::ConvertUtf8ToUtf16(fi->description, forumDescription);
/* create anonymous public forum */
fi->forumId = rsForums->createForum(forumName, forumDescription, RS_DISTRIB_PUBLIC | RS_DISTRIB_AUTHEN_ANON);
- forumId = fi->forumId;
-
if (fi->forumId.empty()) {
errorState = RS_FEED_ERRORSTATE_FORUM_CREATE;
#ifdef FEEDREADER_DEBUG
- std::cerr << "p3FeedReader::onProcessSuccess - can't create forum for feed " << feedId << " (" << it->second->name << ") - ignore all messages" << std::endl;
+ std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - can't create forum for feed " << feedId << " (" << it->second->name << ") - ignore all messages" << std::endl;
} else {
- std::cerr << "p3FeedReader::onProcessSuccess - forum " << forumId << " (" << fi->name << ") created" << std::endl;
+ std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - forum " << forumId << " (" << fi->name << ") created" << std::endl;
#endif
}
} else {
@@ -1498,8 +1509,6 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listforumId;
}
} else {
errorState = RS_FEED_ERRORSTATE_FORUM_NOT_FOUND;
@@ -1508,15 +1517,11 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list::iterator newMsgIt;
for (newMsgIt = msgs.begin(); newMsgIt != msgs.end(); ) {
RsFeedReaderMsg *miNew = *newMsgIt;
- /* search for exisiting msg */
+ /* search for existing msg */
std::map::iterator msgIt;
for (msgIt = fi->mMsgs.begin(); msgIt != fi->mMsgs.end(); ++msgIt) {
RsFeedReaderMsg *mi = msgIt->second;
@@ -1525,38 +1530,103 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listmMsgs.end()) {
- /* add new msg */
- rs_sprintf(miNew->msgId, "%lu", mNextMsgId++);
- if (forum) {
- miNew->flag = RS_FEEDMSG_FLAG_DELETED;
- forumMsgs.push_back(*miNew);
-// miNew->description.clear();
- } else {
- miNew->flag = RS_FEEDMSG_FLAG_NEW;
- addedMsgs.push_back(miNew->msgId);
- }
- fi->mMsgs[miNew->msgId] = miNew;
+ if (msgIt != fi->mMsgs.end()) {
+ /* msg exists */
+ delete(*newMsgIt);
newMsgIt = msgs.erase(newMsgIt);
-
-#ifdef FEEDREADER_DEBUG
- ++newMsgs;
-#endif
} else {
- /* msg was updated */
++newMsgIt;
}
+ }
+ } else {
+ result = false;
+ }
+
+ fi->content.clear();
+ fi->errorState = errorState;
+ fi->errorString.clear();
+
+ IndicateConfigChanged();
+ }
+
+ if (mNotify) {
+ mNotify->feedChanged(feedId, NOTIFY_TYPE_MOD);
+ }
+
+ return result;
+}
+
+void p3FeedReader::onProcessSuccess_addMsgs(const std::string &feedId, bool result, std::list &msgs)
+{
#ifdef FEEDREADER_DEBUG
- std::cerr << "p3FeedReader::onProcessSuccess - feed " << fi->feedId << " (" << fi->name << ") added " << newMsgs << "/" << msgs.size() << " messages" << std::endl;
+ std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
+#endif
+
+ std::list addedMsgs;
+ std::string forumId;
+ std::list forumMsgs;
+
+ {
+ RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
+
+ /* find feed */
+ std::map::iterator it = mFeeds.find(feedId);
+ if (it == mFeeds.end()) {
+ /* feed not found */
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << feedId << " not found" << std::endl;
+#endif
+ return;
+ }
+
+ RsFeedReaderFeed *fi = it->second;
+ bool forum = (fi->flag & RS_FEED_FLAG_FORUM);
+
+ if (forum) {
+ forumId = fi->forumId;
+ if (forumId.empty()) {
+ /* don't process messages without forum id */
+ result = false;
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << fi->feedId << " (" << fi->name << ") don't process messages without forum id" << std::endl;
#endif
}
}
+ /* process msgs */
+#ifdef FEEDREADER_DEBUG
+ uint32_t newMsgs = 0;
+#endif
+
+ if (result) {
+ std::list::iterator newMsgIt;
+ for (newMsgIt = msgs.begin(); newMsgIt != msgs.end(); ) {
+ RsFeedReaderMsg *miNew = *newMsgIt;
+ /* add new msg */
+ rs_sprintf(miNew->msgId, "%lu", mNextMsgId++);
+ if (forum) {
+ miNew->flag = RS_FEEDMSG_FLAG_DELETED;
+ forumMsgs.push_back(*miNew);
+// miNew->description.clear();
+ } else {
+ miNew->flag = RS_FEEDMSG_FLAG_NEW;
+ addedMsgs.push_back(miNew->msgId);
+ }
+ fi->mMsgs[miNew->msgId] = miNew;
+ newMsgIt = msgs.erase(newMsgIt);
+
+#ifdef FEEDREADER_DEBUG
+ ++newMsgs;
+#endif
+ }
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << fi->feedId << " (" << fi->name << ") added " << newMsgs << "/" << msgs.size() << " messages" << std::endl;
+#endif
+ }
+
fi->workstate = RsFeedReaderFeed::WAITING;
fi->content.clear();
fi->lastUpdate = time(NULL);
- fi->errorState = errorState;
- fi->errorString.clear();
IndicateConfigChanged();
}
@@ -1584,7 +1654,7 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::listsetMessageStatus(forumMsgInfo.forumId, forumMsgInfo.msgId, 0, FORUM_MSG_STATUS_MASK);
} else {
#ifdef FEEDREADER_DEBUG
- std::cerr << "p3FeedReader::onProcessSuccess - can't add forum message " << mi.title << " for feed " << forumId << std::endl;
+ std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - can't add forum message " << mi.title << " for feed " << forumId << std::endl;
#endif
}
}
@@ -1652,7 +1722,7 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
{
bool changed = false;
std::string forumId;
- ForumInfo forumInfo;
+ ForumInfo forumInfoNew;
{
RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
@@ -1683,11 +1753,12 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
changed = true;
}
- if (changed && (fi->flag & RS_FEED_FLAG_FORUM) && (fi->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO) && !fi->forumId.empty()) {
+ if ((fi->flag & RS_FEED_FLAG_FORUM) && (fi->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO) && !fi->forumId.empty()) {
/* change forum too */
forumId = fi->forumId;
- librs::util::ConvertUtf8ToUtf16(fi->name, forumInfo.forumName);
- librs::util::ConvertUtf8ToUtf16(fi->description, forumInfo.forumDesc);
+ librs::util::ConvertUtf8ToUtf16(fi->name, forumInfoNew.forumName);
+ librs::util::ConvertUtf8ToUtf16(fi->description, forumInfoNew.forumDesc);
+ forumInfoNew.forumName.insert(0, FEEDREADER_FORUM_PREFIX);
}
}
@@ -1700,10 +1771,22 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
}
if (!forumId.empty()) {
- /* name or description changed, update forum */
- if (!rsForums->setForumInfo(forumId, forumInfo)) {
+ ForumInfo forumInfo;
+ if (rsForums->getForumInfo(forumId, forumInfo)) {
+ if (forumInfo.forumName != forumInfoNew.forumName || forumInfo.forumDesc != forumInfoNew.forumDesc) {
+ /* name or description changed, update forum */
#ifdef FEEDREADER_DEBUG
- std::cerr << "p3FeedReader::setFeed - can't change forum " << forumId << std::endl;
+ std::cerr << "p3FeedReader::setFeed - change forum " << forumId << std::endl;
+#endif
+ if (!rsForums->setForumInfo(forumId, forumInfoNew)) {
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReader::setFeed - can't change forum " << forumId << std::endl;
+#endif
+ }
+ }
+ } else {
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReader::setFeed - can't get forum info " << forumId << std::endl;
#endif
}
}
diff --git a/plugins/FeedReader/services/p3FeedReader.h b/plugins/FeedReader/services/p3FeedReader.h
index 2e62e16da..e4525a65b 100644
--- a/plugins/FeedReader/services/p3FeedReader.h
+++ b/plugins/FeedReader/services/p3FeedReader.h
@@ -19,7 +19,7 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
- #ifndef P3_FEEDREADER
+#ifndef P3_FEEDREADER
#define P3_FEEDREADER
#include "retroshare/rsplugin.h"
@@ -71,7 +71,8 @@ public:
bool getFeedToDownload(RsFeedReaderFeed &feed);
void onDownloadSuccess(const std::string &feedId, const std::string &content, std::string &icon);
void onDownloadError(const std::string &feedId, p3FeedReaderThread::DownloadResult result, const std::string &error);
- void onProcessSuccess(const std::string &feedId, std::list &msgs);
+ bool onProcessSuccess_filterMsg(const std::string &feedId, std::list &msgs);
+ void onProcessSuccess_addMsgs(const std::string &feedId, bool result, std::list &msgs);
void onProcessError(const std::string &feedId, p3FeedReaderThread::ProcessResult result);
bool getFeedToProcess(RsFeedReaderFeed &feed);
diff --git a/plugins/FeedReader/services/p3FeedReaderThread.cc b/plugins/FeedReader/services/p3FeedReaderThread.cc
index cbc03fe8b..b95589c01 100644
--- a/plugins/FeedReader/services/p3FeedReaderThread.cc
+++ b/plugins/FeedReader/services/p3FeedReaderThread.cc
@@ -22,9 +22,11 @@
#include "p3FeedReaderThread.h"
#include "rsFeedReaderItems.h"
#include "util/rsstring.h"
+#include "util/CURLWrapper.h"
-#include
#include
+#include
+#include
#include
enum FeedFormat { FORMAT_RSS, FORMAT_RDF };
@@ -32,6 +34,7 @@ enum FeedFormat { FORMAT_RSS, FORMAT_RDF };
/*********
* #define FEEDREADER_DEBUG
*********/
+#define FEEDREADER_DEBUG
p3FeedReaderThread::p3FeedReaderThread(p3FeedReader *feedReader, Type type) : RsThread(), mFeedReader(feedReader), mType(type)
{
@@ -88,21 +91,34 @@ void p3FeedReaderThread::run()
{
RsFeedReaderFeed feed;
if (mFeedReader->getFeedToProcess(feed)) {
- std::list entries;
+ std::list msgs;
std::string error;
+ std::list::iterator it;
- ProcessResult result = process(feed, entries, error);
+ ProcessResult result = process(feed, msgs, error);
if (result == PROCESS_SUCCESS) {
- mFeedReader->onProcessSuccess(feed.feedId, entries);
+ /* first, filter the messages */
+ bool result = mFeedReader->onProcessSuccess_filterMsg(feed.feedId, msgs);
+ if (isRunning()) {
+ if (result) {
+ long todo; // process new items
+ /* second, process the descriptions */
+ for (it = msgs.begin(); it != msgs.end(); ++it) {
+ RsFeedReaderMsg *mi = *it;
+ processMsg(feed, mi);
+ }
+ }
+ /* third, add messages */
+ mFeedReader->onProcessSuccess_addMsgs(feed.feedId, result, msgs);
+ }
} else {
mFeedReader->onProcessError(feed.feedId, result);
}
- std::list::iterator it;
- for (it = entries.begin(); it != entries.end(); ++it) {
+ for (it = msgs.begin(); it != msgs.end(); ++it) {
delete (*it);
}
- entries.clear();
+ msgs.clear();
}
}
break;
@@ -114,121 +130,110 @@ void p3FeedReaderThread::run()
/****************************** Download ***********************************/
/***************************************************************************/
-static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream)
+static bool isContentType(const std::string &contentType, const char *type)
{
- std::string *s = (std::string*) stream;
- s->append ((char*) ptr, size * nmemb);
-
- return nmemb * size;
+ return (strncasecmp(contentType.c_str(), type, strlen(type)) == 0);
}
-static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream)
+static bool toBase64(const std::vector &data, std::string &base64)
{
- std::vector *bytes = (std::vector*) stream;
+ bool result = false;
- std::vector newBytes;
- newBytes.resize(size * nmemb);
- memcpy(newBytes.data(), ptr, newBytes.size());
-
- bytes->insert(bytes->end(), newBytes.begin(), newBytes.end());
-
- return nmemb * size;
-}
-
-static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/)
-{
- p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp;
-
- if (!thread->isRunning()) {
- /* thread was stopped */
- return 1;
+ /* Set up a base64 encoding BIO that writes to a memory BIO */
+ BIO *b64 = BIO_new(BIO_f_base64());
+ if (b64) {
+ BIO_set_flags(b64, BIO_FLAGS_BASE64_NO_NL);
+ BIO *bmem = BIO_new(BIO_s_mem());
+ if (bmem) {
+ BIO_set_flags(bmem, BIO_CLOSE); // probably redundant
+ b64 = BIO_push(b64, bmem);
+ /* Send the data */
+ BIO_write(b64, data.data(), data.size());
+ /* Collect the encoded data */
+ BIO_flush(b64);
+ char* temp;
+ int count = BIO_get_mem_data(bmem, &temp);
+ if (count && temp) {
+ base64.assign(temp, count);
+ result = true;
+ }
+ }
+ BIO_free_all(b64);
}
- long todo; // show progress in gui
-
- return 0;
+ return result;
}
-static bool getFavicon (std::string url, const std::string &proxy, std::string &icon)
+static std::string getBaseLink(std::string link)
+{
+ size_t found = link.rfind('/');
+ if (found != std::string::npos) {
+ link.erase(found + 1);
+ }
+
+ return link;
+}
+
+static std::string calculateLink(const std::string &baseLink, const std::string &link)
+{
+ if (link.substr(0, 7) == "http://") {
+ /* absolute link */
+ return link;
+ }
+
+ /* calculate link of base link */
+ std::string resultLink = baseLink;
+
+ /* link should begin with "http://" */
+ if (resultLink.substr(0, 7) != "http://") {
+ resultLink.insert(0, "http://");
+ }
+
+ if (link.empty()) {
+ /* no link */
+ return resultLink;
+ }
+
+ if (*link.begin() == '/') {
+ /* link begins with "/" */
+ size_t found = resultLink.find('/', 7);
+ if (found != std::string::npos) {
+ resultLink.erase(found);
+ }
+ } else {
+ /* check for "/" at the end */
+ std::string::reverse_iterator it = resultLink.rend ();
+ it--;
+ if (*it != '/') {
+ resultLink += "/";
+ }
+ }
+
+ resultLink += link;
+
+ return resultLink;
+}
+
+static bool getFavicon(CURLWrapper &CURL, const std::string &url, std::string &icon)
{
icon.clear();
- if (url.substr(0, 7) == "http://") {
- int found = url.find("/", 7);
- if (found >= 0) {
- url.erase(found, url.length() - found);
- }
- } else {
- return false;
- }
-
bool result = false;
- CURL *curl = curl_easy_init();
- if (curl) {
- url += "/favicon.ico";
-
- std::vector vicon;
- curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
- curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionBinary);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, &vicon);
- curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
- curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
-
- if (!proxy.empty()) {
- curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str());
- }
-
- CURLcode code = curl_easy_perform(curl);
- if (code == CURLE_OK) {
- long response;
- curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
- if (response == 200) {
- char *contentType = NULL;
- curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &contentType);
- if (contentType &&
- (strncasecmp(contentType, "image/x-icon", 12) == 0 ||
- strncasecmp(contentType, "application/octet-stream", 24) == 0 ||
- strncasecmp(contentType, "text/plain", 10) == 0)) {
- if (!vicon.empty()) {
- long todo; // check it
- // Set up a base64 encoding BIO that writes to a memory BIO
- BIO *b64 = BIO_new(BIO_f_base64());
- if (b64) {
- BIO *out = BIO_new(BIO_s_mem());
- if (out) {
- BIO_set_flags(out, BIO_CLOSE); // probably redundant
- b64 = BIO_push(b64, out);
- // Send the data
- BIO_write(b64, vicon.data(), vicon.size());
- // Collect the encoded data
- BIO_flush(b64);
- char* temp;
- int count = BIO_get_mem_data(out, &temp);
- if (count && temp) {
- icon = temp;
- result = true;
- }
- }
- BIO_free_all(b64);
- }
- }
-// char *encode = NULL;
-// size_t encodeSize = 0;
-// code = Curl_base64_encode(curl, (const char*) vicon.data(), vicon.size(), &encode, &encodeSize);
-// if (code == CURLE_OK && encodeSize) {
-// icon = encode;
-// free(encode);
-// encode = NULL;
-// result = true;
-// }
+ std::vector vicon;
+ CURLcode code = CURL.downloadBinary(calculateLink(url, "/favicon.ico"), vicon);
+ if (code == CURLE_OK) {
+ if (CURL.responseCode() == 200) {
+ std::string contentType = CURL.contentType();
+ if (isContentType(contentType, "image/x-icon") ||
+ isContentType(contentType, "application/octet-stream") ||
+ isContentType(contentType, "text/plain")) {
+ if (!vicon.empty()) {
+ long todo; // check it
+ result = toBase64(vicon, icon);
}
}
}
-
- curl_easy_cleanup(curl);
- curl = NULL;
}
return result;
@@ -245,68 +250,42 @@ p3FeedReaderThread::DownloadResult p3FeedReaderThread::download(const RsFeedRead
DownloadResult result;
- CURL *curl = curl_easy_init();
- if (curl) {
- std::string proxy;
- if (feed.flag & RS_FEED_FLAG_STANDARD_PROXY) {
- std::string standardProxyAddress;
- uint16_t standardProxyPort;
- if (mFeedReader->getStandardProxy(standardProxyAddress, standardProxyPort)) {
- rs_sprintf(proxy, "%s:%u", standardProxyAddress.c_str(), standardProxyPort);
- }
- } else {
- if (!feed.proxyAddress.empty() && feed.proxyPort) {
- rs_sprintf(proxy, "%s:%u", feed.proxyAddress.c_str(), feed.proxyPort);
- }
- }
+ std::string proxy = getProxyForFeed(feed);
+ CURLWrapper CURL(proxy);
+ CURLcode code = CURL.downloadText(feed.url, content);
- curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
- curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback);
- curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
- curl_easy_setopt(curl, CURLOPT_URL, feed.url.c_str());
- curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionString);
- curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
- curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+ if (code == CURLE_OK) {
+ long responseCode = CURL.responseCode();
- if (!proxy.empty()) {
- curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str());
- }
+ switch (responseCode) {
+ case 200:
+ {
+ std::string contentType = CURL.contentType();
- CURLcode code = curl_easy_perform(curl);
- if (code == CURLE_OK) {
- long response;
- curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
- if (response == 200) {
- char *contentType = NULL;
- curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &contentType);
- if (contentType &&
- (strncasecmp(contentType, "text/xml", 8) == 0 ||
- strncasecmp(contentType, "application/rss+xml", 19) == 0 ||
- strncasecmp(contentType, "application/xml", 15) == 0 ||
- strncasecmp(contentType, "application/xhtml+xml", 21) == 0)) {
+ if (isContentType(contentType, "text/xml") ||
+ isContentType(contentType, "application/rss+xml") ||
+ isContentType(contentType, "application/xml") ||
+ isContentType(contentType, "application/xhtml+xml")) {
/* ok */
result = DOWNLOAD_SUCCESS;
} else {
result = DOWNLOAD_UNKNOWN_CONTENT_TYPE;
- error = contentType ? contentType : "";
+ error = contentType;
}
- } else if (response == 404) {
- result = DOWNLOAD_NOT_FOUND;
- } else {
- result = DOWNLOAD_UNKOWN_RESPONSE_CODE;
- rs_sprintf(error, "%ld", response);
}
- } else {
- result = DOWNLOAD_ERROR;
- error = curl_easy_strerror(code);
+ break;
+ case 404:
+ result = DOWNLOAD_NOT_FOUND;
+ break;
+ default:
+ result = DOWNLOAD_UNKOWN_RESPONSE_CODE;
+ rs_sprintf(error, "%ld", responseCode);
}
- curl_easy_cleanup(curl);
- curl = NULL;
-
- getFavicon(feed.url, proxy, icon);
+ getFavicon(CURL, feed.url, icon);
} else {
- result = DOWNLOAD_ERROR_INIT;
+ result = DOWNLOAD_ERROR;
+ error = curl_easy_strerror(code);
}
#ifdef FEEDREADER_DEBUG
@@ -320,12 +299,12 @@ p3FeedReaderThread::DownloadResult p3FeedReaderThread::download(const RsFeedRead
/****************************** Process ************************************/
/***************************************************************************/
-static bool convertOutput(xmlCharEncodingHandlerPtr charEncodingHandler, const xmlChar *output, std::string &text)
+static bool convertToString(xmlCharEncodingHandlerPtr charEncodingHandler, const xmlChar *xmlText, std::string &text)
{
bool result = false;
+ xmlBufferPtr in = xmlBufferCreateStatic((void*) xmlText, xmlStrlen(xmlText));
xmlBufferPtr out = xmlBufferCreate();
- xmlBufferPtr in = xmlBufferCreateStatic((void*) output, xmlStrlen(output));
int ret = xmlCharEncOutFunc(charEncodingHandler, out, in);
if (ret >= 0) {
result = true;
@@ -338,6 +317,24 @@ static bool convertOutput(xmlCharEncodingHandlerPtr charEncodingHandler, const x
return result;
}
+static bool convertFromString(xmlCharEncodingHandlerPtr charEncodingHandler, const char *text, xmlChar *&xmlText)
+{
+ bool result = false;
+
+ xmlBufferPtr in = xmlBufferCreateStatic((void*) text, strlen(text));
+ xmlBufferPtr out = xmlBufferCreate();
+ int ret = xmlCharEncOutFunc(charEncodingHandler, out, in);
+ if (ret >= 0) {
+ result = true;
+ xmlText = xmlBufferDetach(out);
+ }
+
+ xmlBufferFree(in);
+ xmlBufferFree(out);
+
+ return result;
+}
+
static xmlNodePtr findNode(xmlNodePtr node, const char *name, bool children = false)
{
if (node->name) {
@@ -419,12 +416,46 @@ static bool getChildText(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler
}
if (child->children->content) {
- return convertOutput((xmlCharEncodingHandlerPtr) charEncodingHandler, child->children->content, text);
+ return convertToString((xmlCharEncodingHandlerPtr) charEncodingHandler, child->children->content, text);
}
return true;
}
+static std::string xmlGetAttr(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler, xmlNodePtr node, const char *name)
+{
+ if (!node || !name) {
+ return "";
+ }
+
+ std::string value;
+
+ xmlChar *xmlValue = xmlGetProp(node, (const xmlChar*) name);
+ if (xmlValue) {
+ convertToString((xmlCharEncodingHandlerPtr) charEncodingHandler, xmlValue, value);
+ xmlFree(xmlValue);
+ }
+
+ return value;
+}
+
+static bool xmlSetAttr(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler, xmlNodePtr node, const char *name, const char *value)
+{
+ if (!node || !name) {
+ return false;
+ }
+
+ xmlChar *xmlValue = NULL;
+ if (!convertFromString((xmlCharEncodingHandlerPtr) charEncodingHandler, value, xmlValue)) {
+ return false;
+ }
+
+ xmlAttrPtr xmlAttr = xmlSetProp (node, (const xmlChar*) name, xmlValue);
+ xmlFree(xmlValue);
+
+ return xmlAttr != NULL;
+}
+
static void splitString(std::string s, std::vector &v, const char d)
{
v.clear();
@@ -689,7 +720,7 @@ static time_t parseRFC822Date(const std::string &pubDate)
// broken mail-/news-clients omit the time zone
if (*dateString) {
if ((strncasecmp(dateString, "gmt", 3) == 0) ||
- (strncasecmp(dateString, "utc", 3) == 0))
+ (strncasecmp(dateString, "utc", 3) == 0))
{
dateString += 3;
while(*dateString && isspace(*dateString))
@@ -942,7 +973,10 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
item->feedId = feed.feedId;
item->title = title;
- getChildText(mCharEncodingHandler, node, "link", item->link);
+ /* try feedburner:origLink */
+ if (!getChildText(mCharEncodingHandler, node, "origLink", item->link) || item->link.empty()) {
+ getChildText(mCharEncodingHandler, node, "link", item->link);
+ }
long todo; // remove sid
// // remove sid=
@@ -969,6 +1003,7 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
// }
getChildText(mCharEncodingHandler, node, "author", item->author);
+
getChildText(mCharEncodingHandler, node, "description", item->description);
std::string pubDate;
@@ -1007,3 +1042,148 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
return result;
}
+
+std::string p3FeedReaderThread::getProxyForFeed(const RsFeedReaderFeed &feed)
+{
+ std::string proxy;
+ if (feed.flag & RS_FEED_FLAG_STANDARD_PROXY) {
+ std::string standardProxyAddress;
+ uint16_t standardProxyPort;
+ if (mFeedReader->getStandardProxy(standardProxyAddress, standardProxyPort)) {
+ rs_sprintf(proxy, "%s:%u", standardProxyAddress.c_str(), standardProxyPort);
+ }
+ } else {
+ if (!feed.proxyAddress.empty() && feed.proxyPort) {
+ rs_sprintf(proxy, "%s:%u", feed.proxyAddress.c_str(), feed.proxyPort);
+ }
+ }
+ return proxy;
+}
+
+bool p3FeedReaderThread::processMsg(const RsFeedReaderFeed &feed, RsFeedReaderMsg *msg)
+{
+ if (!msg) {
+ return false;
+ }
+
+ std::string proxy = getProxyForFeed(feed);
+
+ std::string url;
+ if (feed.flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE) {
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download page " << msg->link << std::endl;
+#endif
+ std::string content;
+ CURLWrapper CURL(proxy);
+ CURLcode code = CURL.downloadText(msg->link, content);
+
+ if (code == CURLE_OK && CURL.responseCode() == 200 && isContentType(CURL.contentType(), "text/html")) {
+ /* ok */
+ msg->description = content;
+ } else {
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot download page, CURLCode = " << code << ", responseCode = " << CURL.responseCode() << ", contentType = " << CURL.contentType() << std::endl;
+#endif
+ return false;
+ }
+
+ /* get effective url (moved location) */
+ std::string effectiveUrl = CURL.effectiveUrl();
+ url = getBaseLink(effectiveUrl.empty() ? msg->link : effectiveUrl);
+ }
+
+ /* check if string contains xml chars (very simple test) */
+ if (msg->description.find('<') == std::string::npos) {
+ return true;
+ }
+
+ bool result = true;
+
+ /* process description */
+ long todo; // encoding
+ htmlDocPtr document = htmlReadMemory(msg->description.c_str(), msg->description.length(), url.c_str(), "", HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING | HTML_PARSE_COMPACT);
+ if (document) {
+ xmlNodePtr root = xmlDocGetRootElement(document);
+ if (root) {
+ /* process all children */
+ std::list parents;
+ parents.push_back(root);
+
+ while (!parents.empty()) {
+ if (!isRunning()) {
+ break;
+ }
+ xmlNodePtr node = parents.front();
+ parents.pop_front();
+
+ if (node->type == XML_ELEMENT_NODE) {
+ /* check for image */
+ if (strcasecmp((char*) node->name, "img") == 0) {
+ bool removeImage = true;
+
+ if (feed.flag & RS_FEED_FLAG_EMBED_IMAGES) {
+ /* embed image */
+ std::string src = xmlGetAttr(mCharEncodingHandler, node, "src");
+ if (!src.empty()) {
+ /* download image */
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download image " << src << std::endl;
+#endif
+ std::vector data;
+ CURLWrapper CURL(proxy);
+ CURLcode code = CURL.downloadBinary(calculateLink(url, src), data);
+ if (code == CURLE_OK && CURL.responseCode() == 200) {
+ std::string contentType = CURL.contentType();
+ if (isContentType(contentType, "image/")) {
+ std::string base64;
+ if (toBase64(data, base64)) {
+ std::string imageBase64;
+ rs_sprintf(imageBase64, "data:%s;base64,%s", contentType.c_str(), base64.c_str());
+ if (xmlSetAttr(mCharEncodingHandler, node, "src", imageBase64.c_str())) {
+ removeImage = false;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (removeImage) {
+ /* remove image */
+ xmlUnlinkNode(node);
+ xmlFreeNode(node);
+ continue;
+ }
+ }
+
+ xmlNodePtr child;
+ for (child = node->children; child; child = child->next) {
+ parents.push_back(child);
+ }
+ }
+ }
+
+ xmlChar *html = NULL;
+ int htmlSize = 0;
+ htmlDocDumpMemoryFormat(document, &html, &htmlSize, 0);
+ if (html) {
+ convertToString((xmlCharEncodingHandlerPtr) mCharEncodingHandler, html, msg->description);
+ xmlFree(html);
+ } else {
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot dump html" << std::endl;
+#endif
+ result = false;
+ }
+ } else {
+#ifdef FEEDREADER_DEBUG
+ std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") no root element" << std::endl;
+#endif
+ result = false;
+ }
+
+ xmlFreeDoc(document);
+ }
+
+ return result;
+}
diff --git a/plugins/FeedReader/services/p3FeedReaderThread.h b/plugins/FeedReader/services/p3FeedReaderThread.h
index 051cd0325..b405b8822 100644
--- a/plugins/FeedReader/services/p3FeedReaderThread.h
+++ b/plugins/FeedReader/services/p3FeedReaderThread.h
@@ -19,7 +19,7 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
- #ifndef P3_FEEDREADERTHREAD
+#ifndef P3_FEEDREADERTHREAD
#define P3_FEEDREADERTHREAD
#include "util/rsthreads.h"
@@ -40,7 +40,6 @@ public:
enum DownloadResult
{
DOWNLOAD_SUCCESS,
- DOWNLOAD_ERROR_INIT,
DOWNLOAD_ERROR,
DOWNLOAD_UNKNOWN_CONTENT_TYPE,
DOWNLOAD_NOT_FOUND,
@@ -64,6 +63,9 @@ private:
DownloadResult download(const RsFeedReaderFeed &feed, std::string &content, std::string &icon, std::string &error);
ProcessResult process(const RsFeedReaderFeed &feed, std::list &entries, std::string &error);
+ std::string getProxyForFeed(const RsFeedReaderFeed &feed);
+ bool processMsg(const RsFeedReaderFeed &feed, RsFeedReaderMsg *msg);
+
p3FeedReader *mFeedReader;
Type mType;
/*xmlCharEncodingHandlerPtr*/ void *mCharEncodingHandler;
diff --git a/plugins/FeedReader/services/rsFeedReaderItems.h b/plugins/FeedReader/services/rsFeedReaderItems.h
index 92c911864..7f6703290 100644
--- a/plugins/FeedReader/services/rsFeedReaderItems.h
+++ b/plugins/FeedReader/services/rsFeedReaderItems.h
@@ -19,7 +19,7 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
- #ifndef RS_FEEDREADER_ITEMS_H
+#ifndef RS_FEEDREADER_ITEMS_H
#define RS_FEEDREADER_ITEMS_H
#include "serialiser/rsserial.h"
@@ -55,6 +55,8 @@ const uint8_t RS_PKT_SUBTYPE_FEEDREADER_MSG = 0x03;
#define RS_FEED_FLAG_DEACTIVATED 0x040
#define RS_FEED_FLAG_FORUM 0x080
#define RS_FEED_FLAG_UPDATE_FORUM_INFO 0x100
+#define RS_FEED_FLAG_EMBED_IMAGES 0x200
+#define RS_FEED_FLAG_SAVE_COMPLETE_PAGE 0x400
class RsFeedReaderFeed : public RsItem
{
diff --git a/plugins/FeedReader/services/util/CURLWrapper.cc b/plugins/FeedReader/services/util/CURLWrapper.cc
new file mode 100644
index 000000000..e39d5e99c
--- /dev/null
+++ b/plugins/FeedReader/services/util/CURLWrapper.cc
@@ -0,0 +1,136 @@
+/****************************************************************
+ * RetroShare GUI is distributed under the following license:
+ *
+ * Copyright (C) 2012 by Thunder
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ ****************************************************************/
+
+#include "CURLWrapper.h"
+
+//static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/)
+//{
+// p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp;
+
+// if (!thread->isRunning()) {
+// /* thread was stopped */
+// return 1;
+// }
+
+// long todo; // show progress in gui
+
+// return 0;
+//}
+
+CURLWrapper::CURLWrapper(const std::string &proxy)
+{
+ mCurl = curl_easy_init();
+ if (mCurl) {
+ curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 0);
+// curl_easy_setopt(mCurl, CURLOPT_PROGRESSFUNCTION, progressCallback);
+// curl_easy_setopt(mCurl, CURLOPT_PROGRESSDATA, feedReader);
+ curl_easy_setopt(mCurl, CURLOPT_FOLLOWLOCATION, 1);
+ curl_easy_setopt(mCurl, CURLOPT_CONNECTTIMEOUT, 60);
+ curl_easy_setopt(mCurl, CURLOPT_TIMEOUT, 120);
+
+ if (!proxy.empty()) {
+ curl_easy_setopt(mCurl, CURLOPT_PROXY, proxy.c_str());
+ }
+ }
+}
+
+CURLWrapper::~CURLWrapper()
+{
+ if (mCurl) {
+ curl_easy_cleanup(mCurl);
+ }
+}
+
+static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+ std::string *s = (std::string*) stream;
+ s->append ((char*) ptr, size * nmemb);
+
+ return nmemb * size;
+}
+
+CURLcode CURLWrapper::downloadText(const std::string &link, std::string &data)
+{
+ data.clear();
+
+ if (!mCurl) {
+ return CURLE_FAILED_INIT;
+ }
+
+ curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str());
+ curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionString);
+ curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data);
+
+ return curl_easy_perform(mCurl);
+}
+
+static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+ std::vector *bytes = (std::vector*) stream;
+
+ std::vector newBytes;
+ newBytes.resize(size * nmemb);
+ memcpy(newBytes.data(), ptr, newBytes.size());
+
+ bytes->insert(bytes->end(), newBytes.begin(), newBytes.end());
+
+ return nmemb * size;
+}
+
+CURLcode CURLWrapper::downloadBinary(const std::string &link, std::vector &data)
+{
+ data.clear();
+
+ if (!mCurl) {
+ return CURLE_FAILED_INIT;
+ }
+
+ curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 1);
+ curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str());
+ curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionBinary);
+ curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data);
+
+ return curl_easy_perform(mCurl);
+}
+
+long CURLWrapper::longInfo(CURLINFO info)
+{
+ if (!mCurl) {
+ return 0;
+ }
+
+ long value;
+ curl_easy_getinfo(mCurl, info, &value);
+
+ return value;
+}
+
+std::string CURLWrapper::stringInfo(CURLINFO info)
+{
+ if (!mCurl) {
+ return "";
+ }
+
+ char *value;
+ curl_easy_getinfo(mCurl, info, &value);
+
+ return value ? value : "";
+}
diff --git a/plugins/FeedReader/services/util/CURLWrapper.h b/plugins/FeedReader/services/util/CURLWrapper.h
new file mode 100644
index 000000000..66ac68f53
--- /dev/null
+++ b/plugins/FeedReader/services/util/CURLWrapper.h
@@ -0,0 +1,50 @@
+/****************************************************************
+ * RetroShare GUI is distributed under the following license:
+ *
+ * Copyright (C) 2012 by Thunder
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ * Boston, MA 02110-1301, USA.
+ ****************************************************************/
+
+#ifndef CURLWRAPPER
+#define CURLWRAPPER
+
+#include
+#include
+#include
+
+class CURLWrapper
+{
+public:
+ CURLWrapper(const std::string &proxy);
+ ~CURLWrapper();
+
+ CURLcode downloadText(const std::string &link, std::string &data);
+ CURLcode downloadBinary(const std::string &link, std::vector &data);
+
+ long responseCode() { return longInfo(CURLINFO_RESPONSE_CODE); }
+ std::string contentType() { return stringInfo(CURLINFO_CONTENT_TYPE); }
+ std::string effectiveUrl() { return stringInfo(CURLINFO_EFFECTIVE_URL); }
+
+protected:
+ long longInfo(CURLINFO info);
+ std::string stringInfo(CURLINFO info);
+
+private:
+ CURL *mCurl;
+};
+
+#endif