Git-Mirrors/RetroShare
1
0
Fork 0
mirror of https://github.com/RetroShare/RetroShare.git synced 2025-01-26 15:28:28 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added new button to open the link of the message in browser or copy the link of the message.

Browse Source 
Added "RSS: " for the forum feeds.
Parse the feedburner:origLink in the rss feed.
Moved download functions to a new class CURLWrapper for easy use.
Added two new functions (currently only for local feeds for testing):
- embed images into the message (works for Qt 4.7 and higher)
- save complete web page

git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs-b1@5399 b45a01b8-16f6-495d-af2f-9b41ad6348cc
This commit is contained in:
thunder2 2012-08-10 18:06:29 +00:00
parent 8f2ff3eaf5
commit 148d1310a2
17 changed files with 806 additions and 237 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
plugins/FeedReader/FeedReader.pro
View File

@ -6,6 +6,7 @@ SOURCES = FeedReaderPlugin.cpp \
services/p3FeedReader.cc \
services/p3FeedReaderThread.cc \
services/rsFeedReaderItems.cc \
services/util/CURLWrapper.cc \
gui/FeedReaderDialog.cpp \
gui/AddFeedDialog.cpp \
gui/FeedReaderNotify.cpp \
@ -16,6 +17,7 @@ HEADERS = FeedReaderPlugin.h \
services/p3FeedReader.h \
services/p3FeedReaderThread.h \
services/rsFeedReaderItems.h \
services/util/CURLWrapper.h \
gui/FeedReaderDialog.h \
gui/AddFeedDialog.h \
gui/FeedReaderNotify.h \
@ -39,7 +41,7 @@ linux-* {
}
win32 {
DEFINES += CURL_STATICLIB
DEFINES += CURL_STATICLIB LIBXML_STATIC
CURL_DIR = ../../../curl-7.26.0
LIBXML2_DIR = ../../../libxml2-2.8.0

32
plugins/FeedReader/gui/AddFeedDialog.cpp
View File

@ -46,12 +46,17 @@ AddFeedDialog::AddFeedDialog(RsFeedReader *feedReader, QWidget *parent)
connect(ui->useStandardProxyCheckBox, SIGNAL(toggled(bool)), this, SLOT(useStandardProxyToggled()));
connect(ui->typeForumRadio, SIGNAL(toggled(bool)), this, SLOT(typeForumToggled()));
/* currently only for loacl feeds */
connect(ui->saveCompletePageCheckBox, SIGNAL(toggled(bool)), this, SLOT(denyForumToggled()));
connect(ui->embedImagesCheckBox, SIGNAL(toggled(bool)), this, SLOT(denyForumToggled()));
connect(ui->urlLineEdit, SIGNAL(textChanged(QString)), this, SLOT(validate()));
connect(ui->nameLineEdit, SIGNAL(textChanged(QString)), this, SLOT(validate()));
connect(ui->useInfoFromFeedCheckBox, SIGNAL(toggled(bool)), this, SLOT(validate()));
connect(ui->typeLocalRadio, SIGNAL(toggled(bool)), this, SLOT(validate()));
connect(ui->typeForumRadio, SIGNAL(toggled(bool)), this, SLOT(validate()));
ui->activatedCheckBox->setChecked(true);
ui->typeLocalRadio->setChecked(true);
ui->forumComboBox->setEnabled(false);
ui->useInfoFromFeedCheckBox->setChecked(true);
ui->updateForumInfoCheckBox->setEnabled(false);
@ -124,6 +129,16 @@ void AddFeedDialog::typeForumToggled()
ui->updateForumInfoCheckBox->setEnabled(checked);
}
void AddFeedDialog::denyForumToggled()
{
if (ui->saveCompletePageCheckBox->isChecked() || ui->embedImagesCheckBox->isChecked()) {
ui->typeForumRadio->setEnabled(false);
ui->typeLocalRadio->setChecked(true);
} else {
ui->typeForumRadio->setEnabled(true);
}
}
void AddFeedDialog::validate()
{
bool ok = true;
@ -134,6 +149,9 @@ void AddFeedDialog::validate()
if (ui->nameLineEdit->text().isEmpty() && !ui->useInfoFromFeedCheckBox->isChecked()) {
ok = false;
}
if (!ui->typeLocalRadio->isChecked() && !ui->typeForumRadio->isChecked()) {
ok = false;
}
ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(ok);
}
@ -196,6 +214,8 @@ bool AddFeedDialog::fillFeed(const std::string &feedId)
ui->useInfoFromFeedCheckBox->setChecked(feedInfo.flag.infoFromFeed);
ui->updateForumInfoCheckBox->setChecked(feedInfo.flag.updateForumInfo);
ui->activatedCheckBox->setChecked(!feedInfo.flag.deactivated);
ui->embedImagesCheckBox->setChecked(feedInfo.flag.embedImages);
ui->saveCompletePageCheckBox->setChecked(feedInfo.flag.saveCompletePage);
ui->descriptionPlainTextEdit->setPlainText(QString::fromUtf8(feedInfo.description.c_str()));
@ -206,6 +226,8 @@ bool AddFeedDialog::fillFeed(const std::string &feedId)
if (feedInfo.flag.forum) {
ui->typeForumRadio->setChecked(true);
ui->saveCompletePageCheckBox->setEnabled(false);
ui->embedImagesCheckBox->setEnabled(false);
if (feedInfo.forumId.empty()) {
ui->forumNameLabel->setText(tr("Not yet created"));
@ -259,13 +281,17 @@ void AddFeedDialog::createFeed()
feedInfo.flag.infoFromFeed = ui->useInfoFromFeedCheckBox->isChecked();
feedInfo.flag.updateForumInfo = ui->updateForumInfoCheckBox->isChecked() && ui->updateForumInfoCheckBox->isEnabled();
feedInfo.flag.deactivated = !ui->activatedCheckBox->isChecked();
feedInfo.flag.embedImages = ui->embedImagesCheckBox->isChecked();
feedInfo.flag.saveCompletePage = ui->saveCompletePageCheckBox->isChecked();
feedInfo.description = ui->descriptionPlainTextEdit->toPlainText().toUtf8().constData();
feedInfo.flag.forum = ui->typeForumRadio->isChecked();
if (mFeedId.empty()) {
/* set forum (only when create a new feed) */
feedInfo.forumId = ui->forumComboBox->itemData(ui->forumComboBox->currentIndex()).toString().toStdString();
if (feedInfo.flag.forum) {
/* set forum (only when create a new feed) */
feedInfo.forumId = ui->forumComboBox->itemData(ui->forumComboBox->currentIndex()).toString().toStdString();
}
}
feedInfo.flag.authentication = ui->useAuthenticationCheckBox->isChecked();

1
plugins/FeedReader/gui/AddFeedDialog.h
View File

@ -50,6 +50,7 @@ private slots:
void useStandardUpdateIntervalToggled();
void useStandardProxyToggled();
void typeForumToggled();
void denyForumToggled();
void validate();
void createFeed();

31
plugins/FeedReader/gui/AddFeedDialog.ui
View File

@ -7,7 +7,7 @@
<x>0</x>
<y>0</y>
<width>715</width>
<height>559</height>
<height>605</height>
</rect>
</property>
<property name="windowTitle">
@ -289,13 +289,6 @@ p, li { white-space: pre-wrap; }
<property name="margin">
<number>6</number>
</property>
<item>
<widget class="QRadioButton" name="typeLocalRadio">
<property name="text">
<string>Local Feed</string>
</property>
</widget>
</item>
<item>
<layout class="QHBoxLayout" name="horizontalLayout_2">
<item>
@ -323,6 +316,13 @@ p, li { white-space: pre-wrap; }
</item>
</layout>
</item>
<item>
<widget class="QRadioButton" name="typeLocalRadio">
<property name="text">
<string>Local Feed</string>
</property>
</widget>
</item>
<item>
<spacer name="verticalSpacer">
<property name="orientation">
@ -366,6 +366,20 @@ p, li { white-space: pre-wrap; }
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="embedImagesCheckBox">
<property name="text">
<string>Embed images (experimental for local feeds)</string>
</property>
</widget>
</item>
<item>
<widget class="QCheckBox" name="saveCompletePageCheckBox">
<property name="text">
<string>Save complete web page (experimental for local feeds)</string>
</property>
</widget>
</item>
</layout>
</widget>
</item>
@ -416,7 +430,6 @@ p, li { white-space: pre-wrap; }
<tabstop>urlLineEdit</tabstop>
<tabstop>nameLineEdit</tabstop>
<tabstop>descriptionPlainTextEdit</tabstop>
<tabstop>typeLocalRadio</tabstop>
<tabstop>typeForumRadio</tabstop>
<tabstop>forumComboBox</tabstop>
<tabstop>activatedCheckBox</tabstop>

60
plugins/FeedReader/gui/FeedReaderDialog.cpp
View File

@ -27,6 +27,7 @@
#include <QPainter>
#include <QMessageBox>
#include <QClipboard>
#include <QDesktopServices>
#include "FeedReaderDialog.h"
#include "ui_FeedReaderDialog.h"
@ -118,6 +119,7 @@ FeedReaderDialog::FeedReaderDialog(RsFeedReader *feedReader, QWidget *parent)
connect(ui->filterLineEdit, SIGNAL(textChanged(QString)), this, SLOT(filterItems(QString)));
connect(ui->filterColumnComboBox, SIGNAL(currentIndexChanged(int)), this, SLOT(filterColumnChanged()));
connect(ui->linkButton, SIGNAL(clicked()), this, SLOT(openLinkMsg()));
connect(ui->expandButton, SIGNAL(clicked()), this, SLOT(toggleMsgText()));
mFeedCompareRole = new RSTreeWidgetItemCompareRole;
@ -168,6 +170,18 @@ FeedReaderDialog::FeedReaderDialog(RsFeedReader *feedReader, QWidget *parent)
/* initialize feed list */
ui->feedTreeWidget->sortItems(COLUMN_FEED_NAME, Qt::AscendingOrder);
/* build menu for link button */
QMenu *menu = new QMenu(this);
QAction *action = menu->addAction(tr("Open link in browser"), this, SLOT(openLinkMsg()));
menu->addAction(tr("Copy link to clipboard"), this, SLOT(copyLinkMsg()));
QFont font = action->font();
font.setBold(true);
action->setFont(font);
ui->linkButton->setMenu(menu);
ui->linkButton->setEnabled(false);
ui->msgTreeWidget->installEventFilter(this);
}
@ -351,7 +365,7 @@ void FeedReaderDialog::msgTreeCustomPopupMenu(QPoint /*point*/)
contextMnu.addSeparator();
action = contextMnu.addAction(QIcon(""), tr("Copy link"), this, SLOT(copyLinkMsg()));
action = contextMnu.addAction(QIcon(""), tr("Copy link"), this, SLOT(copyLinskMsg()));
action->setEnabled(!selectedItems.empty());
action = contextMnu.addAction(QIcon(""), tr("Remove"), this, SLOT(removeMsg()));
@ -522,10 +536,14 @@ void FeedReaderDialog::updateFeedItem(QTreeWidgetItem *item, FeedInfo &info)
case FeedInfo::WAITING:
break;
case FeedInfo::WAITING_TO_DOWNLOAD:
workState = tr("waiting for download");
break;
case FeedInfo::DOWNLOADING:
workState = tr("loading");
workState = tr("downloading");
break;
case FeedInfo::WAITING_TO_PROCESS:
workState = tr("waiting for process");
break;
case FeedInfo::PROCESSING:
workState = tr("processing");
break;
@ -791,6 +809,7 @@ void FeedReaderDialog::msgItemChanged()
ui->msgTitle->clear();
// ui->msgLink->clear();
ui->msgText->clear();
ui->linkButton->setEnabled(false);
return;
}
@ -800,6 +819,7 @@ void FeedReaderDialog::msgItemChanged()
ui->msgTitle->clear();
// ui->msgLink->clear();
ui->msgText->clear();
ui->linkButton->setEnabled(false);
return;
}
@ -809,6 +829,7 @@ void FeedReaderDialog::msgItemChanged()
ui->msgTitle->clear();
// ui->msgLink->clear();
ui->msgText->clear();
ui->linkButton->setEnabled(false);
return;
}
@ -832,7 +853,8 @@ void FeedReaderDialog::msgItemChanged()
ui->msgText->setHtml(msgTxt);
ui->msgTitle->setText(QString::fromUtf8(msgInfo.title.c_str()));
// ui->msgLink->setHtml(RsHtml().formatText(NULL, QString::fromUtf8(msgInfo.link.c_str()), RSHTML_FORMATTEXT_EMBED_LINKS));
ui->linkButton->setEnabled(!msgInfo.link.empty());
}
void FeedReaderDialog::setMsgAsReadUnread(QList<QTreeWidgetItem *> &rows, bool read)
@ -1049,7 +1071,7 @@ void FeedReaderDialog::markAllAsReadMsg()
setMsgAsReadUnread(items, true);
}
void FeedReaderDialog::copyLinkMsg()
void FeedReaderDialog::copyLinksMsg()
{
QString links;
@ -1086,3 +1108,33 @@ void FeedReaderDialog::removeMsg()
}
mFeedReader->removeMsgs(feedId, msgIds);
}
void FeedReaderDialog::copyLinkMsg()
{
QTreeWidgetItem *item = ui->msgTreeWidget->currentItem();
if (!item) {
return;
}
QString link = item->data(COLUMN_MSG_DATA, ROLE_MSG_LINK).toString();
if (link.isEmpty()) {
return;
}
QApplication::clipboard()->setText(link);
}
void FeedReaderDialog::openLinkMsg()
{
QTreeWidgetItem *item = ui->msgTreeWidget->currentItem();
if (!item) {
return;
}
QString link = item->data(COLUMN_MSG_DATA, ROLE_MSG_LINK).toString();
if (link.isEmpty()) {
return;
}
QDesktopServices::openUrl(QUrl(link));
}

4
plugins/FeedReader/gui/FeedReaderDialog.h
View File

@ -64,8 +64,10 @@ private slots:
void markAsReadMsg();
void markAsUnreadMsg();
void markAllAsReadMsg();
void copyLinkMsg();
void copyLinksMsg();
void removeMsg();
void openLinkMsg();
void copyLinkMsg();
/* FeedReaderNotify */
void feedChanged(const QString &feedId, int type);

22
plugins/FeedReader/gui/FeedReaderDialog.ui
View File

@ -268,8 +268,8 @@ border: 1px solid #CCCCCC;}</string>
</widget>
</item>
<item row="4" column="0">
<layout class="QGridLayout" name="navFrame">
<item row="0" column="0">
<layout class="QHBoxLayout" name="horizontalLayout_3">
<item>
<widget class="QLabel" name="msgLabel">
<property name="sizePolicy">
<sizepolicy hsizetype="Fixed" vsizetype="Preferred">
@ -289,7 +289,7 @@ border: 1px solid #CCCCCC;}</string>
</property>
</widget>
</item>
<item row="0" column="1">
<item>
<widget class="QLabel" name="msgTitle">
<property name="styleSheet">
<string notr="true">QLabel#msgTitle{
@ -305,7 +305,21 @@ background: white;}</string>
</property>
</widget>
</item>
<item row="0" column="2">
<item>
<widget class="QToolButton" name="linkButton">
<property name="icon">
<iconset resource="FeedReader_images.qrc">
<normaloff>:/images/Link.png</normaloff>:/images/Link.png</iconset>
</property>
<property name="popupMode">
<enum>QToolButton::MenuButtonPopup</enum>
</property>
<property name="autoRaise">
<bool>true</bool>
</property>
</widget>
</item>
<item>
<widget class="QPushButton" name="expandButton">
<property name="maximumSize">
<size>

1
plugins/FeedReader/gui/FeedReader_images.qrc
View File

@ -8,6 +8,7 @@
<file>images/FeedErrorOverlay.png</file>
<file>images/FolderAdd.png</file>
<file>images/FeedAdd.png</file>
<file>images/Link.png</file>
<file>images/Update.png</file>
</qresource>
</RCC>

BIN
plugins/FeedReader/gui/images/Link.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 2.7 KiB

6
plugins/FeedReader/interface/rsFeedReader.h
View File

@ -19,7 +19,7 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
#ifndef RETROSHARE_FEEDREADER_GUI_INTERFACE_H
#ifndef RETROSHARE_FEEDREADER_GUI_INTERFACE_H
#define RETROSHARE_FEEDREADER_GUI_INTERFACE_H
#include <inttypes.h>
@ -68,6 +68,8 @@ public:
flag.deactivated = false;
flag.forum = false;
flag.updateForumInfo = false;
flag.embedImages = false;
flag.saveCompletePage = false;
}
std::string feedId;
@ -98,6 +100,8 @@ public:
bool deactivated : 1;
bool forum : 1;
bool updateForumInfo : 1;
bool embedImages : 1;
bool saveCompletePage : 1;
} flag;
};

181
plugins/FeedReader/services/p3FeedReader.cc
View File

@ -30,6 +30,7 @@
RsFeedReader *rsFeedReader = NULL;
#define FEEDREADER_CLEAN_INTERVAL 1 * 60 * 60 // check every hour
#define FEEDREADER_FORUM_PREFIX L"RSS: "
/*********
* #define FEEDREADER_DEBUG
@ -91,6 +92,8 @@ static void feedToInfo(const RsFeedReaderFeed *feed, FeedInfo &info)
info.flag.deactivated = (feed->flag & RS_FEED_FLAG_DEACTIVATED);
info.flag.forum = (feed->flag & RS_FEED_FLAG_FORUM);
info.flag.updateForumInfo = (feed->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO);
info.flag.embedImages = (feed->flag & RS_FEED_FLAG_EMBED_IMAGES);
info.flag.saveCompletePage = (feed->flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE);
switch (feed->workstate) {
case RsFeedReaderFeed::WAITING:
@ -151,6 +154,12 @@ static void infoToFeed(const FeedInfo &info, RsFeedReaderFeed *feed, bool add)
if (info.flag.deactivated) {
feed->flag |= RS_FEED_FLAG_DEACTIVATED;
}
if (info.flag.embedImages) {
feed->flag |= RS_FEED_FLAG_EMBED_IMAGES;
}
if (info.flag.saveCompletePage) {
feed->flag |= RS_FEED_FLAG_SAVE_COMPLETE_PAGE;
}
if (add) {
/* only set when adding a new feed */
if (info.flag.folder) {
@ -950,6 +959,11 @@ int p3FeedReader::tick()
} else {
updateInterval = fi->updateInterval;
}
if (updateInterval == 0) {
continue;
}
if (fi->lastUpdate == 0 || fi->lastUpdate + (long) updateInterval <= currentTime) {
/* add to download list */
feedToDownload.push_back(fi->feedId);
@ -1340,7 +1354,6 @@ void p3FeedReader::onDownloadError(const std::string &feedId, p3FeedReaderThread
case p3FeedReaderThread::DOWNLOAD_ERROR:
fi->errorState = RS_FEED_ERRORSTATE_DOWNLOAD_ERROR;
break;
case p3FeedReaderThread::DOWNLOAD_ERROR_INIT:
case p3FeedReaderThread::DOWNLOAD_INTERNAL_ERROR:
fi->errorState = RS_FEED_ERRORSTATE_DOWNLOAD_INTERNAL_ERROR;
break;
@ -1428,15 +1441,13 @@ bool p3FeedReader::getFeedToProcess(RsFeedReaderFeed &feed)
return true;
}
void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedReaderMsg*> &msgs)
bool p3FeedReader::onProcessSuccess_filterMsg(const std::string &feedId, std::list<RsFeedReaderMsg*> &msgs)
{
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::onProcessSuccess - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
#endif
std::list<std::string> addedMsgs;
std::string forumId;
std::list<RsFeedReaderMsg> forumMsgs;
bool result = true;
{
RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
@ -1446,9 +1457,9 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
if (it == mFeeds.end()) {
/* feed not found */
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::onProcessSuccess - feed " << feedId << " not found" << std::endl;
std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - feed " << feedId << " not found" << std::endl;
#endif
return;
return false;
}
RsFeedReaderFeed *fi = it->second;
@ -1458,6 +1469,10 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
if (forum) {
if (fi->forumId.empty()) {
/* create new forum */
std::wstring forumName;
librs::util::ConvertUtf8ToUtf16(fi->name, forumName);
forumName.insert(0, FEEDREADER_FORUM_PREFIX);
long todo; // search for existing forum?
/* search for existing own forum */
@ -1472,22 +1487,18 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
// }
// }
std::wstring forumName;
librs::util::ConvertUtf8ToUtf16(fi->name, forumName);
std::wstring forumDescription;
librs::util::ConvertUtf8ToUtf16(fi->description, forumDescription);
/* create anonymous public forum */
fi->forumId = rsForums->createForum(forumName, forumDescription, RS_DISTRIB_PUBLIC | RS_DISTRIB_AUTHEN_ANON);
forumId = fi->forumId;
if (fi->forumId.empty()) {
errorState = RS_FEED_ERRORSTATE_FORUM_CREATE;
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::onProcessSuccess - can't create forum for feed " << feedId << " (" << it->second->name << ") - ignore all messages" << std::endl;
std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - can't create forum for feed " << feedId << " (" << it->second->name << ") - ignore all messages" << std::endl;
} else {
std::cerr << "p3FeedReader::onProcessSuccess - forum " << forumId << " (" << fi->name << ") created" << std::endl;
std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - forum " << forumId << " (" << fi->name << ") created" << std::endl;
#endif
}
} else {
@ -1498,8 +1509,6 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
errorState = RS_FEED_ERRORSTATE_FORUM_NO_ADMIN;
} else if ((forumInfo.forumFlags & RS_DISTRIB_AUTHEN_REQ) || (forumInfo.forumFlags & RS_DISTRIB_AUTHEN_ANON) == 0) {
errorState = RS_FEED_ERRORSTATE_FORUM_NO_ANONYMOUS_FORUM;
} else {
forumId = fi->forumId;
}
} else {
errorState = RS_FEED_ERRORSTATE_FORUM_NOT_FOUND;
@ -1508,15 +1517,11 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
}
/* process msgs */
#ifdef FEEDREADER_DEBUG
uint32_t newMsgs = 0;
#endif
if (errorState == RS_FEED_ERRORSTATE_OK) {
std::list<RsFeedReaderMsg*>::iterator newMsgIt;
for (newMsgIt = msgs.begin(); newMsgIt != msgs.end(); ) {
RsFeedReaderMsg *miNew = *newMsgIt;
/* search for exisiting msg */
/* search for existing msg */
std::map<std::string, RsFeedReaderMsg*>::iterator msgIt;
for (msgIt = fi->mMsgs.begin(); msgIt != fi->mMsgs.end(); ++msgIt) {
RsFeedReaderMsg *mi = msgIt->second;
@ -1525,38 +1530,103 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
break;
}
}
if (msgIt == fi->mMsgs.end()) {
/* add new msg */
rs_sprintf(miNew->msgId, "%lu", mNextMsgId++);
if (forum) {
miNew->flag = RS_FEEDMSG_FLAG_DELETED;
forumMsgs.push_back(*miNew);
// miNew->description.clear();
} else {
miNew->flag = RS_FEEDMSG_FLAG_NEW;
addedMsgs.push_back(miNew->msgId);
}
fi->mMsgs[miNew->msgId] = miNew;
if (msgIt != fi->mMsgs.end()) {
/* msg exists */
delete(*newMsgIt);
newMsgIt = msgs.erase(newMsgIt);
#ifdef FEEDREADER_DEBUG
++newMsgs;
#endif
} else {
/* msg was updated */
++newMsgIt;
}
}
} else {
result = false;
}
fi->content.clear();
fi->errorState = errorState;
fi->errorString.clear();
IndicateConfigChanged();
}
if (mNotify) {
mNotify->feedChanged(feedId, NOTIFY_TYPE_MOD);
}
return result;
}
void p3FeedReader::onProcessSuccess_addMsgs(const std::string &feedId, bool result, std::list<RsFeedReaderMsg*> &msgs)
{
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::onProcessSuccess - feed " << fi->feedId << " (" << fi->name << ") added " << newMsgs << "/" << msgs.size() << " messages" << std::endl;
std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
#endif
std::list<std::string> addedMsgs;
std::string forumId;
std::list<RsFeedReaderMsg> forumMsgs;
{
RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
/* find feed */
std::map<std::string, RsFeedReaderFeed*>::iterator it = mFeeds.find(feedId);
if (it == mFeeds.end()) {
/* feed not found */
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << feedId << " not found" << std::endl;
#endif
return;
}
RsFeedReaderFeed *fi = it->second;
bool forum = (fi->flag & RS_FEED_FLAG_FORUM);
if (forum) {
forumId = fi->forumId;
if (forumId.empty()) {
/* don't process messages without forum id */
result = false;
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << fi->feedId << " (" << fi->name << ") don't process messages without forum id" << std::endl;
#endif
}
}
/* process msgs */
#ifdef FEEDREADER_DEBUG
uint32_t newMsgs = 0;
#endif
if (result) {
std::list<RsFeedReaderMsg*>::iterator newMsgIt;
for (newMsgIt = msgs.begin(); newMsgIt != msgs.end(); ) {
RsFeedReaderMsg *miNew = *newMsgIt;
/* add new msg */
rs_sprintf(miNew->msgId, "%lu", mNextMsgId++);
if (forum) {
miNew->flag = RS_FEEDMSG_FLAG_DELETED;
forumMsgs.push_back(*miNew);
// miNew->description.clear();
} else {
miNew->flag = RS_FEEDMSG_FLAG_NEW;
addedMsgs.push_back(miNew->msgId);
}
fi->mMsgs[miNew->msgId] = miNew;
newMsgIt = msgs.erase(newMsgIt);
#ifdef FEEDREADER_DEBUG
++newMsgs;
#endif
}
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << fi->feedId << " (" << fi->name << ") added " << newMsgs << "/" << msgs.size() << " messages" << std::endl;
#endif
}
fi->workstate = RsFeedReaderFeed::WAITING;
fi->content.clear();
fi->lastUpdate = time(NULL);
fi->errorState = errorState;
fi->errorString.clear();
IndicateConfigChanged();
}
@ -1584,7 +1654,7 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
rsForums->setMessageStatus(forumMsgInfo.forumId, forumMsgInfo.msgId, 0, FORUM_MSG_STATUS_MASK);
} else {
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::onProcessSuccess - can't add forum message " << mi.title << " for feed " << forumId << std::endl;
std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - can't add forum message " << mi.title << " for feed " << forumId << std::endl;
#endif
}
}
@ -1652,7 +1722,7 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
{
bool changed = false;
std::string forumId;
ForumInfo forumInfo;
ForumInfo forumInfoNew;
{
RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
@ -1683,11 +1753,12 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
changed = true;
}
if (changed && (fi->flag & RS_FEED_FLAG_FORUM) && (fi->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO) && !fi->forumId.empty()) {
if ((fi->flag & RS_FEED_FLAG_FORUM) && (fi->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO) && !fi->forumId.empty()) {
/* change forum too */
forumId = fi->forumId;
librs::util::ConvertUtf8ToUtf16(fi->name, forumInfo.forumName);
librs::util::ConvertUtf8ToUtf16(fi->description, forumInfo.forumDesc);
librs::util::ConvertUtf8ToUtf16(fi->name, forumInfoNew.forumName);
librs::util::ConvertUtf8ToUtf16(fi->description, forumInfoNew.forumDesc);
forumInfoNew.forumName.insert(0, FEEDREADER_FORUM_PREFIX);
}
}
@ -1700,10 +1771,22 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
}
if (!forumId.empty()) {
/* name or description changed, update forum */
if (!rsForums->setForumInfo(forumId, forumInfo)) {
ForumInfo forumInfo;
if (rsForums->getForumInfo(forumId, forumInfo)) {
if (forumInfo.forumName != forumInfoNew.forumName || forumInfo.forumDesc != forumInfoNew.forumDesc) {
/* name or description changed, update forum */
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::setFeed - can't change forum " << forumId << std::endl;
std::cerr << "p3FeedReader::setFeed - change forum " << forumId << std::endl;
#endif
if (!rsForums->setForumInfo(forumId, forumInfoNew)) {
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::setFeed - can't change forum " << forumId << std::endl;
#endif
}
}
} else {
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReader::setFeed - can't get forum info " << forumId << std::endl;
#endif
}
}

5
plugins/FeedReader/services/p3FeedReader.h
View File

@ -19,7 +19,7 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
#ifndef P3_FEEDREADER
#ifndef P3_FEEDREADER
#define P3_FEEDREADER
#include "retroshare/rsplugin.h"
@ -71,7 +71,8 @@ public:
bool getFeedToDownload(RsFeedReaderFeed &feed);
void onDownloadSuccess(const std::string &feedId, const std::string &content, std::string &icon);
void onDownloadError(const std::string &feedId, p3FeedReaderThread::DownloadResult result, const std::string &error);
void onProcessSuccess(const std::string &feedId, std::list<RsFeedReaderMsg*> &msgs);
bool onProcessSuccess_filterMsg(const std::string &feedId, std::list<RsFeedReaderMsg*> &msgs);
void onProcessSuccess_addMsgs(const std::string &feedId, bool result, std::list<RsFeedReaderMsg*> &msgs);
void onProcessError(const std::string &feedId, p3FeedReaderThread::ProcessResult result);
bool getFeedToProcess(RsFeedReaderFeed &feed);

500
plugins/FeedReader/services/p3FeedReaderThread.cc
View File

@ -22,9 +22,11 @@
#include "p3FeedReaderThread.h"
#include "rsFeedReaderItems.h"
#include "util/rsstring.h"
#include "util/CURLWrapper.h"
#include <curl/curl.h>
#include <libxml/parser.h>
#include <libxml/HTMLparser.h>
#include <libxml/HTMLtree.h>
#include <openssl/evp.h>
enum FeedFormat { FORMAT_RSS, FORMAT_RDF };
@ -32,6 +34,7 @@ enum FeedFormat { FORMAT_RSS, FORMAT_RDF };
/*********
* #define FEEDREADER_DEBUG
*********/
#define FEEDREADER_DEBUG
p3FeedReaderThread::p3FeedReaderThread(p3FeedReader *feedReader, Type type) : RsThread(), mFeedReader(feedReader), mType(type)
{
@ -88,21 +91,34 @@ void p3FeedReaderThread::run()
{
RsFeedReaderFeed feed;
if (mFeedReader->getFeedToProcess(feed)) {
std::list<RsFeedReaderMsg*> entries;
std::list<RsFeedReaderMsg*> msgs;
std::string error;
std::list<RsFeedReaderMsg*>::iterator it;
ProcessResult result = process(feed, entries, error);
ProcessResult result = process(feed, msgs, error);
if (result == PROCESS_SUCCESS) {
mFeedReader->onProcessSuccess(feed.feedId, entries);
/* first, filter the messages */
bool result = mFeedReader->onProcessSuccess_filterMsg(feed.feedId, msgs);
if (isRunning()) {
if (result) {
long todo; // process new items
/* second, process the descriptions */
for (it = msgs.begin(); it != msgs.end(); ++it) {
RsFeedReaderMsg *mi = *it;
processMsg(feed, mi);
}
}
/* third, add messages */
mFeedReader->onProcessSuccess_addMsgs(feed.feedId, result, msgs);
}
} else {
mFeedReader->onProcessError(feed.feedId, result);
}
std::list<RsFeedReaderMsg*>::iterator it;
for (it = entries.begin(); it != entries.end(); ++it) {
for (it = msgs.begin(); it != msgs.end(); ++it) {
delete (*it);
}
entries.clear();
msgs.clear();
}
}
break;
@ -114,121 +130,110 @@ void p3FeedReaderThread::run()
/****************************** Download ***********************************/
/***************************************************************************/
static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream)
static bool isContentType(const std::string &contentType, const char *type)
{
std::string *s = (std::string*) stream;
s->append ((char*) ptr, size * nmemb);
return nmemb * size;
return (strncasecmp(contentType.c_str(), type, strlen(type)) == 0);
}
static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream)
static bool toBase64(const std::vector<unsigned char> &data, std::string &base64)
{
std::vector<unsigned char> *bytes = (std::vector<unsigned char>*) stream;
bool result = false;
std::vector<unsigned char> newBytes;
newBytes.resize(size * nmemb);
memcpy(newBytes.data(), ptr, newBytes.size());
bytes->insert(bytes->end(), newBytes.begin(), newBytes.end());
return nmemb * size;
}
static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/)
{
p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp;
if (!thread->isRunning()) {
/* thread was stopped */
return 1;
/* Set up a base64 encoding BIO that writes to a memory BIO */
BIO *b64 = BIO_new(BIO_f_base64());
if (b64) {
BIO_set_flags(b64, BIO_FLAGS_BASE64_NO_NL);
BIO *bmem = BIO_new(BIO_s_mem());
if (bmem) {
BIO_set_flags(bmem, BIO_CLOSE); // probably redundant
b64 = BIO_push(b64, bmem);
/* Send the data */
BIO_write(b64, data.data(), data.size());
/* Collect the encoded data */
BIO_flush(b64);
char* temp;
int count = BIO_get_mem_data(bmem, &temp);
if (count && temp) {
base64.assign(temp, count);
result = true;
}
}
BIO_free_all(b64);
}
long todo; // show progress in gui
return 0;
return result;
}
static bool getFavicon (std::string url, const std::string &proxy, std::string &icon)
static std::string getBaseLink(std::string link)
{
size_t found = link.rfind('/');
if (found != std::string::npos) {
link.erase(found + 1);
}
return link;
}
static std::string calculateLink(const std::string &baseLink, const std::string &link)
{
if (link.substr(0, 7) == "http://") {
/* absolute link */
return link;
}
/* calculate link of base link */
std::string resultLink = baseLink;
/* link should begin with "http://" */
if (resultLink.substr(0, 7) != "http://") {
resultLink.insert(0, "http://");
}
if (link.empty()) {
/* no link */
return resultLink;
}
if (*link.begin() == '/') {
/* link begins with "/" */
size_t found = resultLink.find('/', 7);
if (found != std::string::npos) {
resultLink.erase(found);
}
} else {
/* check for "/" at the end */
std::string::reverse_iterator it = resultLink.rend ();
it--;
if (*it != '/') {
resultLink += "/";
}
}
resultLink += link;
return resultLink;
}
static bool getFavicon(CURLWrapper &CURL, const std::string &url, std::string &icon)
{
icon.clear();
if (url.substr(0, 7) == "http://") {
int found = url.find("/", 7);
if (found >= 0) {
url.erase(found, url.length() - found);
}
} else {
return false;
}
bool result = false;
CURL *curl = curl_easy_init();
if (curl) {
url += "/favicon.ico";
std::vector<unsigned char> vicon;
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionBinary);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &vicon);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
if (!proxy.empty()) {
curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str());
}
CURLcode code = curl_easy_perform(curl);
if (code == CURLE_OK) {
long response;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
if (response == 200) {
char *contentType = NULL;
curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &contentType);
if (contentType &&
(strncasecmp(contentType, "image/x-icon", 12) == 0 ||
strncasecmp(contentType, "application/octet-stream", 24) == 0 ||
strncasecmp(contentType, "text/plain", 10) == 0)) {
if (!vicon.empty()) {
long todo; // check it
// Set up a base64 encoding BIO that writes to a memory BIO
BIO *b64 = BIO_new(BIO_f_base64());
if (b64) {
BIO *out = BIO_new(BIO_s_mem());
if (out) {
BIO_set_flags(out, BIO_CLOSE); // probably redundant
b64 = BIO_push(b64, out);
// Send the data
BIO_write(b64, vicon.data(), vicon.size());
// Collect the encoded data
BIO_flush(b64);
char* temp;
int count = BIO_get_mem_data(out, &temp);
if (count && temp) {
icon = temp;
result = true;
}
}
BIO_free_all(b64);
}
}
// char *encode = NULL;
// size_t encodeSize = 0;
// code = Curl_base64_encode(curl, (const char*) vicon.data(), vicon.size(), &encode, &encodeSize);
// if (code == CURLE_OK && encodeSize) {
// icon = encode;
// free(encode);
// encode = NULL;
// result = true;
// }
std::vector<unsigned char> vicon;
CURLcode code = CURL.downloadBinary(calculateLink(url, "/favicon.ico"), vicon);
if (code == CURLE_OK) {
if (CURL.responseCode() == 200) {
std::string contentType = CURL.contentType();
if (isContentType(contentType, "image/x-icon") ||
isContentType(contentType, "application/octet-stream") ||
isContentType(contentType, "text/plain")) {
if (!vicon.empty()) {
long todo; // check it
result = toBase64(vicon, icon);
}
}
}
curl_easy_cleanup(curl);
curl = NULL;
}
return result;
@ -245,68 +250,42 @@ p3FeedReaderThread::DownloadResult p3FeedReaderThread::download(const RsFeedRead
DownloadResult result;
CURL *curl = curl_easy_init();
if (curl) {
std::string proxy;
if (feed.flag & RS_FEED_FLAG_STANDARD_PROXY) {
std::string standardProxyAddress;
uint16_t standardProxyPort;
if (mFeedReader->getStandardProxy(standardProxyAddress, standardProxyPort)) {
rs_sprintf(proxy, "%s:%u", standardProxyAddress.c_str(), standardProxyPort);
}
} else {
if (!feed.proxyAddress.empty() && feed.proxyPort) {
rs_sprintf(proxy, "%s:%u", feed.proxyAddress.c_str(), feed.proxyPort);
}
}
std::string proxy = getProxyForFeed(feed);
CURLWrapper CURL(proxy);
CURLcode code = CURL.downloadText(feed.url, content);
curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback);
curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
curl_easy_setopt(curl, CURLOPT_URL, feed.url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionString);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
if (code == CURLE_OK) {
long responseCode = CURL.responseCode();
if (!proxy.empty()) {
curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str());
}
switch (responseCode) {
case 200:
{
std::string contentType = CURL.contentType();
CURLcode code = curl_easy_perform(curl);
if (code == CURLE_OK) {
long response;
curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
if (response == 200) {
char *contentType = NULL;
curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &contentType);
if (contentType &&
(strncasecmp(contentType, "text/xml", 8) == 0 ||
strncasecmp(contentType, "application/rss+xml", 19) == 0 ||
strncasecmp(contentType, "application/xml", 15) == 0 ||
strncasecmp(contentType, "application/xhtml+xml", 21) == 0)) {
if (isContentType(contentType, "text/xml") ||
isContentType(contentType, "application/rss+xml") ||
isContentType(contentType, "application/xml") ||
isContentType(contentType, "application/xhtml+xml")) {
/* ok */
result = DOWNLOAD_SUCCESS;
} else {
result = DOWNLOAD_UNKNOWN_CONTENT_TYPE;
error = contentType ? contentType : "";
error = contentType;
}
} else if (response == 404) {
result = DOWNLOAD_NOT_FOUND;
} else {
result = DOWNLOAD_UNKOWN_RESPONSE_CODE;
rs_sprintf(error, "%ld", response);
}
} else {
result = DOWNLOAD_ERROR;
error = curl_easy_strerror(code);
break;
case 404:
result = DOWNLOAD_NOT_FOUND;
break;
default:
result = DOWNLOAD_UNKOWN_RESPONSE_CODE;
rs_sprintf(error, "%ld", responseCode);
}
curl_easy_cleanup(curl);
curl = NULL;
getFavicon(feed.url, proxy, icon);
getFavicon(CURL, feed.url, icon);
} else {
result = DOWNLOAD_ERROR_INIT;
result = DOWNLOAD_ERROR;
error = curl_easy_strerror(code);
}
#ifdef FEEDREADER_DEBUG
@ -320,12 +299,12 @@ p3FeedReaderThread::DownloadResult p3FeedReaderThread::download(const RsFeedRead
/****************************** Process ************************************/
/***************************************************************************/
static bool convertOutput(xmlCharEncodingHandlerPtr charEncodingHandler, const xmlChar *output, std::string &text)
static bool convertToString(xmlCharEncodingHandlerPtr charEncodingHandler, const xmlChar *xmlText, std::string &text)
{
bool result = false;
xmlBufferPtr in = xmlBufferCreateStatic((void*) xmlText, xmlStrlen(xmlText));
xmlBufferPtr out = xmlBufferCreate();
xmlBufferPtr in = xmlBufferCreateStatic((void*) output, xmlStrlen(output));
int ret = xmlCharEncOutFunc(charEncodingHandler, out, in);
if (ret >= 0) {
result = true;
@ -338,6 +317,24 @@ static bool convertOutput(xmlCharEncodingHandlerPtr charEncodingHandler, const x
return result;
}
static bool convertFromString(xmlCharEncodingHandlerPtr charEncodingHandler, const char *text, xmlChar *&xmlText)
{
bool result = false;
xmlBufferPtr in = xmlBufferCreateStatic((void*) text, strlen(text));
xmlBufferPtr out = xmlBufferCreate();
int ret = xmlCharEncOutFunc(charEncodingHandler, out, in);
if (ret >= 0) {
result = true;
xmlText = xmlBufferDetach(out);
}
xmlBufferFree(in);
xmlBufferFree(out);
return result;
}
static xmlNodePtr findNode(xmlNodePtr node, const char *name, bool children = false)
{
if (node->name) {
@ -419,12 +416,46 @@ static bool getChildText(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler
}
if (child->children->content) {
return convertOutput((xmlCharEncodingHandlerPtr) charEncodingHandler, child->children->content, text);
return convertToString((xmlCharEncodingHandlerPtr) charEncodingHandler, child->children->content, text);
}
return true;
}
static std::string xmlGetAttr(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler, xmlNodePtr node, const char *name)
{
if (!node || !name) {
return "";
}
std::string value;
xmlChar *xmlValue = xmlGetProp(node, (const xmlChar*) name);
if (xmlValue) {
convertToString((xmlCharEncodingHandlerPtr) charEncodingHandler, xmlValue, value);
xmlFree(xmlValue);
}
return value;
}
static bool xmlSetAttr(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler, xmlNodePtr node, const char *name, const char *value)
{
if (!node || !name) {
return false;
}
xmlChar *xmlValue = NULL;
if (!convertFromString((xmlCharEncodingHandlerPtr) charEncodingHandler, value, xmlValue)) {
return false;
}
xmlAttrPtr xmlAttr = xmlSetProp (node, (const xmlChar*) name, xmlValue);
xmlFree(xmlValue);
return xmlAttr != NULL;
}
static void splitString(std::string s, std::vector<std::string> &v, const char d)
{
v.clear();
@ -689,7 +720,7 @@ static time_t parseRFC822Date(const std::string &pubDate)
// broken mail-/news-clients omit the time zone
if (*dateString) {
if ((strncasecmp(dateString, "gmt", 3) == 0) ||
(strncasecmp(dateString, "utc", 3) == 0))
(strncasecmp(dateString, "utc", 3) == 0))
{
dateString += 3;
while(*dateString && isspace(*dateString))
@ -942,7 +973,10 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
item->feedId = feed.feedId;
item->title = title;
getChildText(mCharEncodingHandler, node, "link", item->link);
/* try feedburner:origLink */
if (!getChildText(mCharEncodingHandler, node, "origLink", item->link) || item->link.empty()) {
getChildText(mCharEncodingHandler, node, "link", item->link);
}
long todo; // remove sid
// // remove sid=
@ -969,6 +1003,7 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
// }
getChildText(mCharEncodingHandler, node, "author", item->author);
getChildText(mCharEncodingHandler, node, "description", item->description);
std::string pubDate;
@ -1007,3 +1042,148 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
return result;
}
std::string p3FeedReaderThread::getProxyForFeed(const RsFeedReaderFeed &feed)
{
std::string proxy;
if (feed.flag & RS_FEED_FLAG_STANDARD_PROXY) {
std::string standardProxyAddress;
uint16_t standardProxyPort;
if (mFeedReader->getStandardProxy(standardProxyAddress, standardProxyPort)) {
rs_sprintf(proxy, "%s:%u", standardProxyAddress.c_str(), standardProxyPort);
}
} else {
if (!feed.proxyAddress.empty() && feed.proxyPort) {
rs_sprintf(proxy, "%s:%u", feed.proxyAddress.c_str(), feed.proxyPort);
}
}
return proxy;
}
bool p3FeedReaderThread::processMsg(const RsFeedReaderFeed &feed, RsFeedReaderMsg *msg)
{
if (!msg) {
return false;
}
std::string proxy = getProxyForFeed(feed);
std::string url;
if (feed.flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE) {
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download page " << msg->link << std::endl;
#endif
std::string content;
CURLWrapper CURL(proxy);
CURLcode code = CURL.downloadText(msg->link, content);
if (code == CURLE_OK && CURL.responseCode() == 200 && isContentType(CURL.contentType(), "text/html")) {
/* ok */
msg->description = content;
} else {
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot download page, CURLCode = " << code << ", responseCode = " << CURL.responseCode() << ", contentType = " << CURL.contentType() << std::endl;
#endif
return false;
}
/* get effective url (moved location) */
std::string effectiveUrl = CURL.effectiveUrl();
url = getBaseLink(effectiveUrl.empty() ? msg->link : effectiveUrl);
}
/* check if string contains xml chars (very simple test) */
if (msg->description.find('<') == std::string::npos) {
return true;
}
bool result = true;
/* process description */
long todo; // encoding
htmlDocPtr document = htmlReadMemory(msg->description.c_str(), msg->description.length(), url.c_str(), "", HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING | HTML_PARSE_COMPACT);
if (document) {
xmlNodePtr root = xmlDocGetRootElement(document);
if (root) {
/* process all children */
std::list<xmlNodePtr> parents;
parents.push_back(root);
while (!parents.empty()) {
if (!isRunning()) {
break;
}
xmlNodePtr node = parents.front();
parents.pop_front();
if (node->type == XML_ELEMENT_NODE) {
/* check for image */
if (strcasecmp((char*) node->name, "img") == 0) {
bool removeImage = true;
if (feed.flag & RS_FEED_FLAG_EMBED_IMAGES) {
/* embed image */
std::string src = xmlGetAttr(mCharEncodingHandler, node, "src");
if (!src.empty()) {
/* download image */
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download image " << src << std::endl;
#endif
std::vector<unsigned char> data;
CURLWrapper CURL(proxy);
CURLcode code = CURL.downloadBinary(calculateLink(url, src), data);
if (code == CURLE_OK && CURL.responseCode() == 200) {
std::string contentType = CURL.contentType();
if (isContentType(contentType, "image/")) {
std::string base64;
if (toBase64(data, base64)) {
std::string imageBase64;
rs_sprintf(imageBase64, "data:%s;base64,%s", contentType.c_str(), base64.c_str());
if (xmlSetAttr(mCharEncodingHandler, node, "src", imageBase64.c_str())) {
removeImage = false;
}
}
}
}
}
}
if (removeImage) {
/* remove image */
xmlUnlinkNode(node);
xmlFreeNode(node);
continue;
}
}
xmlNodePtr child;
for (child = node->children; child; child = child->next) {
parents.push_back(child);
}
}
}
xmlChar *html = NULL;
int htmlSize = 0;
htmlDocDumpMemoryFormat(document, &html, &htmlSize, 0);
if (html) {
convertToString((xmlCharEncodingHandlerPtr) mCharEncodingHandler, html, msg->description);
xmlFree(html);
} else {
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot dump html" << std::endl;
#endif
result = false;
}
} else {
#ifdef FEEDREADER_DEBUG
std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") no root element" << std::endl;
#endif
result = false;
}
xmlFreeDoc(document);
}
return result;
}

6
plugins/FeedReader/services/p3FeedReaderThread.h
View File

@ -19,7 +19,7 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
#ifndef P3_FEEDREADERTHREAD
#ifndef P3_FEEDREADERTHREAD
#define P3_FEEDREADERTHREAD
#include "util/rsthreads.h"
@ -40,7 +40,6 @@ public:
enum DownloadResult
{
DOWNLOAD_SUCCESS,
DOWNLOAD_ERROR_INIT,
DOWNLOAD_ERROR,
DOWNLOAD_UNKNOWN_CONTENT_TYPE,
DOWNLOAD_NOT_FOUND,
@ -64,6 +63,9 @@ private:
DownloadResult download(const RsFeedReaderFeed &feed, std::string &content, std::string &icon, std::string &error);
ProcessResult process(const RsFeedReaderFeed &feed, std::list<RsFeedReaderMsg*> &entries, std::string &error);
std::string getProxyForFeed(const RsFeedReaderFeed &feed);
bool processMsg(const RsFeedReaderFeed &feed, RsFeedReaderMsg *msg);
p3FeedReader *mFeedReader;
Type mType;
/*xmlCharEncodingHandlerPtr*/ void *mCharEncodingHandler;

4
plugins/FeedReader/services/rsFeedReaderItems.h
View File

@ -19,7 +19,7 @@
* Boston, MA 02110-1301, USA.
****************************************************************/
#ifndef RS_FEEDREADER_ITEMS_H
#ifndef RS_FEEDREADER_ITEMS_H
#define RS_FEEDREADER_ITEMS_H
#include "serialiser/rsserial.h"
@ -55,6 +55,8 @@ const uint8_t RS_PKT_SUBTYPE_FEEDREADER_MSG = 0x03;
#define RS_FEED_FLAG_DEACTIVATED 0x040
#define RS_FEED_FLAG_FORUM 0x080
#define RS_FEED_FLAG_UPDATE_FORUM_INFO 0x100
#define RS_FEED_FLAG_EMBED_IMAGES 0x200
#define RS_FEED_FLAG_SAVE_COMPLETE_PAGE 0x400
class RsFeedReaderFeed : public RsItem
{

136
plugins/FeedReader/services/util/CURLWrapper.cc Normal file
View File

@ -0,0 +1,136 @@
/****************************************************************
* RetroShare GUI is distributed under the following license:
*
* Copyright (C) 2012 by Thunder
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
****************************************************************/
#include "CURLWrapper.h"
//static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/)
//{
// p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp;
// if (!thread->isRunning()) {
// /* thread was stopped */
// return 1;
// }
// long todo; // show progress in gui
// return 0;
//}
CURLWrapper::CURLWrapper(const std::string &proxy)
{
mCurl = curl_easy_init();
if (mCurl) {
curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 0);
// curl_easy_setopt(mCurl, CURLOPT_PROGRESSFUNCTION, progressCallback);
// curl_easy_setopt(mCurl, CURLOPT_PROGRESSDATA, feedReader);
curl_easy_setopt(mCurl, CURLOPT_FOLLOWLOCATION, 1);
curl_easy_setopt(mCurl, CURLOPT_CONNECTTIMEOUT, 60);
curl_easy_setopt(mCurl, CURLOPT_TIMEOUT, 120);
if (!proxy.empty()) {
curl_easy_setopt(mCurl, CURLOPT_PROXY, proxy.c_str());
}
}
}
CURLWrapper::~CURLWrapper()
{
if (mCurl) {
curl_easy_cleanup(mCurl);
}
}
static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream)
{
std::string *s = (std::string*) stream;
s->append ((char*) ptr, size * nmemb);
return nmemb * size;
}
CURLcode CURLWrapper::downloadText(const std::string &link, std::string &data)
{
data.clear();
if (!mCurl) {
return CURLE_FAILED_INIT;
}
curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str());
curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionString);
curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data);
return curl_easy_perform(mCurl);
}
static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream)
{
std::vector<unsigned char> *bytes = (std::vector<unsigned char>*) stream;
std::vector<unsigned char> newBytes;
newBytes.resize(size * nmemb);
memcpy(newBytes.data(), ptr, newBytes.size());
bytes->insert(bytes->end(), newBytes.begin(), newBytes.end());
return nmemb * size;
}
CURLcode CURLWrapper::downloadBinary(const std::string &link, std::vector<unsigned char> &data)
{
data.clear();
if (!mCurl) {
return CURLE_FAILED_INIT;
}
curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 1);
curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str());
curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionBinary);
curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data);
return curl_easy_perform(mCurl);
}
long CURLWrapper::longInfo(CURLINFO info)
{
if (!mCurl) {
return 0;
}
long value;
curl_easy_getinfo(mCurl, info, &value);
return value;
}
std::string CURLWrapper::stringInfo(CURLINFO info)
{
if (!mCurl) {
return "";
}
char *value;
curl_easy_getinfo(mCurl, info, &value);
return value ? value : "";
}

50
plugins/FeedReader/services/util/CURLWrapper.h Normal file
View File

@ -0,0 +1,50 @@
/****************************************************************
* RetroShare GUI is distributed under the following license:
*
* Copyright (C) 2012 by Thunder
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License
* as published by the Free Software Foundation; either version 2
* of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor,
* Boston, MA 02110-1301, USA.
****************************************************************/
#ifndef CURLWRAPPER
#define CURLWRAPPER
#include <string>
#include <vector>
#include <curl/curl.h>
class CURLWrapper
{
public:
CURLWrapper(const std::string &proxy);
~CURLWrapper();
CURLcode downloadText(const std::string &link, std::string &data);
CURLcode downloadBinary(const std::string &link, std::vector<unsigned char> &data);
long responseCode() { return longInfo(CURLINFO_RESPONSE_CODE); }
std::string contentType() { return stringInfo(CURLINFO_CONTENT_TYPE); }
std::string effectiveUrl() { return stringInfo(CURLINFO_EFFECTIVE_URL); }
protected:
long longInfo(CURLINFO info);
std::string stringInfo(CURLINFO info);
private:
CURL *mCurl;
};
#endif