Added new button to open the link of the message in browser or copy the link of the message.

Added "RSS: " for the forum feeds.
Parse the feedburner:origLink in the rss feed.
Moved download functions to a new class CURLWrapper for easy use.
Added two new functions (currently only for local feeds for testing):
- embed images into the message (works for Qt 4.7 and higher)
- save complete web page

git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs-b1@5399 b45a01b8-16f6-495d-af2f-9b41ad6348cc

plugins/FeedReader/FeedReader.pro

@@ -6,6 +6,7 @@ SOURCES =	FeedReaderPlugin.cpp \
 			services/p3FeedReader.cc \
 			services/p3FeedReaderThread.cc \
 			services/rsFeedReaderItems.cc \
+			services/util/CURLWrapper.cc \
 			gui/FeedReaderDialog.cpp \
 			gui/AddFeedDialog.cpp \
 			gui/FeedReaderNotify.cpp \
@@ -16,6 +17,7 @@ HEADERS =	FeedReaderPlugin.h \
 			services/p3FeedReader.h \
 			services/p3FeedReaderThread.h \
 			services/rsFeedReaderItems.h \
+			services/util/CURLWrapper.h \
 			gui/FeedReaderDialog.h \
 			gui/AddFeedDialog.h \
 			gui/FeedReaderNotify.h \
@@ -39,7 +41,7 @@ linux-* {
 }
 
 win32 {
-	DEFINES += CURL_STATICLIB
+	DEFINES += CURL_STATICLIB LIBXML_STATIC
 
 	CURL_DIR = ../../../curl-7.26.0
 	LIBXML2_DIR = ../../../libxml2-2.8.0

plugins/FeedReader/gui/AddFeedDialog.cpp

@@ -46,12 +46,17 @@ AddFeedDialog::AddFeedDialog(RsFeedReader *feedReader, QWidget *parent)
 	connect(ui->useStandardProxyCheckBox, SIGNAL(toggled(bool)), this, SLOT(useStandardProxyToggled()));
 	connect(ui->typeForumRadio, SIGNAL(toggled(bool)), this, SLOT(typeForumToggled()));
 
+	/* currently only for loacl feeds */
+	connect(ui->saveCompletePageCheckBox, SIGNAL(toggled(bool)), this, SLOT(denyForumToggled()));
+	connect(ui->embedImagesCheckBox, SIGNAL(toggled(bool)), this, SLOT(denyForumToggled()));
+
 	connect(ui->urlLineEdit, SIGNAL(textChanged(QString)), this, SLOT(validate()));
 	connect(ui->nameLineEdit, SIGNAL(textChanged(QString)), this, SLOT(validate()));
 	connect(ui->useInfoFromFeedCheckBox, SIGNAL(toggled(bool)), this, SLOT(validate()));
+	connect(ui->typeLocalRadio, SIGNAL(toggled(bool)), this, SLOT(validate()));
+	connect(ui->typeForumRadio, SIGNAL(toggled(bool)), this, SLOT(validate()));
 
 	ui->activatedCheckBox->setChecked(true);
-	ui->typeLocalRadio->setChecked(true);
 	ui->forumComboBox->setEnabled(false);
 	ui->useInfoFromFeedCheckBox->setChecked(true);
 	ui->updateForumInfoCheckBox->setEnabled(false);
@@ -124,6 +129,16 @@ void AddFeedDialog::typeForumToggled()
 	ui->updateForumInfoCheckBox->setEnabled(checked);
 }
 
+void AddFeedDialog::denyForumToggled()
+{
+	if (ui->saveCompletePageCheckBox->isChecked() || ui->embedImagesCheckBox->isChecked()) {
+		ui->typeForumRadio->setEnabled(false);
+		ui->typeLocalRadio->setChecked(true);
+	} else {
+		ui->typeForumRadio->setEnabled(true);
+	}
+}
+
 void AddFeedDialog::validate()
 {
 	bool ok = true;
@@ -134,6 +149,9 @@ void AddFeedDialog::validate()
 	if (ui->nameLineEdit->text().isEmpty() && !ui->useInfoFromFeedCheckBox->isChecked()) {
 		ok = false;
 	}
+	if (!ui->typeLocalRadio->isChecked() && !ui->typeForumRadio->isChecked()) {
+		ok = false;
+	}
 
 	ui->buttonBox->button(QDialogButtonBox::Ok)->setEnabled(ok);
 }
@@ -196,6 +214,8 @@ bool AddFeedDialog::fillFeed(const std::string &feedId)
 		ui->useInfoFromFeedCheckBox->setChecked(feedInfo.flag.infoFromFeed);
 		ui->updateForumInfoCheckBox->setChecked(feedInfo.flag.updateForumInfo);
 		ui->activatedCheckBox->setChecked(!feedInfo.flag.deactivated);
+		ui->embedImagesCheckBox->setChecked(feedInfo.flag.embedImages);
+		ui->saveCompletePageCheckBox->setChecked(feedInfo.flag.saveCompletePage);
 
 		ui->descriptionPlainTextEdit->setPlainText(QString::fromUtf8(feedInfo.description.c_str()));
 
@@ -206,6 +226,8 @@ bool AddFeedDialog::fillFeed(const std::string &feedId)
 
 		if (feedInfo.flag.forum) {
 			ui->typeForumRadio->setChecked(true);
+			ui->saveCompletePageCheckBox->setEnabled(false);
+			ui->embedImagesCheckBox->setEnabled(false);
 
 			if (feedInfo.forumId.empty()) {
 				ui->forumNameLabel->setText(tr("Not yet created"));
@@ -259,13 +281,17 @@ void AddFeedDialog::createFeed()
 	feedInfo.flag.infoFromFeed = ui->useInfoFromFeedCheckBox->isChecked();
 	feedInfo.flag.updateForumInfo = ui->updateForumInfoCheckBox->isChecked() && ui->updateForumInfoCheckBox->isEnabled();
 	feedInfo.flag.deactivated = !ui->activatedCheckBox->isChecked();
+	feedInfo.flag.embedImages = ui->embedImagesCheckBox->isChecked();
+	feedInfo.flag.saveCompletePage = ui->saveCompletePageCheckBox->isChecked();
 
 	feedInfo.description = ui->descriptionPlainTextEdit->toPlainText().toUtf8().constData();
 
 	feedInfo.flag.forum = ui->typeForumRadio->isChecked();
 	if (mFeedId.empty()) {
-		/* set forum (only when create a new feed) */
+		if (feedInfo.flag.forum) {
-		feedInfo.forumId = ui->forumComboBox->itemData(ui->forumComboBox->currentIndex()).toString().toStdString();
+			/* set forum (only when create a new feed) */
+			feedInfo.forumId = ui->forumComboBox->itemData(ui->forumComboBox->currentIndex()).toString().toStdString();
+		}
 	}
 
 	feedInfo.flag.authentication = ui->useAuthenticationCheckBox->isChecked();

plugins/FeedReader/gui/AddFeedDialog.h

@@ -50,6 +50,7 @@ private slots:
 	void useStandardUpdateIntervalToggled();
 	void useStandardProxyToggled();
 	void typeForumToggled();
+	void denyForumToggled();
 	void validate();
 	void createFeed();
 

plugins/FeedReader/gui/AddFeedDialog.ui

@@ -7,7 +7,7 @@
     <x>0</x>
     <y>0</y>
     <width>715</width>
-    <height>559</height>
+    <height>605</height>
    </rect>
   </property>
   <property name="windowTitle">
@@ -289,13 +289,6 @@ p, li { white-space: pre-wrap; }
          <property name="margin">
           <number>6</number>
          </property>
-         <item>
-          <widget class="QRadioButton" name="typeLocalRadio">
-           <property name="text">
-            <string>Local Feed</string>
-           </property>
-          </widget>
-         </item>
          <item>
           <layout class="QHBoxLayout" name="horizontalLayout_2">
            <item>
@@ -323,6 +316,13 @@ p, li { white-space: pre-wrap; }
            </item>
           </layout>
          </item>
+         <item>
+          <widget class="QRadioButton" name="typeLocalRadio">
+           <property name="text">
+            <string>Local Feed</string>
+           </property>
+          </widget>
+         </item>
          <item>
           <spacer name="verticalSpacer">
            <property name="orientation">
@@ -366,6 +366,20 @@ p, li { white-space: pre-wrap; }
            </property>
           </widget>
          </item>
+         <item>
+          <widget class="QCheckBox" name="embedImagesCheckBox">
+           <property name="text">
+            <string>Embed images (experimental for local feeds)</string>
+           </property>
+          </widget>
+         </item>
+         <item>
+          <widget class="QCheckBox" name="saveCompletePageCheckBox">
+           <property name="text">
+            <string>Save complete web page (experimental for local feeds)</string>
+           </property>
+          </widget>
+         </item>
         </layout>
        </widget>
       </item>
@@ -416,7 +430,6 @@ p, li { white-space: pre-wrap; }
   <tabstop>urlLineEdit</tabstop>
   <tabstop>nameLineEdit</tabstop>
   <tabstop>descriptionPlainTextEdit</tabstop>
-  <tabstop>typeLocalRadio</tabstop>
   <tabstop>typeForumRadio</tabstop>
   <tabstop>forumComboBox</tabstop>
   <tabstop>activatedCheckBox</tabstop>

plugins/FeedReader/gui/FeedReaderDialog.cpp

@@ -27,6 +27,7 @@
 #include <QPainter>
 #include <QMessageBox>
 #include <QClipboard>
+#include <QDesktopServices>
 
 #include "FeedReaderDialog.h"
 #include "ui_FeedReaderDialog.h"
@@ -118,6 +119,7 @@ FeedReaderDialog::FeedReaderDialog(RsFeedReader *feedReader, QWidget *parent)
 	connect(ui->filterLineEdit, SIGNAL(textChanged(QString)), this, SLOT(filterItems(QString)));
 	connect(ui->filterColumnComboBox, SIGNAL(currentIndexChanged(int)), this, SLOT(filterColumnChanged()));
 
+	connect(ui->linkButton, SIGNAL(clicked()), this, SLOT(openLinkMsg()));
 	connect(ui->expandButton, SIGNAL(clicked()), this, SLOT(toggleMsgText()));
 
 	mFeedCompareRole = new RSTreeWidgetItemCompareRole;
@@ -168,6 +170,18 @@ FeedReaderDialog::FeedReaderDialog(RsFeedReader *feedReader, QWidget *parent)
 	/* initialize feed list */
 	ui->feedTreeWidget->sortItems(COLUMN_FEED_NAME, Qt::AscendingOrder);
 
+	/* build menu for link button */
+	QMenu *menu = new QMenu(this);
+	QAction *action = menu->addAction(tr("Open link in browser"), this, SLOT(openLinkMsg()));
+	menu->addAction(tr("Copy link to clipboard"), this, SLOT(copyLinkMsg()));
+
+	QFont font = action->font();
+	font.setBold(true);
+	action->setFont(font);
+
+	ui->linkButton->setMenu(menu);
+	ui->linkButton->setEnabled(false);
+
 	ui->msgTreeWidget->installEventFilter(this);
 }
 
@@ -351,7 +365,7 @@ void FeedReaderDialog::msgTreeCustomPopupMenu(QPoint /*point*/)
 
 	contextMnu.addSeparator();
 
-	action = contextMnu.addAction(QIcon(""), tr("Copy link"), this, SLOT(copyLinkMsg()));
+	action = contextMnu.addAction(QIcon(""), tr("Copy link"), this, SLOT(copyLinskMsg()));
 	action->setEnabled(!selectedItems.empty());
 
 	action = contextMnu.addAction(QIcon(""), tr("Remove"), this, SLOT(removeMsg()));
@@ -522,10 +536,14 @@ void FeedReaderDialog::updateFeedItem(QTreeWidgetItem *item, FeedInfo &info)
 	case FeedInfo::WAITING:
 		break;
 	case FeedInfo::WAITING_TO_DOWNLOAD:
+		workState = tr("waiting for download");
+		break;
 	case FeedInfo::DOWNLOADING:
-		workState = tr("loading");
+		workState = tr("downloading");
 		break;
 	case FeedInfo::WAITING_TO_PROCESS:
+		workState = tr("waiting for process");
+		break;
 	case FeedInfo::PROCESSING:
 		workState = tr("processing");
 		break;
@@ -791,6 +809,7 @@ void FeedReaderDialog::msgItemChanged()
 		ui->msgTitle->clear();
 //		ui->msgLink->clear();
 		ui->msgText->clear();
+		ui->linkButton->setEnabled(false);
 		return;
 	}
 
@@ -800,6 +819,7 @@ void FeedReaderDialog::msgItemChanged()
 		ui->msgTitle->clear();
 //		ui->msgLink->clear();
 		ui->msgText->clear();
+		ui->linkButton->setEnabled(false);
 		return;
 	}
 
@@ -809,6 +829,7 @@ void FeedReaderDialog::msgItemChanged()
 		ui->msgTitle->clear();
 //		ui->msgLink->clear();
 		ui->msgText->clear();
+		ui->linkButton->setEnabled(false);
 		return;
 	}
 
@@ -832,7 +853,8 @@ void FeedReaderDialog::msgItemChanged()
 
 	ui->msgText->setHtml(msgTxt);
 	ui->msgTitle->setText(QString::fromUtf8(msgInfo.title.c_str()));
-//	ui->msgLink->setHtml(RsHtml().formatText(NULL, QString::fromUtf8(msgInfo.link.c_str()), RSHTML_FORMATTEXT_EMBED_LINKS));
+
+	ui->linkButton->setEnabled(!msgInfo.link.empty());
 }
 
 void FeedReaderDialog::setMsgAsReadUnread(QList<QTreeWidgetItem *> &rows, bool read)
@@ -1049,7 +1071,7 @@ void FeedReaderDialog::markAllAsReadMsg()
 	setMsgAsReadUnread(items, true);
 }
 
-void FeedReaderDialog::copyLinkMsg()
+void FeedReaderDialog::copyLinksMsg()
 {
 	QString links;
 
@@ -1086,3 +1108,33 @@ void FeedReaderDialog::removeMsg()
 	}
 	mFeedReader->removeMsgs(feedId, msgIds);
 }
+
+void FeedReaderDialog::copyLinkMsg()
+{
+	QTreeWidgetItem *item = ui->msgTreeWidget->currentItem();
+	if (!item) {
+		return;
+	}
+
+	QString link = item->data(COLUMN_MSG_DATA, ROLE_MSG_LINK).toString();
+	if (link.isEmpty()) {
+		return;
+	}
+
+	QApplication::clipboard()->setText(link);
+}
+
+void FeedReaderDialog::openLinkMsg()
+{
+	QTreeWidgetItem *item = ui->msgTreeWidget->currentItem();
+	if (!item) {
+		return;
+	}
+
+	QString link = item->data(COLUMN_MSG_DATA, ROLE_MSG_LINK).toString();
+	if (link.isEmpty()) {
+		return;
+	}
+
+	QDesktopServices::openUrl(QUrl(link));
+}

plugins/FeedReader/gui/FeedReaderDialog.h

@@ -64,8 +64,10 @@ private slots:
 	void markAsReadMsg();
 	void markAsUnreadMsg();
 	void markAllAsReadMsg();
-	void copyLinkMsg();
+	void copyLinksMsg();
 	void removeMsg();
+	void openLinkMsg();
+	void copyLinkMsg();
 
 	/* FeedReaderNotify */
 	void feedChanged(const QString &feedId, int type);

plugins/FeedReader/gui/FeedReaderDialog.ui

@@ -268,8 +268,8 @@ border: 1px solid #CCCCCC;}</string>
          </widget>
         </item>
         <item row="4" column="0">
-         <layout class="QGridLayout" name="navFrame">
+         <layout class="QHBoxLayout" name="horizontalLayout_3">
-          <item row="0" column="0">
+          <item>
            <widget class="QLabel" name="msgLabel">
             <property name="sizePolicy">
              <sizepolicy hsizetype="Fixed" vsizetype="Preferred">
@@ -289,7 +289,7 @@ border: 1px solid #CCCCCC;}</string>
             </property>
            </widget>
           </item>
-          <item row="0" column="1">
+          <item>
            <widget class="QLabel" name="msgTitle">
             <property name="styleSheet">
              <string notr="true">QLabel#msgTitle{
@@ -305,7 +305,21 @@ background: white;}</string>
             </property>
            </widget>
           </item>
-          <item row="0" column="2">
+          <item>
+           <widget class="QToolButton" name="linkButton">
+            <property name="icon">
+             <iconset resource="FeedReader_images.qrc">
+              <normaloff>:/images/Link.png</normaloff>:/images/Link.png</iconset>
+            </property>
+            <property name="popupMode">
+             <enum>QToolButton::MenuButtonPopup</enum>
+            </property>
+            <property name="autoRaise">
+             <bool>true</bool>
+            </property>
+           </widget>
+          </item>
+          <item>
            <widget class="QPushButton" name="expandButton">
             <property name="maximumSize">
              <size>

plugins/FeedReader/gui/FeedReader_images.qrc

@@ -8,6 +8,7 @@
 		<file>images/FeedErrorOverlay.png</file>
 		<file>images/FolderAdd.png</file>
 		<file>images/FeedAdd.png</file>
+		<file>images/Link.png</file>
 		<file>images/Update.png</file>
 	</qresource>
 </RCC>

plugins/FeedReader/interface/rsFeedReader.h

@@ -19,7 +19,7 @@
  *  Boston, MA  02110-1301, USA.
  ****************************************************************/
 
- #ifndef RETROSHARE_FEEDREADER_GUI_INTERFACE_H
+#ifndef RETROSHARE_FEEDREADER_GUI_INTERFACE_H
 #define RETROSHARE_FEEDREADER_GUI_INTERFACE_H
 
 #include <inttypes.h>
@@ -68,6 +68,8 @@ public:
 		flag.deactivated = false;
 		flag.forum = false;
 		flag.updateForumInfo = false;
+		flag.embedImages = false;
+		flag.saveCompletePage = false;
 	}
 
 	std::string  feedId;
@@ -98,6 +100,8 @@ public:
 		bool deactivated : 1;
 		bool forum : 1;
 		bool updateForumInfo : 1;
+		bool embedImages : 1;
+		bool saveCompletePage : 1;
 	} flag;
 };
 

plugins/FeedReader/services/p3FeedReader.cc

@@ -30,6 +30,7 @@
 RsFeedReader *rsFeedReader = NULL;
 
 #define FEEDREADER_CLEAN_INTERVAL 1 * 60 * 60 // check every hour
+#define FEEDREADER_FORUM_PREFIX  L"RSS: "
 
 /*********
  * #define FEEDREADER_DEBUG
@@ -91,6 +92,8 @@ static void feedToInfo(const RsFeedReaderFeed *feed, FeedInfo &info)
 	info.flag.deactivated = (feed->flag & RS_FEED_FLAG_DEACTIVATED);
 	info.flag.forum = (feed->flag & RS_FEED_FLAG_FORUM);
 	info.flag.updateForumInfo = (feed->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO);
+	info.flag.embedImages = (feed->flag & RS_FEED_FLAG_EMBED_IMAGES);
+	info.flag.saveCompletePage = (feed->flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE);
 
 	switch (feed->workstate) {
 	case RsFeedReaderFeed::WAITING:
@@ -151,6 +154,12 @@ static void infoToFeed(const FeedInfo &info, RsFeedReaderFeed *feed, bool add)
 	if (info.flag.deactivated) {
 		feed->flag |= RS_FEED_FLAG_DEACTIVATED;
 	}
+	if (info.flag.embedImages) {
+		feed->flag |= RS_FEED_FLAG_EMBED_IMAGES;
+	}
+	if (info.flag.saveCompletePage) {
+		feed->flag |= RS_FEED_FLAG_SAVE_COMPLETE_PAGE;
+	}
 	if (add) {
 		/* only set when adding a new feed */
 		if (info.flag.folder) {
@@ -950,6 +959,11 @@ int p3FeedReader::tick()
 			} else {
 				updateInterval = fi->updateInterval;
 			}
+
+			if (updateInterval == 0) {
+				continue;
+			}
+
 			if (fi->lastUpdate == 0 || fi->lastUpdate + (long) updateInterval <= currentTime) {
 				/* add to download list */
 				feedToDownload.push_back(fi->feedId);
@@ -1340,7 +1354,6 @@ void p3FeedReader::onDownloadError(const std::string &feedId, p3FeedReaderThread
 		case p3FeedReaderThread::DOWNLOAD_ERROR:
 			fi->errorState = RS_FEED_ERRORSTATE_DOWNLOAD_ERROR;
 			break;
-		case p3FeedReaderThread::DOWNLOAD_ERROR_INIT:
 		case p3FeedReaderThread::DOWNLOAD_INTERNAL_ERROR:
 			fi->errorState = RS_FEED_ERRORSTATE_DOWNLOAD_INTERNAL_ERROR;
 			break;
@@ -1428,15 +1441,13 @@ bool p3FeedReader::getFeedToProcess(RsFeedReaderFeed &feed)
 	return true;
 }
 
-void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedReaderMsg*> &msgs)
+bool p3FeedReader::onProcessSuccess_filterMsg(const std::string &feedId, std::list<RsFeedReaderMsg*> &msgs)
 {
 #ifdef FEEDREADER_DEBUG
-	std::cerr << "p3FeedReader::onProcessSuccess - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
+	std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
 #endif
 
-	std::list<std::string> addedMsgs;
+	bool result = true;
-	std::string forumId;
-	std::list<RsFeedReaderMsg> forumMsgs;
 
 	{
 		RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
@@ -1446,9 +1457,9 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
 		if (it == mFeeds.end()) {
 			/* feed not found */
 #ifdef FEEDREADER_DEBUG
-			std::cerr << "p3FeedReader::onProcessSuccess - feed " << feedId << " not found" << std::endl;
+			std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - feed " << feedId << " not found" << std::endl;
 #endif
-			return;
+			return false;
 		}
 
 		RsFeedReaderFeed *fi = it->second;
@@ -1458,6 +1469,10 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
 		if (forum) {
 			if (fi->forumId.empty()) {
 				/* create new forum */
+				std::wstring forumName;
+				librs::util::ConvertUtf8ToUtf16(fi->name, forumName);
+				forumName.insert(0, FEEDREADER_FORUM_PREFIX);
+
 				long todo; // search for existing forum?
 
 				/* search for existing own forum */
@@ -1472,22 +1487,18 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
 //					}
 //				}
 
-				std::wstring forumName;
-				librs::util::ConvertUtf8ToUtf16(fi->name, forumName);
 				std::wstring forumDescription;
 				librs::util::ConvertUtf8ToUtf16(fi->description, forumDescription);
 				/* create anonymous public forum */
 				fi->forumId = rsForums->createForum(forumName, forumDescription, RS_DISTRIB_PUBLIC | RS_DISTRIB_AUTHEN_ANON);
 
-				forumId = fi->forumId;
-
 				if (fi->forumId.empty()) {
 					errorState = RS_FEED_ERRORSTATE_FORUM_CREATE;
 
 #ifdef FEEDREADER_DEBUG
-					std::cerr << "p3FeedReader::onProcessSuccess - can't create forum for feed " << feedId << " (" << it->second->name << ") - ignore all messages" << std::endl;
+					std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - can't create forum for feed " << feedId << " (" << it->second->name << ") - ignore all messages" << std::endl;
 				} else {
-					std::cerr << "p3FeedReader::onProcessSuccess - forum " << forumId << " (" << fi->name << ") created" << std::endl;
+					std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - forum " << forumId << " (" << fi->name << ") created" << std::endl;
 #endif
 				}
 			} else {
@@ -1498,8 +1509,6 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
 						errorState = RS_FEED_ERRORSTATE_FORUM_NO_ADMIN;
 					} else if ((forumInfo.forumFlags & RS_DISTRIB_AUTHEN_REQ) || (forumInfo.forumFlags & RS_DISTRIB_AUTHEN_ANON) == 0) {
 						errorState = RS_FEED_ERRORSTATE_FORUM_NO_ANONYMOUS_FORUM;
-					} else {
-						forumId = fi->forumId;
 					}
 				} else {
 					errorState = RS_FEED_ERRORSTATE_FORUM_NOT_FOUND;
@@ -1508,15 +1517,11 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
 		}
 
 		/* process msgs */
-#ifdef FEEDREADER_DEBUG
-		uint32_t newMsgs = 0;
-#endif
-
 		if (errorState == RS_FEED_ERRORSTATE_OK) {
 			std::list<RsFeedReaderMsg*>::iterator newMsgIt;
 			for (newMsgIt = msgs.begin(); newMsgIt != msgs.end(); ) {
 				RsFeedReaderMsg *miNew = *newMsgIt;
-				/* search for exisiting msg */
+				/* search for existing msg */
 				std::map<std::string, RsFeedReaderMsg*>::iterator msgIt;
 				for (msgIt = fi->mMsgs.begin(); msgIt != fi->mMsgs.end(); ++msgIt) {
 					RsFeedReaderMsg *mi = msgIt->second;
@@ -1525,38 +1530,103 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
 						break;
 					}
 				}
-				if (msgIt == fi->mMsgs.end()) {
+				if (msgIt != fi->mMsgs.end()) {
-					/* add new msg */
+					/* msg exists */
-					rs_sprintf(miNew->msgId, "%lu", mNextMsgId++);
+					delete(*newMsgIt);
-					if (forum) {
-						miNew->flag = RS_FEEDMSG_FLAG_DELETED;
-						forumMsgs.push_back(*miNew);
-//						miNew->description.clear();
-					} else {
-						miNew->flag = RS_FEEDMSG_FLAG_NEW;
-						addedMsgs.push_back(miNew->msgId);
-					}
-					fi->mMsgs[miNew->msgId] = miNew;
 					newMsgIt = msgs.erase(newMsgIt);
-
-#ifdef FEEDREADER_DEBUG
-					++newMsgs;
-#endif
 				} else {
-					/* msg was updated */
 					++newMsgIt;
 				}
+			}
+		} else {
+			result = false;
+		}
+
+		fi->content.clear();
+		fi->errorState = errorState;
+		fi->errorString.clear();
+
+		IndicateConfigChanged();
+	}
+
+	if (mNotify) {
+		mNotify->feedChanged(feedId, NOTIFY_TYPE_MOD);
+	}
+
+	return result;
+}
+
+void p3FeedReader::onProcessSuccess_addMsgs(const std::string &feedId, bool result, std::list<RsFeedReaderMsg*> &msgs)
+{
 #ifdef FEEDREADER_DEBUG
-				std::cerr << "p3FeedReader::onProcessSuccess - feed " << fi->feedId << " (" << fi->name << ") added " << newMsgs << "/" << msgs.size() << " messages" << std::endl;
+	std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << feedId << " got " << msgs.size() << " messages" << std::endl;
+#endif
+
+	std::list<std::string> addedMsgs;
+	std::string forumId;
+	std::list<RsFeedReaderMsg> forumMsgs;
+
+	{
+		RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
+
+		/* find feed */
+		std::map<std::string, RsFeedReaderFeed*>::iterator it = mFeeds.find(feedId);
+		if (it == mFeeds.end()) {
+			/* feed not found */
+#ifdef FEEDREADER_DEBUG
+			std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << feedId << " not found" << std::endl;
+#endif
+			return;
+		}
+
+		RsFeedReaderFeed *fi = it->second;
+		bool forum = (fi->flag & RS_FEED_FLAG_FORUM);
+
+		if (forum) {
+			forumId = fi->forumId;
+			if (forumId.empty()) {
+				/* don't process messages without forum id */
+				result = false;
+#ifdef FEEDREADER_DEBUG
+				std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << fi->feedId << " (" << fi->name << ") don't process messages without forum id" << std::endl;
 #endif
 			}
 		}
 
+		/* process msgs */
+#ifdef FEEDREADER_DEBUG
+		uint32_t newMsgs = 0;
+#endif
+
+		if (result) {
+			std::list<RsFeedReaderMsg*>::iterator newMsgIt;
+			for (newMsgIt = msgs.begin(); newMsgIt != msgs.end(); ) {
+				RsFeedReaderMsg *miNew = *newMsgIt;
+				/* add new msg */
+				rs_sprintf(miNew->msgId, "%lu", mNextMsgId++);
+				if (forum) {
+					miNew->flag = RS_FEEDMSG_FLAG_DELETED;
+					forumMsgs.push_back(*miNew);
+//					miNew->description.clear();
+				} else {
+					miNew->flag = RS_FEEDMSG_FLAG_NEW;
+					addedMsgs.push_back(miNew->msgId);
+				}
+				fi->mMsgs[miNew->msgId] = miNew;
+				newMsgIt = msgs.erase(newMsgIt);
+
+#ifdef FEEDREADER_DEBUG
+				++newMsgs;
+#endif
+			}
+#ifdef FEEDREADER_DEBUG
+			std::cerr << "p3FeedReader::onProcessSuccess_addMsgs - feed " << fi->feedId << " (" << fi->name << ") added " << newMsgs << "/" << msgs.size() << " messages" << std::endl;
+#endif
+		}
+
 		fi->workstate = RsFeedReaderFeed::WAITING;
 		fi->content.clear();
 		fi->lastUpdate = time(NULL);
-		fi->errorState = errorState;
-		fi->errorString.clear();
 
 		IndicateConfigChanged();
 	}
@@ -1584,7 +1654,7 @@ void p3FeedReader::onProcessSuccess(const std::string &feedId, std::list<RsFeedR
 				rsForums->setMessageStatus(forumMsgInfo.forumId, forumMsgInfo.msgId, 0, FORUM_MSG_STATUS_MASK);
 			} else {
 #ifdef FEEDREADER_DEBUG
-				std::cerr << "p3FeedReader::onProcessSuccess - can't add forum message " << mi.title << " for feed " << forumId << std::endl;
+				std::cerr << "p3FeedReader::onProcessSuccess_filterMsg - can't add forum message " << mi.title << " for feed " << forumId << std::endl;
 #endif
 			}
 		}
@@ -1652,7 +1722,7 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
 {
 	bool changed = false;
 	std::string forumId;
-	ForumInfo forumInfo;
+	ForumInfo forumInfoNew;
 
 	{
 		RsStackMutex stack(mFeedReaderMtx); /******* LOCK STACK MUTEX *********/
@@ -1683,11 +1753,12 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
 			changed = true;
 		}
 
-		if (changed && (fi->flag & RS_FEED_FLAG_FORUM) && (fi->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO) && !fi->forumId.empty()) {
+		if ((fi->flag & RS_FEED_FLAG_FORUM) && (fi->flag & RS_FEED_FLAG_UPDATE_FORUM_INFO) && !fi->forumId.empty()) {
 			/* change forum too */
 			forumId = fi->forumId;
-			librs::util::ConvertUtf8ToUtf16(fi->name, forumInfo.forumName);
+			librs::util::ConvertUtf8ToUtf16(fi->name, forumInfoNew.forumName);
-			librs::util::ConvertUtf8ToUtf16(fi->description, forumInfo.forumDesc);
+			librs::util::ConvertUtf8ToUtf16(fi->description, forumInfoNew.forumDesc);
+			forumInfoNew.forumName.insert(0, FEEDREADER_FORUM_PREFIX);
 		}
 	}
 
@@ -1700,10 +1771,22 @@ void p3FeedReader::setFeedInfo(const std::string &feedId, const std::string &nam
 	}
 
 	if (!forumId.empty()) {
-		/* name or description changed, update forum */
+		ForumInfo forumInfo;
-		if (!rsForums->setForumInfo(forumId, forumInfo)) {
+		if (rsForums->getForumInfo(forumId, forumInfo)) {
+			if (forumInfo.forumName != forumInfoNew.forumName || forumInfo.forumDesc != forumInfoNew.forumDesc) {
+				/* name or description changed, update forum */
 #ifdef FEEDREADER_DEBUG
-			std::cerr << "p3FeedReader::setFeed - can't change forum " << forumId << std::endl;
+				std::cerr << "p3FeedReader::setFeed - change forum " << forumId << std::endl;
+#endif
+				if (!rsForums->setForumInfo(forumId, forumInfoNew)) {
+#ifdef FEEDREADER_DEBUG
+					std::cerr << "p3FeedReader::setFeed - can't change forum " << forumId << std::endl;
+#endif
+				}
+			}
+		} else {
+#ifdef FEEDREADER_DEBUG
+			std::cerr << "p3FeedReader::setFeed - can't get forum info " << forumId << std::endl;
 #endif
 		}
 	}

plugins/FeedReader/services/p3FeedReader.h

@@ -19,7 +19,7 @@
  *  Boston, MA  02110-1301, USA.
  ****************************************************************/
 
- #ifndef P3_FEEDREADER
+#ifndef P3_FEEDREADER
 #define P3_FEEDREADER
 
 #include "retroshare/rsplugin.h"
@@ -71,7 +71,8 @@ public:
 	bool getFeedToDownload(RsFeedReaderFeed &feed);
 	void onDownloadSuccess(const std::string &feedId, const std::string &content, std::string &icon);
 	void onDownloadError(const std::string &feedId, p3FeedReaderThread::DownloadResult result, const std::string &error);
-	void onProcessSuccess(const std::string &feedId, std::list<RsFeedReaderMsg*> &msgs);
+	bool onProcessSuccess_filterMsg(const std::string &feedId, std::list<RsFeedReaderMsg*> &msgs);
+	void onProcessSuccess_addMsgs(const std::string &feedId, bool result, std::list<RsFeedReaderMsg*> &msgs);
 	void onProcessError(const std::string &feedId, p3FeedReaderThread::ProcessResult result);
 
 	bool getFeedToProcess(RsFeedReaderFeed &feed);

plugins/FeedReader/services/p3FeedReaderThread.cc

@@ -22,9 +22,11 @@
 #include "p3FeedReaderThread.h"
 #include "rsFeedReaderItems.h"
 #include "util/rsstring.h"
+#include "util/CURLWrapper.h"
 
-#include <curl/curl.h>
 #include <libxml/parser.h>
+#include <libxml/HTMLparser.h>
+#include <libxml/HTMLtree.h>
 #include <openssl/evp.h>
 
 enum FeedFormat { FORMAT_RSS, FORMAT_RDF };
@@ -32,6 +34,7 @@ enum FeedFormat { FORMAT_RSS, FORMAT_RDF };
 /*********
  * #define FEEDREADER_DEBUG
  *********/
+#define FEEDREADER_DEBUG
 
 p3FeedReaderThread::p3FeedReaderThread(p3FeedReader *feedReader, Type type) : RsThread(), mFeedReader(feedReader), mType(type)
 {
@@ -88,21 +91,34 @@ void p3FeedReaderThread::run()
 			{
 				RsFeedReaderFeed feed;
 				if (mFeedReader->getFeedToProcess(feed)) {
-					std::list<RsFeedReaderMsg*> entries;
+					std::list<RsFeedReaderMsg*> msgs;
 					std::string error;
+					std::list<RsFeedReaderMsg*>::iterator it;
 
-					ProcessResult result = process(feed, entries, error);
+					ProcessResult result = process(feed, msgs, error);
 					if (result == PROCESS_SUCCESS) {
-						mFeedReader->onProcessSuccess(feed.feedId, entries);
+						/* first, filter the messages */
+						bool result = mFeedReader->onProcessSuccess_filterMsg(feed.feedId, msgs);
+						if (isRunning()) {
+							if (result) {
+								long todo; // process new items
+								/* second, process the descriptions */
+								for (it = msgs.begin(); it != msgs.end(); ++it) {
+									RsFeedReaderMsg *mi = *it;
+									processMsg(feed, mi);
+								}
+							}
+							/* third, add messages */
+							mFeedReader->onProcessSuccess_addMsgs(feed.feedId, result, msgs);
+						}
 					} else {
 						mFeedReader->onProcessError(feed.feedId, result);
 					}
 
-					std::list<RsFeedReaderMsg*>::iterator it;
+					for (it = msgs.begin(); it != msgs.end(); ++it) {
-					for (it = entries.begin(); it != entries.end(); ++it) {
 						delete (*it);
 					}
-					entries.clear();
+					msgs.clear();
 				}
 			}
 			break;
@@ -114,121 +130,110 @@ void p3FeedReaderThread::run()
 /****************************** Download ***********************************/
 /***************************************************************************/
 
-static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream)
+static bool isContentType(const std::string &contentType, const char *type)
 {
-	std::string *s = (std::string*) stream;
+	return (strncasecmp(contentType.c_str(), type, strlen(type)) == 0);
-	s->append ((char*) ptr, size * nmemb);
-
-	return nmemb * size;
 }
 
-static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream)
+static bool toBase64(const std::vector<unsigned char> &data, std::string &base64)
 {
-	std::vector<unsigned char> *bytes = (std::vector<unsigned char>*) stream;
+	bool result = false;
 
-	std::vector<unsigned char> newBytes;
+	/* Set up a base64 encoding BIO that writes to a memory BIO */
-	newBytes.resize(size * nmemb);
+	BIO *b64 = BIO_new(BIO_f_base64());
-	memcpy(newBytes.data(), ptr, newBytes.size());
+	if (b64) {
-
+		BIO_set_flags(b64, BIO_FLAGS_BASE64_NO_NL);
-	bytes->insert(bytes->end(), newBytes.begin(), newBytes.end());
+		BIO *bmem = BIO_new(BIO_s_mem());
-
+		if (bmem) {
-	return nmemb * size;
+			BIO_set_flags(bmem, BIO_CLOSE);  // probably redundant
-}
+			b64 = BIO_push(b64, bmem);
-
+			/* Send the data */
-static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/)
+			BIO_write(b64, data.data(), data.size());
-{
+			/* Collect the encoded data */
-	p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp;
+			BIO_flush(b64);
-
+			char* temp;
-	if (!thread->isRunning()) {
+			int count = BIO_get_mem_data(bmem, &temp);
-		/* thread was stopped */
+			if (count && temp) {
-		return 1;
+				base64.assign(temp, count);
+				result = true;
+			}
+		}
+		BIO_free_all(b64);
 	}
 
-	long todo; // show progress in gui
+	return result;
-
-	return 0;
 }
 
-static bool getFavicon (std::string url, const std::string &proxy, std::string &icon)
+static std::string getBaseLink(std::string link)
+{
+	size_t found = link.rfind('/');
+	if (found != std::string::npos) {
+		link.erase(found + 1);
+	}
+
+	return link;
+}
+
+static std::string calculateLink(const std::string &baseLink, const std::string &link)
+{
+	if (link.substr(0, 7) == "http://") {
+		/* absolute link */
+		return link;
+	}
+
+	/* calculate link of base link */
+	std::string resultLink = baseLink;
+
+	/* link should begin with "http://" */
+	if (resultLink.substr(0, 7) != "http://") {
+		resultLink.insert(0, "http://");
+	}
+
+	if (link.empty()) {
+		/* no link */
+		return resultLink;
+	}
+
+	if (*link.begin() == '/') {
+		/* link begins with "/" */
+		size_t found = resultLink.find('/', 7);
+		if (found != std::string::npos) {
+			resultLink.erase(found);
+		}
+	} else {
+		/* check for "/" at the end */
+		std::string::reverse_iterator it = resultLink.rend ();
+		it--;
+		if (*it != '/') {
+			resultLink += "/";
+		}
+	}
+
+	resultLink += link;
+
+	return resultLink;
+}
+
+static bool getFavicon(CURLWrapper &CURL, const std::string &url, std::string &icon)
 {
 	icon.clear();
 
-	if (url.substr(0, 7) == "http://") {
-		int found = url.find("/", 7);
-		if (found >= 0) {
-			url.erase(found, url.length() - found);
-		}
-	} else {
-		return false;
-	}
-
 	bool result = false;
 
-	CURL *curl = curl_easy_init();
+	std::vector<unsigned char> vicon;
-	if (curl) {
+	CURLcode code = CURL.downloadBinary(calculateLink(url, "/favicon.ico"), vicon);
-		url += "/favicon.ico";
+	if (code == CURLE_OK) {
-
+		if (CURL.responseCode() == 200) {
-		std::vector<unsigned char> vicon;
+			std::string contentType = CURL.contentType();
-		curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 1);
+			if (isContentType(contentType, "image/x-icon") ||
-		curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
+				isContentType(contentType, "application/octet-stream") ||
-		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionBinary);
+				isContentType(contentType, "text/plain")) {
-		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &vicon);
+				if (!vicon.empty()) {
-		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
+					long todo; // check it
-		curl_easy_setopt(curl, CURLOPT_COOKIESESSION, 1);
+					result = toBase64(vicon, icon);
-
-		if (!proxy.empty()) {
-			curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str());
-		}
-
-		CURLcode code = curl_easy_perform(curl);
-		if (code == CURLE_OK) {
-			long response;
-			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
-			if (response == 200) {
-				char *contentType = NULL;
-				curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &contentType);
-				if (contentType &&
-					(strncasecmp(contentType, "image/x-icon", 12) == 0 ||
-					 strncasecmp(contentType, "application/octet-stream", 24) == 0 ||
-					 strncasecmp(contentType, "text/plain", 10) == 0)) {
-					if (!vicon.empty()) {
-						long todo; // check it
-						//  Set up a base64 encoding BIO that writes to a memory BIO
-						BIO *b64 = BIO_new(BIO_f_base64());
-						if (b64) {
-							BIO *out = BIO_new(BIO_s_mem());
-							if (out) {
-								BIO_set_flags(out, BIO_CLOSE);  // probably redundant
-								b64 = BIO_push(b64, out);
-								//  Send the data
-								BIO_write(b64, vicon.data(), vicon.size());
-								//  Collect the encoded data
-								BIO_flush(b64);
-								char* temp;
-								int count = BIO_get_mem_data(out, &temp);
-								if (count && temp) {
-									icon = temp;
-									result = true;
-								}
-							}
-							BIO_free_all(b64);
-						}
-					}
-//					char *encode = NULL;
-//					size_t encodeSize = 0;
-//					code = Curl_base64_encode(curl, (const char*) vicon.data(), vicon.size(), &encode, &encodeSize);
-//					if (code == CURLE_OK && encodeSize) {
-//						icon = encode;
-//						free(encode);
-//						encode = NULL;
-//						result = true;
-//					}
 				}
 			}
 		}
-
-		curl_easy_cleanup(curl);
-		curl = NULL;
 	}
 
 	return result;
@@ -245,68 +250,42 @@ p3FeedReaderThread::DownloadResult p3FeedReaderThread::download(const RsFeedRead
 
 	DownloadResult result;
 
-	CURL *curl = curl_easy_init();
+	std::string proxy = getProxyForFeed(feed);
-	if (curl) {
+	CURLWrapper CURL(proxy);
-		std::string proxy;
+	CURLcode code = CURL.downloadText(feed.url, content);
-		if (feed.flag & RS_FEED_FLAG_STANDARD_PROXY) {
-			std::string standardProxyAddress;
-			uint16_t standardProxyPort;
-			if (mFeedReader->getStandardProxy(standardProxyAddress, standardProxyPort)) {
-				rs_sprintf(proxy, "%s:%u", standardProxyAddress.c_str(), standardProxyPort);
-			}
-		} else {
-			if (!feed.proxyAddress.empty() && feed.proxyPort) {
-				rs_sprintf(proxy, "%s:%u", feed.proxyAddress.c_str(), feed.proxyPort);
-			}
-		}
 
-		curl_easy_setopt(curl, CURLOPT_NOPROGRESS, 0);
+	if (code == CURLE_OK) {
-		curl_easy_setopt(curl, CURLOPT_PROGRESSFUNCTION, progressCallback);
+		long responseCode = CURL.responseCode();
-		curl_easy_setopt(curl, CURLOPT_PROGRESSDATA, this);
-		curl_easy_setopt(curl, CURLOPT_URL, feed.url.c_str());
-		curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, writeFunctionString);
-		curl_easy_setopt(curl, CURLOPT_WRITEDATA, &content);
-		curl_easy_setopt(curl, CURLOPT_FOLLOWLOCATION, 1);
 
-		if (!proxy.empty()) {
+		switch (responseCode) {
-			curl_easy_setopt(curl, CURLOPT_PROXY, proxy.c_str());
+		case 200:
-		}
+			{
+				std::string contentType = CURL.contentType();
 
-		CURLcode code = curl_easy_perform(curl);
+				if (isContentType(contentType, "text/xml") ||
-		if (code == CURLE_OK) {
+					isContentType(contentType, "application/rss+xml") ||
-			long response;
+					isContentType(contentType, "application/xml") ||
-			curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &response);
+					isContentType(contentType, "application/xhtml+xml")) {
-			if (response == 200) {
-				char *contentType = NULL;
-				curl_easy_getinfo(curl, CURLINFO_CONTENT_TYPE, &contentType);
-				if (contentType &&
-					(strncasecmp(contentType, "text/xml", 8) == 0 ||
-					 strncasecmp(contentType, "application/rss+xml", 19) == 0 ||
-					 strncasecmp(contentType, "application/xml", 15) == 0 ||
-					 strncasecmp(contentType, "application/xhtml+xml", 21) == 0)) {
 					/* ok */
 					result = DOWNLOAD_SUCCESS;
 				} else {
 					result = DOWNLOAD_UNKNOWN_CONTENT_TYPE;
-					error = contentType ? contentType : "";
+					error = contentType;
 				}
-			} else if (response == 404) {
-				result = DOWNLOAD_NOT_FOUND;
-			} else {
-				result = DOWNLOAD_UNKOWN_RESPONSE_CODE;
-				rs_sprintf(error, "%ld", response);
 			}
-		} else {
+			break;
-			result = DOWNLOAD_ERROR;
+		case 404:
-			error = curl_easy_strerror(code);
+			result = DOWNLOAD_NOT_FOUND;
+			break;
+		default:
+			result = DOWNLOAD_UNKOWN_RESPONSE_CODE;
+			rs_sprintf(error, "%ld", responseCode);
 		}
 
-		curl_easy_cleanup(curl);
+		getFavicon(CURL, feed.url, icon);
-		curl = NULL;
-
-		getFavicon(feed.url, proxy, icon);
 	} else {
-		result = DOWNLOAD_ERROR_INIT;
+		result = DOWNLOAD_ERROR;
+		error = curl_easy_strerror(code);
 	}
 
 #ifdef FEEDREADER_DEBUG
@@ -320,12 +299,12 @@ p3FeedReaderThread::DownloadResult p3FeedReaderThread::download(const RsFeedRead
 /****************************** Process ************************************/
 /***************************************************************************/
 
-static bool convertOutput(xmlCharEncodingHandlerPtr charEncodingHandler, const xmlChar *output, std::string &text)
+static bool convertToString(xmlCharEncodingHandlerPtr charEncodingHandler, const xmlChar *xmlText, std::string &text)
 {
 	bool result = false;
 
+	xmlBufferPtr in = xmlBufferCreateStatic((void*) xmlText, xmlStrlen(xmlText));
 	xmlBufferPtr out = xmlBufferCreate();
-	xmlBufferPtr in = xmlBufferCreateStatic((void*) output, xmlStrlen(output));
 	int ret = xmlCharEncOutFunc(charEncodingHandler, out, in);
 	if (ret >= 0) {
 		result = true;
@@ -338,6 +317,24 @@ static bool convertOutput(xmlCharEncodingHandlerPtr charEncodingHandler, const x
 	return result;
 }
 
+static bool convertFromString(xmlCharEncodingHandlerPtr charEncodingHandler, const char *text, xmlChar *&xmlText)
+{
+	bool result = false;
+
+	xmlBufferPtr in = xmlBufferCreateStatic((void*) text, strlen(text));
+	xmlBufferPtr out = xmlBufferCreate();
+	int ret = xmlCharEncOutFunc(charEncodingHandler, out, in);
+	if (ret >= 0) {
+		result = true;
+		xmlText = xmlBufferDetach(out);
+	}
+
+	xmlBufferFree(in);
+	xmlBufferFree(out);
+
+	return result;
+}
+
 static xmlNodePtr findNode(xmlNodePtr node, const char *name, bool children = false)
 {
 	if (node->name) {
@@ -419,12 +416,46 @@ static bool getChildText(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler
 	}
 
 	if (child->children->content) {
-		return convertOutput((xmlCharEncodingHandlerPtr) charEncodingHandler, child->children->content, text);
+		return convertToString((xmlCharEncodingHandlerPtr) charEncodingHandler, child->children->content, text);
 	}
 
 	return true;
 }
 
+static std::string xmlGetAttr(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler, xmlNodePtr node, const char *name)
+{
+	if (!node || !name) {
+		return "";
+	}
+
+	std::string value;
+
+	xmlChar *xmlValue = xmlGetProp(node, (const xmlChar*) name);
+	if (xmlValue) {
+		convertToString((xmlCharEncodingHandlerPtr) charEncodingHandler, xmlValue, value);
+		xmlFree(xmlValue);
+	}
+
+	return value;
+}
+
+static bool xmlSetAttr(/*xmlCharEncodingHandlerPtr*/ void *charEncodingHandler, xmlNodePtr node, const char *name, const char *value)
+{
+	if (!node || !name) {
+		return false;
+	}
+
+	xmlChar *xmlValue = NULL;
+	if (!convertFromString((xmlCharEncodingHandlerPtr) charEncodingHandler, value, xmlValue)) {
+		return false;
+	}
+
+	xmlAttrPtr xmlAttr = xmlSetProp (node, (const xmlChar*) name, xmlValue);
+	xmlFree(xmlValue);
+
+	return xmlAttr != NULL;
+}
+
 static void splitString(std::string s, std::vector<std::string> &v, const char d)
 {
 	v.clear();
@@ -689,7 +720,7 @@ static time_t parseRFC822Date(const std::string &pubDate)
 	// broken mail-/news-clients omit the time zone
 	if (*dateString) {
 		if ((strncasecmp(dateString, "gmt", 3) == 0) ||
-				(strncasecmp(dateString, "utc", 3) == 0))
+			(strncasecmp(dateString, "utc", 3) == 0))
 		{
 			dateString += 3;
 			while(*dateString && isspace(*dateString))
@@ -942,7 +973,10 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
 						item->feedId = feed.feedId;
 						item->title = title;
 
-						getChildText(mCharEncodingHandler, node, "link", item->link);
+						/* try feedburner:origLink */
+						if (!getChildText(mCharEncodingHandler, node, "origLink", item->link) || item->link.empty()) {
+							getChildText(mCharEncodingHandler, node, "link", item->link);
+						}
 
 						long todo; // remove sid
 //						// remove sid=
@@ -969,6 +1003,7 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
 //						}
 
 						getChildText(mCharEncodingHandler, node, "author", item->author);
+
 						getChildText(mCharEncodingHandler, node, "description", item->description);
 
 						std::string pubDate;
@@ -1007,3 +1042,148 @@ p3FeedReaderThread::ProcessResult p3FeedReaderThread::process(const RsFeedReader
 
 	return result;
 }
+
+std::string p3FeedReaderThread::getProxyForFeed(const RsFeedReaderFeed &feed)
+{
+	std::string proxy;
+	if (feed.flag & RS_FEED_FLAG_STANDARD_PROXY) {
+		std::string standardProxyAddress;
+		uint16_t standardProxyPort;
+		if (mFeedReader->getStandardProxy(standardProxyAddress, standardProxyPort)) {
+			rs_sprintf(proxy, "%s:%u", standardProxyAddress.c_str(), standardProxyPort);
+		}
+	} else {
+		if (!feed.proxyAddress.empty() && feed.proxyPort) {
+			rs_sprintf(proxy, "%s:%u", feed.proxyAddress.c_str(), feed.proxyPort);
+		}
+	}
+	return proxy;
+}
+
+bool p3FeedReaderThread::processMsg(const RsFeedReaderFeed &feed, RsFeedReaderMsg *msg)
+{
+	if (!msg) {
+		return false;
+	}
+
+	std::string proxy = getProxyForFeed(feed);
+
+	std::string url;
+	if (feed.flag & RS_FEED_FLAG_SAVE_COMPLETE_PAGE) {
+#ifdef FEEDREADER_DEBUG
+		std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download page " << msg->link << std::endl;
+#endif
+		std::string content;
+		CURLWrapper CURL(proxy);
+		CURLcode code = CURL.downloadText(msg->link, content);
+
+		if (code == CURLE_OK && CURL.responseCode() == 200 && isContentType(CURL.contentType(), "text/html")) {
+			/* ok */
+			msg->description = content;
+		} else {
+#ifdef FEEDREADER_DEBUG
+			std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot download page, CURLCode = " << code << ", responseCode = " << CURL.responseCode() << ", contentType = " << CURL.contentType() << std::endl;
+#endif
+			return false;
+		}
+
+		/* get effective url (moved location) */
+		std::string effectiveUrl = CURL.effectiveUrl();
+		url = getBaseLink(effectiveUrl.empty() ? msg->link : effectiveUrl);
+	}
+
+	/* check if string contains xml chars (very simple test) */
+	if (msg->description.find('<') == std::string::npos) {
+		return true;
+	}
+
+	bool result = true;
+
+	/* process description */
+	long todo; // encoding
+	htmlDocPtr document = htmlReadMemory(msg->description.c_str(), msg->description.length(), url.c_str(), "", HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING | HTML_PARSE_COMPACT);
+	if (document) {
+		xmlNodePtr root = xmlDocGetRootElement(document);
+		if (root) {
+			/* process all children */
+			std::list<xmlNodePtr> parents;
+			parents.push_back(root);
+
+			while (!parents.empty()) {
+				if (!isRunning()) {
+					break;
+				}
+				xmlNodePtr node = parents.front();
+				parents.pop_front();
+
+				if (node->type == XML_ELEMENT_NODE) {
+					/* check for image */
+					if (strcasecmp((char*) node->name, "img") == 0) {
+						bool removeImage = true;
+
+						if (feed.flag & RS_FEED_FLAG_EMBED_IMAGES) {
+							/* embed image */
+							std::string src = xmlGetAttr(mCharEncodingHandler, node, "src");
+							if (!src.empty()) {
+								/* download image */
+#ifdef FEEDREADER_DEBUG
+								std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") download image " << src << std::endl;
+#endif
+								std::vector<unsigned char> data;
+								CURLWrapper CURL(proxy);
+								CURLcode code = CURL.downloadBinary(calculateLink(url, src), data);
+								if (code == CURLE_OK && CURL.responseCode() == 200) {
+									std::string contentType = CURL.contentType();
+									if (isContentType(contentType, "image/")) {
+										std::string base64;
+										if (toBase64(data, base64)) {
+											std::string imageBase64;
+											rs_sprintf(imageBase64, "data:%s;base64,%s", contentType.c_str(), base64.c_str());
+											if (xmlSetAttr(mCharEncodingHandler, node, "src", imageBase64.c_str())) {
+												removeImage = false;
+											}
+										}
+									}
+								}
+							}
+						}
+
+						if (removeImage) {
+							/* remove image */
+							xmlUnlinkNode(node);
+							xmlFreeNode(node);
+							continue;
+						}
+					}
+
+					xmlNodePtr child;
+					for (child = node->children; child; child = child->next) {
+						parents.push_back(child);
+					}
+				}
+			}
+
+			xmlChar *html = NULL;
+			int htmlSize = 0;
+			htmlDocDumpMemoryFormat(document, &html, &htmlSize, 0);
+			if (html) {
+				convertToString((xmlCharEncodingHandlerPtr) mCharEncodingHandler, html, msg->description);
+				xmlFree(html);
+			} else {
+#ifdef FEEDREADER_DEBUG
+				std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") cannot dump html" << std::endl;
+#endif
+				result = false;
+			}
+		} else {
+#ifdef FEEDREADER_DEBUG
+			std::cerr << "p3FeedReaderThread::processHTML - feed " << feed.feedId << " (" << feed.name << ") no root element" << std::endl;
+#endif
+			result = false;
+		}
+
+		xmlFreeDoc(document);
+	}
+
+	return result;
+}

plugins/FeedReader/services/p3FeedReaderThread.h

@@ -19,7 +19,7 @@
  *  Boston, MA  02110-1301, USA.
  ****************************************************************/
 
- #ifndef P3_FEEDREADERTHREAD
+#ifndef P3_FEEDREADERTHREAD
 #define P3_FEEDREADERTHREAD
 
 #include "util/rsthreads.h"
@@ -40,7 +40,6 @@ public:
 	enum DownloadResult
 	{
 		DOWNLOAD_SUCCESS,
-		DOWNLOAD_ERROR_INIT,
 		DOWNLOAD_ERROR,
 		DOWNLOAD_UNKNOWN_CONTENT_TYPE,
 		DOWNLOAD_NOT_FOUND,
@@ -64,6 +63,9 @@ private:
 	DownloadResult download(const RsFeedReaderFeed &feed, std::string &content, std::string &icon, std::string &error);
 	ProcessResult process(const RsFeedReaderFeed &feed, std::list<RsFeedReaderMsg*> &entries, std::string &error);
 
+	std::string getProxyForFeed(const RsFeedReaderFeed &feed);
+	bool processMsg(const RsFeedReaderFeed &feed, RsFeedReaderMsg *msg);
+
 	p3FeedReader *mFeedReader;
 	Type mType;
 	/*xmlCharEncodingHandlerPtr*/ void *mCharEncodingHandler;

plugins/FeedReader/services/rsFeedReaderItems.h

@@ -19,7 +19,7 @@
  *  Boston, MA  02110-1301, USA.
  ****************************************************************/
 
- #ifndef RS_FEEDREADER_ITEMS_H
+#ifndef RS_FEEDREADER_ITEMS_H
 #define RS_FEEDREADER_ITEMS_H
 
 #include "serialiser/rsserial.h"
@@ -55,6 +55,8 @@ const uint8_t RS_PKT_SUBTYPE_FEEDREADER_MSG   = 0x03;
 #define RS_FEED_FLAG_DEACTIVATED                   0x040
 #define RS_FEED_FLAG_FORUM                         0x080
 #define RS_FEED_FLAG_UPDATE_FORUM_INFO             0x100
+#define RS_FEED_FLAG_EMBED_IMAGES                  0x200
+#define RS_FEED_FLAG_SAVE_COMPLETE_PAGE            0x400
 
 class RsFeedReaderFeed : public RsItem
 {

plugins/FeedReader/services/util/CURLWrapper.cc

@@ -0,0 +1,136 @@
+/****************************************************************
+ *  RetroShare GUI is distributed under the following license:
+ *
+ *  Copyright (C) 2012 by Thunder
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#include "CURLWrapper.h"
+
+//static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/)
+//{
+//	p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp;
+
+//	if (!thread->isRunning()) {
+//		/* thread was stopped */
+//		return 1;
+//	}
+
+//	long todo; // show progress in gui
+
+//	return 0;
+//}
+
+CURLWrapper::CURLWrapper(const std::string &proxy)
+{
+	mCurl = curl_easy_init();
+	if (mCurl) {
+		curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 0);
+//		curl_easy_setopt(mCurl, CURLOPT_PROGRESSFUNCTION, progressCallback);
+//		curl_easy_setopt(mCurl, CURLOPT_PROGRESSDATA, feedReader);
+		curl_easy_setopt(mCurl, CURLOPT_FOLLOWLOCATION, 1);
+		curl_easy_setopt(mCurl, CURLOPT_CONNECTTIMEOUT, 60);
+		curl_easy_setopt(mCurl, CURLOPT_TIMEOUT, 120);
+
+		if (!proxy.empty()) {
+			curl_easy_setopt(mCurl, CURLOPT_PROXY, proxy.c_str());
+		}
+	}
+}
+
+CURLWrapper::~CURLWrapper()
+{
+	if (mCurl) {
+		curl_easy_cleanup(mCurl);
+	}
+}
+
+static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+	std::string *s = (std::string*) stream;
+	s->append ((char*) ptr, size * nmemb);
+
+	return nmemb * size;
+}
+
+CURLcode CURLWrapper::downloadText(const std::string &link, std::string &data)
+{
+	data.clear();
+
+	if (!mCurl) {
+		return CURLE_FAILED_INIT;
+	}
+
+	curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str());
+	curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionString);
+	curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data);
+
+	return curl_easy_perform(mCurl);
+}
+
+static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+	std::vector<unsigned char> *bytes = (std::vector<unsigned char>*) stream;
+
+	std::vector<unsigned char> newBytes;
+	newBytes.resize(size * nmemb);
+	memcpy(newBytes.data(), ptr, newBytes.size());
+
+	bytes->insert(bytes->end(), newBytes.begin(), newBytes.end());
+
+	return nmemb * size;
+}
+
+CURLcode CURLWrapper::downloadBinary(const std::string &link, std::vector<unsigned char> &data)
+{
+	data.clear();
+
+	if (!mCurl) {
+		return CURLE_FAILED_INIT;
+	}
+
+	curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 1);
+	curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str());
+	curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionBinary);
+	curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data);
+
+	return curl_easy_perform(mCurl);
+}
+
+long CURLWrapper::longInfo(CURLINFO info)
+{
+	if (!mCurl) {
+		return 0;
+	}
+
+	long value;
+	curl_easy_getinfo(mCurl, info, &value);
+
+	return value;
+}
+
+std::string CURLWrapper::stringInfo(CURLINFO info)
+{
+	if (!mCurl) {
+		return "";
+	}
+
+	char *value;
+	curl_easy_getinfo(mCurl, info, &value);
+
+	return value ? value : "";
+}

plugins/FeedReader/services/util/CURLWrapper.h

@@ -0,0 +1,50 @@
+/****************************************************************
+ *  RetroShare GUI is distributed under the following license:
+ *
+ *  Copyright (C) 2012 by Thunder
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#ifndef CURLWRAPPER
+#define CURLWRAPPER
+
+#include <string>
+#include <vector>
+#include <curl/curl.h>
+
+class CURLWrapper
+{
+public:
+	CURLWrapper(const std::string &proxy);
+	~CURLWrapper();
+
+	CURLcode downloadText(const std::string &link, std::string &data);
+	CURLcode downloadBinary(const std::string &link, std::vector<unsigned char> &data);
+
+	long responseCode() { return longInfo(CURLINFO_RESPONSE_CODE); }
+	std::string contentType() { return stringInfo(CURLINFO_CONTENT_TYPE); }
+	std::string effectiveUrl() { return stringInfo(CURLINFO_EFFECTIVE_URL); }
+
+protected:
+	long longInfo(CURLINFO info);
+	std::string stringInfo(CURLINFO info);
+
+private:
+	CURL *mCurl;
+};
+
+#endif