FeedReader plugin:

- added new classes for XML/HTML parse and modify
- added basic error handling
- added new GUI for a preview and a tree to show the structure of the page (will be continued)

git-svn-id: http://svn.code.sf.net/p/retroshare/code/branches/v0.5-gxs-b1@5412 b45a01b8-16f6-495d-af2f-9b41ad6348cc

plugins/FeedReader/util/CURLWrapper.cpp

@@ -0,0 +1,136 @@
+/****************************************************************
+ *  RetroShare GUI is distributed under the following license:
+ *
+ *  Copyright (C) 2012 by Thunder
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#include "CURLWrapper.h"
+
+//static int progressCallback (void *clientp, double /*dltotal*/, double /*dlnow*/, double /*ultotal*/, double /*ulnow*/)
+//{
+//	p3FeedReaderThread *thread = (p3FeedReaderThread*) clientp;
+
+//	if (!thread->isRunning()) {
+//		/* thread was stopped */
+//		return 1;
+//	}
+
+//	long todo; // show progress in gui
+
+//	return 0;
+//}
+
+CURLWrapper::CURLWrapper(const std::string &proxy)
+{
+	mCurl = curl_easy_init();
+	if (mCurl) {
+		curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 0);
+//		curl_easy_setopt(mCurl, CURLOPT_PROGRESSFUNCTION, progressCallback);
+//		curl_easy_setopt(mCurl, CURLOPT_PROGRESSDATA, feedReader);
+		curl_easy_setopt(mCurl, CURLOPT_FOLLOWLOCATION, 1);
+		curl_easy_setopt(mCurl, CURLOPT_CONNECTTIMEOUT, 60);
+		curl_easy_setopt(mCurl, CURLOPT_TIMEOUT, 120);
+
+		if (!proxy.empty()) {
+			curl_easy_setopt(mCurl, CURLOPT_PROXY, proxy.c_str());
+		}
+	}
+}
+
+CURLWrapper::~CURLWrapper()
+{
+	if (mCurl) {
+		curl_easy_cleanup(mCurl);
+	}
+}
+
+static size_t writeFunctionString (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+	std::string *s = (std::string*) stream;
+	s->append ((char*) ptr, size * nmemb);
+
+	return nmemb * size;
+}
+
+CURLcode CURLWrapper::downloadText(const std::string &link, std::string &data)
+{
+	data.clear();
+
+	if (!mCurl) {
+		return CURLE_FAILED_INIT;
+	}
+
+	curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str());
+	curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionString);
+	curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data);
+
+	return curl_easy_perform(mCurl);
+}
+
+static size_t writeFunctionBinary (void *ptr, size_t size, size_t nmemb, void *stream)
+{
+	std::vector<unsigned char> *bytes = (std::vector<unsigned char>*) stream;
+
+	std::vector<unsigned char> newBytes;
+	newBytes.resize(size * nmemb);
+	memcpy(newBytes.data(), ptr, newBytes.size());
+
+	bytes->insert(bytes->end(), newBytes.begin(), newBytes.end());
+
+	return nmemb * size;
+}
+
+CURLcode CURLWrapper::downloadBinary(const std::string &link, std::vector<unsigned char> &data)
+{
+	data.clear();
+
+	if (!mCurl) {
+		return CURLE_FAILED_INIT;
+	}
+
+	curl_easy_setopt(mCurl, CURLOPT_NOPROGRESS, 1);
+	curl_easy_setopt(mCurl, CURLOPT_URL, link.c_str());
+	curl_easy_setopt(mCurl, CURLOPT_WRITEFUNCTION, writeFunctionBinary);
+	curl_easy_setopt(mCurl, CURLOPT_WRITEDATA, &data);
+
+	return curl_easy_perform(mCurl);
+}
+
+long CURLWrapper::longInfo(CURLINFO info)
+{
+	if (!mCurl) {
+		return 0;
+	}
+
+	long value;
+	curl_easy_getinfo(mCurl, info, &value);
+
+	return value;
+}
+
+std::string CURLWrapper::stringInfo(CURLINFO info)
+{
+	if (!mCurl) {
+		return "";
+	}
+
+	char *value;
+	curl_easy_getinfo(mCurl, info, &value);
+
+	return value ? value : "";
+}

plugins/FeedReader/util/CURLWrapper.h

@@ -0,0 +1,50 @@
+/****************************************************************
+ *  RetroShare GUI is distributed under the following license:
+ *
+ *  Copyright (C) 2012 by Thunder
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#ifndef CURLWRAPPER
+#define CURLWRAPPER
+
+#include <string>
+#include <vector>
+#include <curl/curl.h>
+
+class CURLWrapper
+{
+public:
+	CURLWrapper(const std::string &proxy);
+	~CURLWrapper();
+
+	CURLcode downloadText(const std::string &link, std::string &data);
+	CURLcode downloadBinary(const std::string &link, std::vector<unsigned char> &data);
+
+	long responseCode() { return longInfo(CURLINFO_RESPONSE_CODE); }
+	std::string contentType() { return stringInfo(CURLINFO_CONTENT_TYPE); }
+	std::string effectiveUrl() { return stringInfo(CURLINFO_EFFECTIVE_URL); }
+
+protected:
+	long longInfo(CURLINFO info);
+	std::string stringInfo(CURLINFO info);
+
+private:
+	CURL *mCurl;
+};
+
+#endif 

plugins/FeedReader/util/HTMLWrapper.cpp

@@ -0,0 +1,60 @@
+/****************************************************************
+ *  RetroShare GUI is distributed under the following license:
+ *
+ *  Copyright (C) 2012 by Thunder
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#include <string.h>
+
+#include "HTMLWrapper.h"
+#include <libxml/HTMLtree.h>
+
+HTMLWrapper::HTMLWrapper() : XMLWrapper()
+{
+}
+
+bool HTMLWrapper::readHTML(const char *html, const char *url)
+{
+	cleanup();
+
+	mDocument = htmlReadMemory(html, strlen(html), url, "", HTML_PARSE_NOERROR | HTML_PARSE_NOWARNING/* | HTML_PARSE_COMPACT*/);
+	if (mDocument) {
+		return true;
+	}
+
+	return false;
+}
+
+bool HTMLWrapper::saveHTML(std::string &html)
+{
+	if (!mDocument) {
+		return false;
+	}
+
+	xmlChar *newHtml = NULL;
+	int newHtmlSize = 0;
+	htmlDocDumpMemoryFormat(mDocument, &newHtml, &newHtmlSize, 0);
+	if (newHtml) {
+		convertToString(newHtml, html);
+		xmlFree(newHtml);
+
+		return true;
+	}
+
+	return false;
+}

plugins/FeedReader/util/HTMLWrapper.h

@@ -0,0 +1,36 @@
+/****************************************************************
+ *  RetroShare GUI is distributed under the following license:
+ *
+ *  Copyright (C) 2012 by Thunder
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#ifndef HTMLWRAPPER
+#define HTMLWRAPPER
+
+#include "XMLWrapper.h"
+
+class HTMLWrapper : public XMLWrapper
+{
+public:
+	HTMLWrapper();
+
+	bool readHTML(const char *html, const char *url);
+	bool saveHTML(std::string &html);
+};
+
+#endif 

plugins/FeedReader/util/XMLWrapper.cpp

@@ -0,0 +1,245 @@
+/****************************************************************
+ *  RetroShare GUI is distributed under the following license:
+ *
+ *  Copyright (C) 2012 by Thunder
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#include <iostream>
+#include <string.h>
+
+#include "XMLWrapper.h"
+
+XMLWrapper::XMLWrapper()
+{
+	mDocument = NULL;
+	mCharEncodingHandler = xmlFindCharEncodingHandler ("UTF8");
+
+	if (!mCharEncodingHandler) {
+		/* no encoding handler found */
+		std::cerr << "XMLWrapper::XMLWrapper - no encoding handler found" << std::endl;
+	}
+}
+
+XMLWrapper::~XMLWrapper()
+{
+	cleanup();
+	xmlCharEncCloseFunc(mCharEncodingHandler);
+}
+
+void XMLWrapper::cleanup()
+{
+	if (mDocument) {
+		xmlFreeDoc(mDocument);
+		mDocument = NULL;
+	}
+}
+
+bool XMLWrapper::convertToString(const xmlChar *xmlText, std::string &text)
+{
+	bool result = false;
+
+	xmlBufferPtr in = xmlBufferCreateStatic((void*) xmlText, xmlStrlen(xmlText));
+	xmlBufferPtr out = xmlBufferCreate();
+	int ret = xmlCharEncOutFunc(mCharEncodingHandler, out, in);
+	if (ret >= 0) {
+		result = true;
+		text = (char*) xmlBufferContent(out);
+	}
+
+	xmlBufferFree(in);
+	xmlBufferFree(out);
+
+	return result;
+}
+
+bool XMLWrapper::convertFromString(const char *text, xmlChar *&xmlText)
+{
+	bool result = false;
+
+	xmlBufferPtr in = xmlBufferCreateStatic((void*) text, strlen(text));
+	xmlBufferPtr out = xmlBufferCreate();
+	int ret = xmlCharEncOutFunc(mCharEncodingHandler, out, in);
+	if (ret >= 0) {
+		result = true;
+		xmlText = xmlBufferDetach(out);
+	}
+
+	xmlBufferFree(in);
+	xmlBufferFree(out);
+
+	return result;
+}
+
+xmlNodePtr XMLWrapper::getRootElement()
+{
+	if (mDocument) {
+		return xmlDocGetRootElement(mDocument);
+	}
+
+	return NULL;
+}
+
+bool XMLWrapper::readXML(const char *xml)
+{
+	cleanup();
+
+	mDocument = xmlReadDoc((const xmlChar*) xml, "", NULL, XML_PARSE_NOERROR | XML_PARSE_NOWARNING/* | XML_PARSE_COMPACT | XML_PARSE_NOCDATA*/);
+	if (mDocument) {
+		return true;
+	}
+
+	return false;
+}
+
+bool XMLWrapper::getContent(xmlNodePtr node, std::string &content)
+{
+	content.clear();
+
+	if (!node) {
+		return false;
+	}
+
+	xmlChar *xmlContent = xmlNodeListGetString(mDocument, node, 1);
+	if (!xmlContent) {
+		return true;
+	}
+
+	bool result = convertToString(xmlContent, content);
+	xmlFree(xmlContent);
+
+	return result;
+}
+
+std::string XMLWrapper::nodeName(xmlNodePtr node)
+{
+	std::string name;
+
+	if (node) {
+		convertToString(node->name, name);
+	}
+
+	return name;
+}
+
+std::string XMLWrapper::attrName(xmlAttrPtr attr)
+{
+	std::string name;
+
+	if (attr) {
+		convertToString(attr->name, name);
+	}
+
+	return name;
+}
+
+xmlNodePtr XMLWrapper::findNode(xmlNodePtr node, const char *name, bool children)
+{
+	if (node->name) {
+		if (xmlStrcasecmp(node->name, (xmlChar*) name) == 0) {
+			return node;
+		}
+	}
+
+	xmlNodePtr nodeFound = NULL;
+	if (children) {
+		if (node->children) {
+			nodeFound = findNode(node->children, name, children);
+			if (nodeFound) {
+				return nodeFound;
+			}
+		}
+	}
+
+	if (node->next) {
+		nodeFound = findNode(node->next, name, children);
+		if (nodeFound) {
+			return nodeFound;
+		}
+	}
+
+	return NULL;
+}
+
+bool XMLWrapper::getChildText(xmlNodePtr node, const char *childName, std::string &text)
+{
+	if (node == NULL || node->children == NULL) {
+		return false;
+	}
+
+	xmlNodePtr child = findNode(node->children, childName, true);
+	if (!child) {
+		return false;
+	}
+
+	if (child->type != XML_ELEMENT_NODE) {
+		return false;
+	}
+
+	if (!child->children) {
+		return false;
+	}
+
+	if (child->children->type != XML_TEXT_NODE) {
+		return false;
+	}
+
+	if (child->children->content) {
+		return convertToString(child->children->content, text);
+	}
+
+	return true;
+}
+
+std::string XMLWrapper::getAttr(xmlNodePtr node, xmlAttrPtr attr)
+{
+	return getAttr(node, (const char*) attr->name);
+}
+
+std::string XMLWrapper::getAttr(xmlNodePtr node, const char *name)
+{
+	if (!node || !name) {
+		return "";
+	}
+
+	std::string value;
+
+	xmlChar *xmlValue = xmlGetProp(node, (const xmlChar*) name);
+	if (xmlValue) {
+		convertToString(xmlValue, value);
+		xmlFree(xmlValue);
+	}
+
+	return value;
+}
+
+bool XMLWrapper::setAttr(xmlNodePtr node, const char *name, const char *value)
+{
+	if (!node || !name) {
+		return false;
+	}
+
+	xmlChar *xmlValue = NULL;
+	if (!convertFromString(value, xmlValue)) {
+		return false;
+	}
+
+	xmlAttrPtr xmlAttr = xmlSetProp (node, (const xmlChar*) name, xmlValue);
+	xmlFree(xmlValue);
+
+	return xmlAttr != NULL;
+}

plugins/FeedReader/util/XMLWrapper.h

@@ -0,0 +1,60 @@
+/****************************************************************
+ *  RetroShare GUI is distributed under the following license:
+ *
+ *  Copyright (C) 2012 by Thunder
+ *
+ *  This program is free software; you can redistribute it and/or
+ *  modify it under the terms of the GNU General Public License
+ *  as published by the Free Software Foundation; either version 2
+ *  of the License, or (at your option) any later version.
+ *
+ *  This program is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with this program; if not, write to the Free Software
+ *  Foundation, Inc., 51 Franklin Street, Fifth Floor,
+ *  Boston, MA  02110-1301, USA.
+ ****************************************************************/
+
+#ifndef XMLWRAPPER
+#define XMLWRAPPER
+
+#include <string>
+#include <libxml/parser.h>
+
+class XMLWrapper
+{
+public:
+	XMLWrapper();
+	~XMLWrapper();
+
+	void cleanup();
+
+	bool readXML(const char *xml);
+
+	std::string nodeName(xmlNodePtr node);
+	std::string attrName(xmlAttrPtr attr);
+
+	xmlNodePtr findNode(xmlNodePtr node, const char *name, bool children = false);
+	bool getChildText(xmlNodePtr node, const char *childName, std::string &text);
+
+	bool getContent(xmlNodePtr node, std::string &content);
+
+	std::string getAttr(xmlNodePtr node, xmlAttrPtr attr);
+	std::string getAttr(xmlNodePtr node, const char *name);
+	bool setAttr(xmlNodePtr node, const char *name, const char *value);
+
+	xmlNodePtr getRootElement();
+
+protected:
+	xmlDocPtr mDocument;
+	xmlCharEncodingHandlerPtr mCharEncodingHandler;
+
+	bool convertToString(const xmlChar *xmlText, std::string &text);
+	bool convertFromString(const char *text, xmlChar *&xmlText);
+};
+
+#endif