checked in first bits of code for cache service data optimisation,

distrib can build grps xml document
added pugixml src code into utilities

pls see design document:
http://retroshare.sourceforge.net/wiki/index.php/Documentation:design_services_histories

git-svn-id: http://svn.code.sf.net/p/retroshare/code/trunk@4076 b45a01b8-16f6-495d-af2f-9b41ad6348cc

libretroshare/src/dbase/cachestrapper.cc

@@ -259,6 +259,21 @@ bool	CacheStore::getStoredCache(CacheData &data)
 	return ok;
 }
 
+bool CacheStore::cached(const std::string cacheId)
+{
+
+	return false;
+}
+
+void CacheStore::updateCacheDocument(pugi::xml_document& cacheDoc)
+{
+	return;
+}
+
+void CacheStore::updateCacheTable()
+{
+	return;
+}
 
 bool	CacheStore::locked_getStoredCache(CacheData &data)
 {

libretroshare/src/dbase/cachestrapper.h

@@ -27,6 +27,7 @@
 #include "pqi/p3cfgmgr.h"
 #include "pqi/pqimonitor.h"
 #include "util/rsthreads.h"
+#include "util/pugixml.h"
 
 #include <list>
 #include <map>
@@ -217,6 +218,7 @@ class CacheSource
 
 /*!
  * Base Class for data cache. eg. FileCache/Store.
+ * This is best used to deal with external caches from other peers
  * @see p3Distrib. pqiMonitor
  */
 class CacheStore
@@ -313,7 +315,33 @@ class CacheStore
 		 */
 		bool    locked_getStoredCache(CacheData &data);
 
+
+		//////////////// Cache Optimisation //////////////////
+
+		/**
+		 *@param id the key for determing whether this data has been cached or not
+		 *@return true if data referenced by key has been cached false otherwise
+		 */
+		bool cached(const std::string cacheId);
+
+		/**
+		 * TODO: will be abstract
+		 * The deriving class should return a document which accurately reflects its data
+		 * structure
+		 * @param cacheDoc document reflecting derving class's cache data structure
+		 */
+		virtual void updateCacheDocument(pugi::xml_document& cacheDoc);
+
+
+
+
+		//////////////// Cache Optimisation //////////////////
 		private:
+		/**
+		 * This updates the cache table with information from the xml document
+		 *
+		 */
+		void updateCacheTable();
 
 		uint16_t cacheType;    /* for checking */
 		bool     multiCache;   /* do we care about subid's */
@@ -324,9 +352,16 @@ class CacheStore
 		std::string cacheDir;
 
 		mutable RsMutex cMutex;
-
 		std::map<RsPeerId, CacheSet> caches;
 
+		////////////// cache optimisation ////////////////
+
+		/// whether to run in cache optimisation mode
+		bool cacheOptMode;
+
+		/// stores whether given instance of cache data has been loaded already or not
+		std::map<std::string, bool>  cacheTable;
+
 };
 
 

libretroshare/src/libretroshare.pro

@@ -416,6 +416,8 @@ HEADERS +=	util/folderiterator.h \
 			util/rsversion.h \
 			util/rswin.h \
 			util/rsrandom.h \
+			util/pugiconfig.h \
+			util/pugixml.h
 
 SOURCES +=	dbase/cachestrapper.cc \
 			dbase/fimonitor.cc \
@@ -536,7 +538,8 @@ SOURCES +=	util/folderiterator.cc \
 			util/rsthreads.cc \
 			util/rsversion.cc \
 			util/rswin.cc \
-			util/rsrandom.cc
+			util/rsrandom.cc \
+			util/pugixml.cc
 
 minimal {
 	SOURCES -= rsserver/p3msgs.cc \

libretroshare/src/services/p3distrib.cc

@@ -155,6 +155,15 @@ int	p3GroupDistrib::tick()
                 receivePubKeys();
 	}
 
+	// update cache table every minute
+	bool updateCacheDoc = false;
+	{
+		updateCacheDoc = (now > (time_t) (mLastCacheDocUpdate + 60.));
+	}
+
+	if(false)
+		updateCacheDocument(mCacheDoc);
+
 	return 0;
 }
 
@@ -221,6 +230,66 @@ bool 	p3GroupDistrib::loadLocalCache(const CacheData &data)
 }
 
 
+void p3GroupDistrib::updateCacheDocument(pugi::xml_document& cacheDoc)
+{
+
+#ifdef DISTRIB_DEBUG
+	std::cerr << "p3GroupDistrib::updateCacheDocument() "
+			  << std::endl;
+#endif
+
+	/*
+	 * find group ids put down in mGroupCacheIds to be set into
+	 * the cache document then their subsequent messages
+	 */
+
+	std::map<std::string, std::string>::iterator cit =
+			mGroupCacheIds.begin(), mit = mMsgCacheIds.begin();
+
+	std::map<std::string, GroupInfo>::iterator git;
+	std::map<std::string, RsDistribMsg *>::iterator msgIt;
+
+
+	for(; cit != mGroupCacheIds.end(); cit++){
+
+		// add the group at depth on
+		git = mGroups.find(cit->first);
+
+		if(git != mGroups.end()){
+
+			// add another group node
+			cacheDoc.append_child("group");
+
+			// then add title
+			cacheDoc.last_child().append_child("title").append_child(
+					pugi::node_pcdata).set_value(git->second.grpId.c_str());
+
+			// then add the id
+			cacheDoc.last_child().append_child("id").append_child(
+					pugi::node_pcdata).set_value(cit->second
+							.c_str());
+
+		}
+	}
+
+#ifdef DISTRIB_DEBUG
+	std::cerr << "p3GroupDistrib::updateCacheDocument() "
+			  << "\nFinished Building xml doc, \n saving to file"
+			  << std::endl;
+#endif
+
+
+	cacheDoc.save_file("distrib_group.xml");
+	mLastCacheDocUpdate = time(NULL);
+
+	// essentially build by gathering all your group
+
+	// now do the same with messages which may have arrived for a group
+
+
+}
+
+
 /* Handle the Cache Pending Setup */
 CacheDataPending::CacheDataPending(const CacheData &data, bool local, bool historical)
 	:mData(data), mLocal(local), mHistorical(historical)
@@ -376,6 +445,7 @@ void	p3GroupDistrib::loadFileGroups(const std::string &filename, const std::stri
 		newKey = dynamic_cast<RsDistribGrpKey *>(item);
 		if ((newGrp = dynamic_cast<RsDistribGrp *>(item)))
 		{
+
 			loadGroup(newGrp, historical);
 		}
 		else if ((newKey = dynamic_cast<RsDistribGrpKey *>(item)))

libretroshare/src/services/p3distrib.h

@@ -314,6 +314,10 @@ class p3GroupDistrib: public CacheSource, public CacheStore, public p3Config, pu
 
 		void 	HistoricalCachesDone(); // called when Stored Caches have been added to Pending List.
 
+	protected:
+
+		void updateCacheDocument(pugi::xml_document& cacheDoc);
+
 	private:
 
 		/* these lists are filled by the overloaded fns... then cleared by the thread */
@@ -755,6 +759,14 @@ RsDistribDummyMsg *locked_getGroupDummyMsg(std::string grpId, std::string msgId)
 		std::map<std::string, std::list<std::string> > mPendingPubKeyRecipients; /// peers to receive publics key for a given grp
                 std::set<std::string> mPubKeyAvailableGrpId; // groups id for which public keys are available
 		time_t mLastKeyPublishTime, mLastRecvdKeyTime;
+
+		////////////// cache optimisation ////////////////
+
+		/// look table to calculate data location
+		std::map<std::string, std::string> mGroupCacheIds, mMsgCacheIds;
+		std::map<std::string, std::string> mGroupMessageIds;
+		time_t mLastCacheDocUpdate;
+		pugi::xml_document mCacheDoc;
 	
 };
 

libretroshare/src/util/pugiconfig.h

@@ -0,0 +1,62 @@
+/**
+ * pugixml parser - version 1.0
+ * --------------------------------------------------------
+ * Copyright (C) 2006-2010, by Arseny Kapoulkine (arseny.kapoulkine@gmail.com)
+ * Report bugs and download new versions at http://pugixml.org/
+ *
+ * This library is distributed under the MIT License. See notice at the end
+ * of this file.
+ *
+ * This work is based on the pugxml parser, which is:
+ * Copyright (C) 2003, by Kristen Wegner (kristen@tima.net)
+ */
+
+#ifndef HEADER_PUGICONFIG_HPP
+#define HEADER_PUGICONFIG_HPP
+
+// Uncomment this to enable wchar_t mode
+// #define PUGIXML_WCHAR_MODE
+
+// Uncomment this to disable XPath
+// #define PUGIXML_NO_XPATH
+
+// Uncomment this to disable STL
+// Note: you can't use XPath with PUGIXML_NO_STL
+// #define PUGIXML_NO_STL
+
+// Uncomment this to disable exceptions
+// Note: you can't use XPath with PUGIXML_NO_EXCEPTIONS
+// #define PUGIXML_NO_EXCEPTIONS
+
+// Set this to control attributes for public classes/functions, i.e.:
+// #define PUGIXML_API __declspec(dllexport) // to export all public symbols from DLL
+// #define PUGIXML_CLASS __declspec(dllimport) // to import all classes from DLL
+// #define PUGIXML_FUNCTION __fastcall // to set calling conventions to all public functions to fastcall
+// In absence of PUGIXML_CLASS/PUGIXML_FUNCTION definitions PUGIXML_API is used instead
+
+#endif
+
+/**
+ * Copyright (c) 2006-2010 Arseny Kapoulkine
+ *
+ * Permission is hereby granted, free of charge, to any person
+ * obtaining a copy of this software and associated documentation
+ * files (the "Software"), to deal in the Software without
+ * restriction, including without limitation the rights to use,
+ * copy, modify, merge, publish, distribute, sublicense, and/or sell
+ * copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following
+ * conditions:
+ *
+ * The above copyright notice and this permission notice shall be
+ * included in all copies or substantial portions of the Software.
+ * 
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+ * OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+ * HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+ * WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+ * OTHER DEALINGS IN THE SOFTWARE.
+ */